From a8d04943247ca02cd30e1f451235d01100fa10fe Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Thu, 29 Sep 2022 14:46:21 -0400 Subject: [PATCH 1/8] first pass tweaks for local dev prefs and foibles --- .gitignore | 6 ++++++ README.md | 12 +++++++++++- app/build_containers.py | 7 ++++++- compose/docker-compose-go-ethereum-foundry.yml | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6137a646 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.idea +venv +.vscode +laconic-so +laconic_stack_orchestrator.egg-info +__pycache__ \ No newline at end of file diff --git a/README.md b/README.md index 94fd9fc6..69e7e234 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,21 @@ Ensure that the following are already installed: $ python3 --version Python 3.8.10 ``` -1. Docker (Install a current version from dockerco, don't use the version from any Linux distro) +2. Docker (Install a current version from dockerco, don't use the version from any Linux distro) ``` $ docker --version Docker version 20.10.17, build 100c701 ``` +3. If installed from regular package repository, be aware that the compose plugin may need to be installed, as well. + ``` + DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} + mkdir -p $DOCKER_CONFIG/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.11.2/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose + chmod +x ~/.docker/cli-plugins/docker-compose + + # see https://docs.docker.com/compose/install/linux/#install-the-plugin-manually for further details + # or to install for all users. + ``` #### Install 1. Clone this repository: ``` diff --git a/app/build_containers.py b/app/build_containers.py index 25e8f8dc..8ab3e77f 100644 --- a/app/build_containers.py +++ b/app/build_containers.py @@ -41,8 +41,13 @@ def command(ctx, include, exclude): quiet = ctx.obj.quiet verbose = ctx.obj.verbose dry_run = ctx.obj.dry_run + local_stack = ctx.obj.local_stack - dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + if local_stack: + dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]) + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) if not quiet: print(f'Dev Root is: {dev_root_path}') diff --git a/compose/docker-compose-go-ethereum-foundry.yml b/compose/docker-compose-go-ethereum-foundry.yml index c4a496b5..20d008f9 100644 --- a/compose/docker-compose-go-ethereum-foundry.yml +++ b/compose/docker-compose-go-ethereum-foundry.yml @@ -18,7 +18,7 @@ services: DB_HOST: ipld-eth-db DB_PORT: 5432 DB_PASSWORD: password - DB_WRITE: true + DB_WRITE: "true" DB_TYPE: postgres DB_DRIVER: sqlx DB_WAIT_FOR_SYNC: "true" From 273a5c9f1de59ec919553ad5961f2fab2bc70567 Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Thu, 29 Sep 2022 14:51:29 -0400 Subject: [PATCH 2/8] adding local_stack option for repository setup --- app/setup_repositories.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/setup_repositories.py b/app/setup_repositories.py index f595e7dd..7861aac8 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -68,7 +68,13 @@ def command(ctx, check_only, pull, branches_file): if verbose: print(f"Branches are: {branches}") - dev_root_path = os.path.expanduser(config("DEV_ROOT", default="~/cerc")) + local_stack = ctx.obj.local_stack + + if local_stack: + dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]) + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) if not quiet: print(f'Dev Root is: {dev_root_path}') From b1ee021115a563b0c8a54f02bfc065ab59693b7d Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Thu, 29 Sep 2022 15:05:28 -0400 Subject: [PATCH 3/8] incomplete local_stack option --- cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cli.py b/cli.py index 057ce1a3..04cde0ce 100644 --- a/cli.py +++ b/cli.py @@ -22,20 +22,23 @@ from app import deploy_system CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) class Options(object): - def __init__(self, quiet, verbose, dry_run): + def __init__(self, quiet, verbose, dry_run, local_stack): self.quiet = quiet self.verbose = verbose self.dry_run = dry_run + self.local_stack = local_stack @click.group(context_settings=CONTEXT_SETTINGS) @click.option('--quiet', is_flag=True, default=False) @click.option('--verbose', is_flag=True, default=False) @click.option('--dry-run', is_flag=True, default=False) +@click.option('--local_stack', is_flag=True, default=False) + # See: https://click.palletsprojects.com/en/8.1.x/complex/#building-a-git-clone @click.pass_context -def cli(ctx, quiet, verbose, dry_run): +def cli(ctx, quiet, verbose, dry_run, local_stack): """Laconic Stack Orchestrator""" - ctx.obj = Options(quiet, verbose, dry_run) + ctx.obj = Options(quiet, verbose, dry_run, local_stack) cli.add_command(setup_repositories.command,"setup-repositories") cli.add_command(build_containers.command,"build-containers") From 27009ba16f43789bcdc9bb547ade8d0ec7288509 Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Thu, 29 Sep 2022 15:43:43 -0400 Subject: [PATCH 4/8] fixes and inclusion of naive first time setup script --- .gitignore | 3 ++- app/build_containers.py | 2 +- app/setup_repositories.py | 2 +- first_time_setup.sh | 8 ++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100755 first_time_setup.sh diff --git a/.gitignore b/.gitignore index 6137a646..33e988a5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ venv .vscode laconic-so laconic_stack_orchestrator.egg-info -__pycache__ \ No newline at end of file +__pycache__ +*~ \ No newline at end of file diff --git a/app/build_containers.py b/app/build_containers.py index 8ab3e77f..abe909ac 100644 --- a/app/build_containers.py +++ b/app/build_containers.py @@ -44,7 +44,7 @@ def command(ctx, include, exclude): local_stack = ctx.obj.local_stack if local_stack: - dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]) + dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') else: dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) diff --git a/app/setup_repositories.py b/app/setup_repositories.py index 7861aac8..ead3631d 100644 --- a/app/setup_repositories.py +++ b/app/setup_repositories.py @@ -71,7 +71,7 @@ def command(ctx, check_only, pull, branches_file): local_stack = ctx.obj.local_stack if local_stack: - dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]) + dev_root_path = default=os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') else: dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) diff --git a/first_time_setup.sh b/first_time_setup.sh new file mode 100755 index 00000000..74ebee0d --- /dev/null +++ b/first_time_setup.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +python3 -m venv venv +source ./venv/bin/activate +pip install --editable . +pip install shiv +shiv -c laconic-so -o laconic-so +./laconic-so --verbose --local_stack setup-repositories From 16793ad43a950ad3a2411740b25baf6e900a51fd Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Thu, 29 Sep 2022 15:47:08 -0400 Subject: [PATCH 5/8] missing period --- first_time_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/first_time_setup.sh b/first_time_setup.sh index 74ebee0d..2f7d46ab 100755 --- a/first_time_setup.sh +++ b/first_time_setup.sh @@ -4,5 +4,5 @@ python3 -m venv venv source ./venv/bin/activate pip install --editable . pip install shiv -shiv -c laconic-so -o laconic-so +shiv -c laconic-so -o laconic-so . ./laconic-so --verbose --local_stack setup-repositories From fda5ceaf139e11fbccf9e2609aefcf49e3b1a3fa Mon Sep 17 00:00:00 2001 From: Michael Shaw Date: Thu, 29 Sep 2022 15:56:55 -0400 Subject: [PATCH 6/8] README updates for compose plugin, running local build of ./laconic-so, and first_time_setup.sh --- README.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 69e7e234..96dcccfb 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Ensure that the following are already installed: $ docker --version Docker version 20.10.17, build 100c701 ``` -3. If installed from regular package repository, be aware that the compose plugin may need to be installed, as well. +3. If installed from regular package repository, BE AWARE that the compose plugin may need to be installed, as well. ``` DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} mkdir -p $DOCKER_CONFIG/cli-plugins @@ -35,21 +35,29 @@ Ensure that the following are already installed: ``` $ git clone (https://github.com/cerc-io/stack-orchestrator.git ``` -1. Enter the project directory: +2. Optional first time setup for empty dev root and fresh SO checkout: + ``` + ./first_time_setup.sh + # e.g. /home/USER/workspace1/ + # only contains the stack-orchestrator repo cloned in step 1. + # this will naively attempt to setup and activate the venv, shiv, generate a LOCAL standalone laconic-so + # and then setup-repositories in workspace1/ + ``` +4. Enter the project directory: ``` $ cd stack-orchestrator ``` -1. Create and activate a venv: +5. Create and activate a venv: ``` $ python3 -m venv venv $ source ./venv/bin/activate (venv) $ ``` -1. Install the cli in edit mode: +6. Install the cli in edit mode: ``` $ pip install --editable . ``` -1. Verify installation: +7. Verify installation: ``` (venv) $ laconic-so Usage: laconic-so [OPTIONS] COMMAND [ARGS]... @@ -102,17 +110,23 @@ _write-me_ ## Usage There are three sub-commands: `setup-repositories`, `build-containers` and `deploy-system` that are generally run in order: + +Note: $ laconic-so will run the version installed to ~/bin, while ./laconic-so can be invoked to run locally built +version in a checkout ### Setup Repositories Clones the set of git repositories necessary to build a system. Note: the use of `ssh-agent` is recommended in order to avoid entering your ssh key passphrase for each repository. ``` -$ laconic-so --verbose setup-repositories +$ laconic-so --verbose setup-repositories #this will default to ~/cerc or CERC_REPO_BASE_DIR from an env file +#$ ./laconic-so --verbose --local_stack setup-repositories #this will use cwd ../ as dev_root_path ``` ### Build Containers Builds the set of docker container images required to run a system. It takes around 10 minutes to build all the containers from cold. ``` -$ laconic-so --verbose build-containers +$ laconic-so --verbose build-containers #this will default to ~/cerc or CERC_REPO_BASE_DIR from an env file +#$ ./laconic-so --verbose --local_stack build-containers #this will use cwd ../ as dev_root_path + ``` ### Deploy System Uses `docker compose` to deploy a system. @@ -124,6 +138,8 @@ $ laconic-so --verbose deploy-system --include db-sharding,contract,ipld-eth-ser ``` $ laconic-so --verbose deploy-system --include db-sharding,contract,ipld-eth-server,go-ethereum-foundry down ``` +Note: deploy-system command interacts with most recently built container images. + ## Platform Support Native aarm64 is _not_ currently supported. x64 emulation on ARM64 macos should work (not yet tested). ## Implementation From deb40663961b645d00b9783b67830425d73ca5d6 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 29 Sep 2022 15:21:50 -0600 Subject: [PATCH 7/8] trailing newline --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 33e988a5..a1dcd886 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ venv laconic-so laconic_stack_orchestrator.egg-info __pycache__ -*~ \ No newline at end of file +*~ + From c3acdf9d5f06e645efc3c8ab8d7a97ac4a48e643 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 29 Sep 2022 15:28:23 -0600 Subject: [PATCH 8/8] Clarify non-Docker Desktop install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96dcccfb..ead98ee4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Ensure that the following are already installed: $ docker --version Docker version 20.10.17, build 100c701 ``` -3. If installed from regular package repository, BE AWARE that the compose plugin may need to be installed, as well. +3. If installed from regular package repository (not Docker Desktop), BE AWARE that the compose plugin may need to be installed, as well. ``` DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} mkdir -p $DOCKER_CONFIG/cli-plugins