From 6569a2a2b6193ed3dbafa5d64ceaa688b20e0914 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 11:53:05 -0700 Subject: [PATCH 1/7] Update to use stack syntax --- app/data/stacks/erc20/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/data/stacks/erc20/README.md b/app/data/stacks/erc20/README.md index cab41c79..a6568094 100644 --- a/app/data/stacks/erc20/README.md +++ b/app/data/stacks/erc20/README.md @@ -7,13 +7,13 @@ Instructions to deploy a local ERC20 watcher stack (core + watcher) for demonstr Clone required repositories: ```bash -laconic-so setup-repositories --include cerc-io/go-ethereum,cerc-io/ipld-eth-db,cerc-io/ipld-eth-server,cerc-io/watcher-ts +laconic-so --stack erc20 setup-repositories ``` Build the core and watcher container images: ```bash -laconic-so build-containers --include cerc/go-ethereum,cerc/go-ethereum-foundry,cerc/ipld-eth-db,cerc/ipld-eth-server,cerc/watcher-erc20 +laconic-so --stack erc20 build-containers ``` This should create the required docker images in the local image registry. @@ -21,7 +21,7 @@ This should create the required docker images in the local image registry. Deploy the stack: ```bash -laconic-so deploy-system --include ipld-eth-db,go-ethereum-foundry,ipld-eth-server,watcher-erc20 up +laconic-so --stack erc20 deploy-system up ``` ## Demo -- 2.45.2 From 2bf50383dddcc7158fd171c460d45443a68314ac Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 11:53:58 -0700 Subject: [PATCH 2/7] Missed one update --- app/data/stacks/erc20/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/stacks/erc20/README.md b/app/data/stacks/erc20/README.md index a6568094..43687e3f 100644 --- a/app/data/stacks/erc20/README.md +++ b/app/data/stacks/erc20/README.md @@ -153,5 +153,5 @@ Transfer funds between different accounts using MetaMask and use the playground To stop all the services running in background run: ```bash -laconic-so deploy-system --include ipld-eth-db,go-ethereum-foundry,ipld-eth-server,watcher-erc20 down +laconic-so --stack erc20 deploy-system down ``` -- 2.45.2 From 60c1da725e0900045012a80d2449326d5c55fee7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 13:34:51 -0700 Subject: [PATCH 3/7] Initial commit --- app/base.py | 34 +++++++++++++++ app/build_npms.py | 10 ++++- app/data/stacks/package-registry/stack.yml | 11 +++++ app/deploy_system.py | 51 ++++++++++++++++++++-- 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 app/base.py create mode 100644 app/data/stacks/package-registry/stack.yml diff --git a/app/base.py b/app/base.py new file mode 100644 index 00000000..04cdaf5a --- /dev/null +++ b/app/base.py @@ -0,0 +1,34 @@ +# Copyright © 2022, 2023 Cerc + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +def get_stack(config, stack): + return base_stack(config, stack) + + +class base_stack(): + + def __init__(self, config, stack): + self.config = config + self.stack = stack + + def ensure_available(self): + if self.config.verbose: + print(f"Checking that base stack {self.stack} is available") + return 1 + + def get_url(self): + return "http://gitea.local:3000/api/packages/cerc-io/npm/" + +# need code to \ No newline at end of file diff --git a/app/build_npms.py b/app/build_npms.py index 3cf7d404..d794d027 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -24,6 +24,7 @@ from decouple import config import click import importlib.resources from python_on_whales import docker, DockerException +from .base import get_stack from .util import include_exclude_check, get_parsed_stack_config @click.command() @@ -41,6 +42,12 @@ def command(ctx, include, exclude): stack = ctx.obj.stack continue_on_error = ctx.obj.continue_on_error + # build-npms depends on having access to a writable package registry + # so we check here that it is available + package_registry_stack = get_stack(ctx.obj, 'package-registry') + package_registry_stack.ensure_available() + npm_registry_url = package_registry_stack.get_url('package-registry') + if local_stack: dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') @@ -75,7 +82,7 @@ def command(ctx, include, exclude): repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) # TODO: make the npm registry url configurable. - build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/"] + build_command = ["sh", "-c", f"cd /workspace && build-npm-package-local-dependencies.sh {npm_registry_url}"] if not dry_run: if verbose: print(f"Executing: {build_command}") @@ -87,6 +94,7 @@ def command(ctx, include, exclude): tty=True, user=f"{os.getuid()}:{os.getgid()}", envs=envs, + # TODO: detect this host name in npm_registry_url rather than hard-wiring it add_hosts=[("gitea.local", "host-gateway")], volumes=[(repo_full_path, "/workspace")], command=build_command diff --git a/app/data/stacks/package-registry/stack.yml b/app/data/stacks/package-registry/stack.yml new file mode 100644 index 00000000..771149ed --- /dev/null +++ b/app/data/stacks/package-registry/stack.yml @@ -0,0 +1,11 @@ +version: "1.1" +name: package-registry +decription: "Local Package Registry" +repos: + - cerc-io/hosting +pods: + - name: gitea + repository: cerc-io/hosting + path: gitea + pre_start_command: "run-this-first.sh" + post_start_command: "initialize-gitea.sh" diff --git a/app/deploy_system.py b/app/deploy_system.py index b7fc69a6..f7bf71b1 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -18,6 +18,7 @@ import hashlib import os import sys +from decouple import config from python_on_whales import DockerClient import click import importlib.resources @@ -40,9 +41,16 @@ def command(ctx, include, exclude, cluster, command, extra_args): debug = ctx.obj.debug quiet = ctx.obj.quiet verbose = ctx.obj.verbose + local_stack = ctx.obj.local_stack dry_run = ctx.obj.dry_run stack = ctx.obj.stack + if local_stack: + dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose") @@ -68,19 +76,37 @@ def command(ctx, include, exclude, cluster, command, extra_args): else: pods_in_scope = all_pods + # Convert all pod definitions to v1.1 format + pods_in_scope = _convert_to_new_format(pods_in_scope) + if verbose: print(f"Pods: {pods_in_scope}") # Construct a docker compose command suitable for our purpose compose_files = [] + pre_start_commands = [] + post_start_commands = [] for pod in pods_in_scope: - if include_exclude_check(pod, include, exclude): - compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod}.yml") + pod_name = pod["name"] + pod_repository = pod["repository"] + pod_path = pod["path"] + if include_exclude_check(pod_name, include, exclude): + if pod_repository is None or pod_repository == "internal": + compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml") + else: + pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"]) + compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml") + pod_pre_start_command = pod["pre_start_command"] + pod_post_start_command = pod["post_start_command"] + if pod_pre_start_command is not None: + pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command)) + if pod_post_start_command is not None: + post_start_commands.append(os.path.join(pod_root_dir, pod_post_start_command)) compose_files.append(compose_file_name) else: if verbose: - print(f"Excluding: {pod}") + print(f"Excluding: {pod_name}") if verbose: print(f"files: {compose_files}") @@ -96,7 +122,11 @@ def command(ctx, include, exclude, cluster, command, extra_args): os.environ["CERC_SCRIPT_DEBUG"] = "true" if verbose: print(f"Running compose up for extra_args: {extra_args_list}") + for pre_start_command in pre_start_commands: + 1 docker.compose.up(detach=True, services=extra_args_list) + for post_start_command in post_start_commands: + 1 elif command == "down": if verbose: print("Running compose down") @@ -148,3 +178,18 @@ def command(ctx, include, exclude, cluster, command, extra_args): if verbose: print("Running compose logs") docker.compose.logs() + + +def _convert_to_new_format(old_pod_array): + new_pod_array = [] + for old_pod in old_pod_array: + if isinstance(old_pod, dict): + new_pod_array.append(old_pod) + else: + new_pod = { + "name": old_pod, + "repository": "internal", + "path": old_pod + } + new_pod_array.append(new_pod) + return new_pod_array -- 2.45.2 From a1893aa153bec36426d3d1f294040f31cadbe8cf Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 14:15:35 -0700 Subject: [PATCH 4/7] Basic functionality --- app/deploy_system.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/app/deploy_system.py b/app/deploy_system.py index f7bf71b1..91ffa28c 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -19,6 +19,7 @@ import hashlib import os import sys from decouple import config +import subprocess from python_on_whales import DockerClient import click import importlib.resources @@ -123,10 +124,10 @@ def command(ctx, include, exclude, cluster, command, extra_args): if verbose: print(f"Running compose up for extra_args: {extra_args_list}") for pre_start_command in pre_start_commands: - 1 + _run_command(ctx.obj, pre_start_command) docker.compose.up(detach=True, services=extra_args_list) for post_start_command in post_start_commands: - 1 + _run_command(ctx.obj, post_start_command) elif command == "down": if verbose: print("Running compose down") @@ -193,3 +194,16 @@ def _convert_to_new_format(old_pod_array): } new_pod_array.append(new_pod) return new_pod_array + + +def _run_command(ctx, command): + if ctx.verbose: + print(f"Running command: {command}") + command_dir = os.path.dirname(command) + print(f"command_dir: {command_dir}") + command_file = os.path.join(".", os.path.basename(command)) + my_env = os.environ.copy() + command_result = subprocess.run(f"bash {command_file}", shell=True, env=my_env, cwd=command_dir) + if command_result.returncode != 0: + print(f"FATAL Error running command: {command}") + sys.exit(1) -- 2.45.2 From bb39d90522721a7cdd3bea9b39a8737c736133f9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 15:31:43 -0700 Subject: [PATCH 5/7] Add pre/post script support --- app/deploy_system.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/app/deploy_system.py b/app/deploy_system.py index 91ffa28c..29028a53 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -124,10 +124,10 @@ def command(ctx, include, exclude, cluster, command, extra_args): if verbose: print(f"Running compose up for extra_args: {extra_args_list}") for pre_start_command in pre_start_commands: - _run_command(ctx.obj, pre_start_command) + _run_command(ctx.obj, cluster, pre_start_command) docker.compose.up(detach=True, services=extra_args_list) for post_start_command in post_start_commands: - _run_command(ctx.obj, post_start_command) + _run_command(ctx.obj, cluster, post_start_command) elif command == "down": if verbose: print("Running compose down") @@ -196,14 +196,17 @@ def _convert_to_new_format(old_pod_array): return new_pod_array -def _run_command(ctx, command): +def _run_command(ctx, cluster_name, command): if ctx.verbose: print(f"Running command: {command}") command_dir = os.path.dirname(command) print(f"command_dir: {command_dir}") command_file = os.path.join(".", os.path.basename(command)) - my_env = os.environ.copy() - command_result = subprocess.run(f"bash {command_file}", shell=True, env=my_env, cwd=command_dir) + command_env = os.environ.copy() + command_env["CERC_SO_COMPOSE_PROJECT"] = cluster_name + if ctx.debug: + command_env["CERC_SCRIPT_DEBUG"] = "true" + command_result = subprocess.run(command_file, shell=True, env=command_env, cwd=command_dir) if command_result.returncode != 0: print(f"FATAL Error running command: {command}") sys.exit(1) -- 2.45.2 From 88d81f7df62e0341f4b0ac3b09b05d51a81d24ad Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 15:35:31 -0700 Subject: [PATCH 6/7] Fix comment --- app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/base.py b/app/base.py index 04cdaf5a..e4e44791 100644 --- a/app/base.py +++ b/app/base.py @@ -31,4 +31,4 @@ class base_stack(): def get_url(self): return "http://gitea.local:3000/api/packages/cerc-io/npm/" -# need code to \ No newline at end of file +# TODO: finish this implementation for the npm package registry \ No newline at end of file -- 2.45.2 From 63c93acb83fc02ffafaaebee04353c45b8f858b6 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 17 Feb 2023 15:36:09 -0700 Subject: [PATCH 7/7] Add newline --- app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/base.py b/app/base.py index e4e44791..e70f7289 100644 --- a/app/base.py +++ b/app/base.py @@ -31,4 +31,4 @@ class base_stack(): def get_url(self): return "http://gitea.local:3000/api/packages/cerc-io/npm/" -# TODO: finish this implementation for the npm package registry \ No newline at end of file +# TODO: finish this implementation for the npm package registry -- 2.45.2