From 3c5489681fc6544c76d7defd67be5620b39e1e0d Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Oct 2023 12:49:15 -0600 Subject: [PATCH] Implement deployment config (#554) * Initial deployment config implementation * Complete implementation, add test * Fix funky indentation * Revert test test --- app/data/compose/docker-compose-test.yml | 1 + .../cerc-test-container/run.sh | 4 +- .../stacks/mainnet-laconic/deploy/commands.py | 5 +- app/data/stacks/test/deploy/commands.py | 2 +- app/deployment.py | 18 ++++++-- app/deployment_create.py | 46 +++++++++++++++++-- tests/deploy/run-deploy-test.sh | 13 ++++-- 7 files changed, 72 insertions(+), 17 deletions(-) diff --git a/app/data/compose/docker-compose-test.yml b/app/data/compose/docker-compose-test.yml index 19660f89..5fbf46d0 100644 --- a/app/data/compose/docker-compose-test.yml +++ b/app/data/compose/docker-compose-test.yml @@ -4,6 +4,7 @@ services: restart: always environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED} volumes: - test-data:/data ports: diff --git a/app/data/container-build/cerc-test-container/run.sh b/app/data/container-build/cerc-test-container/run.sh index 205c231a..da0af7d5 100755 --- a/app/data/container-build/cerc-test-container/run.sh +++ b/app/data/container-build/cerc-test-container/run.sh @@ -14,6 +14,8 @@ else echo "Filesystem is fresh" echo `date` > $EXISTSFILENAME fi - +if [ -n "$CERC_TEST_PARAM_1" ]; then + echo "Test-param-1: ${CERC_TEST_PARAM_1}" +fi # Run nginx which will block here forever /usr/sbin/nginx -g "daemon off;" diff --git a/app/data/stacks/mainnet-laconic/deploy/commands.py b/app/data/stacks/mainnet-laconic/deploy/commands.py index 16bf015a..3a62b42d 100644 --- a/app/data/stacks/mainnet-laconic/deploy/commands.py +++ b/app/data/stacks/mainnet-laconic/deploy/commands.py @@ -27,10 +27,7 @@ import sys import tomli import re -default_spec_file_content = """config: - node_moniker: my-node-name - chain_id: my-chain-id -""" +default_spec_file_content = "" class SetupPhase(Enum): diff --git a/app/data/stacks/test/deploy/commands.py b/app/data/stacks/test/deploy/commands.py index 0a836037..fbb9a8ea 100644 --- a/app/data/stacks/test/deploy/commands.py +++ b/app/data/stacks/test/deploy/commands.py @@ -20,7 +20,7 @@ from app.deploy_util import VolumeMapping, run_container_command from pathlib import Path default_spec_file_content = """config: - config_variable: test-value + test-variable-1: test-value-1 """ diff --git a/app/deployment.py b/app/deployment.py index 8c860e46..aeabf61c 100644 --- a/app/deployment.py +++ b/app/deployment.py @@ -25,6 +25,16 @@ from app.deploy import exec_operation, logs_operation, create_deploy_context class DeploymentContext: dir: Path + def get_stack_file(self): + return self.dir.joinpath("stack.yml") + + def get_env_file(self): + return self.dir.joinpath("config.env") + + # TODO: implement me + def get_cluster_name(self): + return None + @click.group() @click.option("--dir", required=True, help="path to deployment directory") @@ -49,10 +59,10 @@ def command(ctx, dir): def make_deploy_context(ctx): - # Get the stack config file name - stack_file_path = ctx.obj.dir.joinpath("stack.yml") - # TODO: add cluster name and env file here - return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, None, None) + stack_file_path = ctx.obj.get_stack_file() + env_file = ctx.obj.get_env_file() + cluster_name = ctx.obj.get_cluster_name() + return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file) @command.command() diff --git a/app/deployment_create.py b/app/deployment_create.py index 837aaa61..76016262 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -204,22 +204,48 @@ def _get_mapped_ports(stack: str, map_recipe: str): return ports +def _parse_config_variables(variable_values: str): + result = None + if variable_values: + value_pairs = variable_values.split(",") + if len(value_pairs): + result_values = {} + for value_pair in value_pairs: + variable_value_pair = value_pair.split("=") + if len(variable_value_pair) != 2: + print(f"ERROR: config argument is not valid: {variable_values}") + sys.exit(1) + variable_name = variable_value_pair[0] + variable_value = variable_value_pair[1] + result_values[variable_name] = variable_value + result = {"config": result_values} + return result + + @click.command() +@click.option("--config", help="Provide config variables for the deployment") @click.option("--output", required=True, help="Write yaml spec file here") @click.option("--map-ports-to-host", required=False, help="Map ports to the host as one of: any-variable-random (default), " "localhost-same, any-same, localhost-fixed-random, any-fixed-random") @click.pass_context -def init(ctx, output, map_ports_to_host): +def init(ctx, config, output, map_ports_to_host): yaml = get_yaml() stack = global_options(ctx).stack - verbose = global_options(ctx).verbose + debug = global_options(ctx).debug default_spec_file_content = call_stack_deploy_init(ctx.obj) spec_file_content = {"stack": stack} if default_spec_file_content: spec_file_content.update(default_spec_file_content) - if verbose: - print(f"Creating spec file for stack: {stack}") + config_variables = _parse_config_variables(config) + if config_variables: + # Implement merge, since update() overwrites + orig_config = spec_file_content["config"] + new_config = config_variables["config"] + merged_config = {**new_config, **orig_config} + spec_file_content.update({"config": merged_config}) + if debug: + print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") ports = _get_mapped_ports(stack, map_ports_to_host) spec_file_content["ports"] = ports @@ -235,6 +261,16 @@ def init(ctx, output, map_ports_to_host): yaml.dump(spec_file_content, output_file) +def _write_config_file(spec_file: Path, config_env_file: Path): + spec_content = get_parsed_deployment_spec(spec_file) + if spec_content["config"]: + config_vars = spec_content["config"] + if config_vars: + with open(config_env_file, "w") as output_file: + for variable_name, variable_value in config_vars.items(): + output_file.write(f"{variable_name}={variable_value}\n") + + @click.command() @click.option("--spec-file", required=True, help="Spec file to use to create this deployment") @click.option("--deployment-dir", help="Create deployment files in this directory") @@ -259,6 +295,8 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): # Copy spec file and the stack file into the deployment dir copyfile(spec_file, os.path.join(deployment_dir, os.path.basename(spec_file))) copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file))) + # Copy any config varibles from the spec file into an env file suitable for compose + _write_config_file(spec_file, os.path.join(deployment_dir, "config.env")) # Copy the pod files into the deployment dir, fixing up content pods = parsed_stack['pods'] destination_compose_dir = os.path.join(deployment_dir, "compose") diff --git a/tests/deploy/run-deploy-test.sh b/tests/deploy/run-deploy-test.sh index 0ebb215f..86f4b689 100755 --- a/tests/deploy/run-deploy-test.sh +++ b/tests/deploy/run-deploy-test.sh @@ -77,7 +77,7 @@ $TEST_TARGET_SO --stack test deploy down --delete-volumes # Basic test of creating a deployment test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml -$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec +$TEST_TARGET_SO --stack test deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED # Check the file now exists if [ ! -f "$test_deployment_spec" ]; then echo "deploy init test: spec file not present" @@ -110,13 +110,20 @@ echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start # Check logs command works -log_output_2=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) -if [[ "$log_output_2" == *"Filesystem is fresh"* ]]; then +log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then echo "deployment logs test: passed" else echo "deployment logs test: FAILED" exit 1 fi +# Check the config variable CERC_TEST_PARAM_1 was passed correctly +if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then + echo "deployment config test: passed" +else + echo "deployment config test: FAILED" + exit 1 +fi # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes echo "Test passed"