Fix stack path check (#877)

Reviewed-on: #877
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

setup.py

@@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 with open("requirements.txt", "r", encoding="utf-8") as fh:
     requirements = fh.read()
+with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
+    version = fh.readlines()[-1].strip(" \n")
 setup(
     name='laconic-stack-orchestrator',
-    version='1.0.12',
+    version=version,
     author='Cerc',
     author_email='info@cerc.io',
     license='GNU Affero General Public License',

stack_orchestrator/build/build_util.py

@@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
 
 def get_containers_in_scope(stack: str):
 
-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
-
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
@@ -33,7 +28,10 @@ def get_containers_in_scope(stack: str):
             warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
-        containers_in_scope = all_containers
+        # See: https://stackoverflow.com/a/20885799/1701505
+        from stack_orchestrator import data
+        with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+            containers_in_scope = container_list_file.read().splitlines()
 
     if opts.o.verbose:
         print(f'Containers: {containers_in_scope}')

stack_orchestrator/data/compose/docker-compose-laconic-explorer.yml

@@ -0,0 +1,10 @@
+services:
+  laconic-explorer:
+    restart: unless-stopped
+    image: cerc/ping-pub:local
+    environment:
+      - LACONIC_LACONICD_API_URL=${LACONIC_LACONICD_API_URL:-http://localhost:1317}
+      - LACONIC_LACONICD_RPC_URL=${LACONIC_LACONICD_RPC_URL:-http://localhost:26657}
+      - LACONIC_LACONICD_CHAIN_ID=${LACONIC_LACONICD_CHAIN_ID:-chain-id-not-set}
+    ports:
+      - "5173"

stack_orchestrator/data/container-build/cerc-laconic-console-host/build.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Build cerc/laconic-registry-cli
+# Build cerc/laconic-console-host
 
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -28,6 +28,8 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     # Install pnpm
     && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -14,6 +14,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -13,6 +13,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -5,7 +5,7 @@ fi
 
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-120}
 tpid=""
 
 ctrl_c() {
@@ -20,6 +20,8 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
     CERC_BUILD_TOOL=pnpm
   elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  elif [ -f "bun.lockb" ]; then
+    CERC_BUILD_TOOL=bun
   else
     CERC_BUILD_TOOL=npm
   fi

stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile

@@ -0,0 +1,10 @@
+FROM cerc/ping-pub-base:local
+
+COPY ./scripts/update-explorer-config.sh /scripts
+COPY ./scripts/start-serving-explorer.sh /scripts
+COPY ./config/laconic-chaindata-template.json /config/chains/laconic-chaindata-template.json
+
+EXPOSE 5173
+
+WORKDIR /app
+CMD ["/scripts/start-serving-explorer.sh"]

stack_orchestrator/data/container-build/cerc-ping-pub/Dockerfile.base

@@ -0,0 +1,8 @@
+FROM cerc/webapp-base:local
+
+WORKDIR /app
+
+COPY . .
+
+RUN yarn
+

stack_orchestrator/data/container-build/cerc-ping-pub/build.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 # Build the ping pub image
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer
+# Two-stage build is to allow us to pick up both the upstream repo's files, and local files here for config
+docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/explorer
+docker build -t cerc/ping-pub:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile $SCRIPT_DIR

stack_orchestrator/data/container-build/cerc-ping-pub/config/laconic-chaindata-template.json

@@ -0,0 +1,22 @@
+{
+    "chain_name": "LACONIC_LACONICD_CHAIN_ID",
+    "registry_name": "LACONIC_LACONICD_CHAIN_ID",
+    "api": [
+        {"provider": "LX-one-tree-one-seven", "address": "LACONIC_LACONICD_API_URL"}
+    ],
+    "rpc": [
+        {"provider": "LX-tendermint-rpc", "address": "LACONIC_LACONICD_RPC_URL"}
+    ],
+    "sdk_version": "0.45.1",
+    "coin_type": "118",
+    "min_tx_fee": "800",
+    "addr_prefix": "ethm",
+    "logo": "/logos/cosmos.svg",
+    "assets": [{
+        "base": "aphoton",
+        "symbol": "LNT",
+        "exponent": "6",
+        "coingecko_id": "cosmos", 
+        "logo": "/logos/cosmos.svg"
+    }]
+}

stack_orchestrator/data/container-build/cerc-ping-pub/scripts/start-serving-explorer.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+${SCRIPT_DIR}/update-explorer-config.sh
+
+echo "Starting serving explorer"
+# Force cache re-build because vite is dumb and can't be restarted otherwise
+yarn serve --host --force

stack_orchestrator/data/container-build/cerc-ping-pub/scripts/update-explorer-config.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Verify that we have the config variables we need
+if [[ -z ${LACONIC_LACONICD_API_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_API_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_RPC_URL} ]]; then
+    echo "Error: LACONIC_LACONICD_RPC_URL not defined"
+    exit 1
+fi
+if [[ -z ${LACONIC_LACONICD_CHAIN_ID} ]]; then
+    echo "Error: LACONIC_LACONICD_CHAIN_ID not defined"
+    exit 1
+fi
+
+# Ping-pub explorer has endlessly confusing behavior where it
+# infers the directory from which to load chain configuration files
+# by the presence or absense of the substring "testnet" in the host name
+# (browser side -- the host name of the host in the address bar of the browser)
+# Accordingly we configure our network in both directories in order to 
+# subvert this lunacy.
+explorer_mainnet_config_dir=/app/chains/mainnet
+explorer_testnet_config_dir=/app/chains/testnet
+config_template_file=/config/chains/laconic-chaindata-template.json
+chain_config_name=laconic.json
+mainnet_config_file=${explorer_mainnet_config_dir}/${chain_config_name}
+testnet_config_file=${explorer_testnet_config_dir}/${chain_config_name}
+
+# Delete the stock config files
+rm -f ${explorer_testnet_config_dir}/*
+rm -f ${explorer_mainnet_config_dir}/*
+
+# Copy in our template file
+cp ${config_template_file} ${mainnet_config_file}
+
+# Update the file with the config variables
+sed -i "s#LACONIC_LACONICD_API_URL#${LACONIC_LACONICD_API_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_RPC_URL#${LACONIC_LACONICD_RPC_URL}#g" ${mainnet_config_file}
+sed -i "s#LACONIC_LACONICD_CHAIN_ID#${LACONIC_LACONICD_CHAIN_ID}#g" ${mainnet_config_file}
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  echo "Updated chaindata file:"
+  cat ${mainnet_config_file}
+fi
+
+# Copy over to the testnet directory
+cp ${mainnet_config_file} ${testnet_config_file}

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -28,11 +28,13 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     # Install pnpm
     && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
+    # Install bun
+    && su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh

@@ -4,6 +4,8 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+# TODO: document what this script does
+
 WORK_DIR="${1:-./}"
 
 cd "${WORK_DIR}" || exit 1

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -27,6 +27,8 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
       CERC_BUILD_TOOL=pnpm
     elif [ -f "yarn.lock" ]; then
       CERC_BUILD_TOOL=yarn
+    elif [ -f "bun.lockb" ]; then
+      CERC_BUILD_TOOL=bun
     else
       CERC_BUILD_TOOL=npm
     fi

stack_orchestrator/data/stacks/mainnet-laconic/deploy/commands.py

@@ -18,7 +18,7 @@ from stack_orchestrator.deploy.deploy_types import DeployCommandContext, Laconic
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
-from stack_orchestrator.command_types import CommandOptions
+from stack_orchestrator.opts import opts
 from enum import Enum
 from pathlib import Path
 from shutil import copyfile, copytree
@@ -62,7 +62,7 @@ def _get_node_moniker_from_config(network_dir: Path):
     return moniker
 
 
-def _get_node_key_from_gentx(options: CommandOptions, gentx_file_name: str):
+def _get_node_key_from_gentx(gentx_file_name: str):
     gentx_file_path = Path(gentx_file_name)
     if gentx_file_path.exists():
         with open(Path(gentx_file_name), "rb") as f:
@@ -77,24 +77,24 @@ def _comma_delimited_to_list(list_str: str):
     return list_str.split(",") if list_str else []
 
 
-def _get_node_keys_from_gentx_files(options: CommandOptions, gentx_file_list: str):
+def _get_node_keys_from_gentx_files(gentx_file_list: str):
     node_keys = []
     gentx_files = _comma_delimited_to_list(gentx_file_list)
     for gentx_file in gentx_files:
-        node_key = _get_node_key_from_gentx(options, gentx_file)
+        node_key = _get_node_key_from_gentx(gentx_file)
         if node_key:
             node_keys.append(node_key)
     return node_keys
 
 
-def _copy_gentx_files(options: CommandOptions, network_dir: Path, gentx_file_list: str):
+def _copy_gentx_files(network_dir: Path, gentx_file_list: str):
     gentx_files = _comma_delimited_to_list(gentx_file_list)
     for gentx_file in gentx_files:
         gentx_file_path = Path(gentx_file)
         copyfile(gentx_file_path, os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path)))
 
 
-def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
+def _remove_persistent_peers(network_dir: Path):
     config_file_path = _config_toml_path(network_dir)
     if not config_file_path.exists():
         print("Error: config.toml not found")
@@ -108,20 +108,45 @@ def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
         output_file.write(config_file_content)
 
 
-def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
+def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str):
     config_file_path = config_dir.joinpath("config.toml")
     if not config_file_path.exists():
         print("Error: config.toml not found")
         sys.exit(1)
     with open(config_file_path, "r") as input_file:
         config_file_content = input_file.read()
-        persistent_peers_pattern = '^persistent_peers = ""'
+        persistent_peers_pattern = r'^persistent_peers = ""'
         replace_with = f"persistent_peers = \"{new_persistent_peers}\""
         config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
     with open(config_file_path, "w") as output_file:
         output_file.write(config_file_content)
 
 
+def _enable_cors(config_dir: Path):
+    config_file_path = config_dir.joinpath("config.toml")
+    if not config_file_path.exists():
+        print("Error: config.toml not found")
+        sys.exit(1)
+    with open(config_file_path, "r") as input_file:
+        config_file_content = input_file.read()
+        cors_pattern = r'^cors_allowed_origins = \[]'
+        replace_with = 'cors_allowed_origins = ["*"]'
+        config_file_content = re.sub(cors_pattern, replace_with, config_file_content, flags=re.MULTILINE)
+    with open(config_file_path, "w") as output_file:
+        output_file.write(config_file_content)
+    app_file_path = config_dir.joinpath("app.toml")
+    if not app_file_path.exists():
+        print("Error: app.toml not found")
+        sys.exit(1)
+    with open(app_file_path, "r") as input_file:
+        app_file_content = input_file.read()
+        cors_pattern = r'^enabled-unsafe-cors = false'
+        replace_with = "enabled-unsafe-cors = true"
+        app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE)
+    with open(app_file_path, "w") as output_file:
+        output_file.write(app_file_content)
+
+
 def _phase_from_params(parameters):
     phase = SetupPhase.ILLEGAL
     if parameters.initialize_network:
@@ -151,7 +176,7 @@ def _phase_from_params(parameters):
 
 def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
 
-    options = command_context.cluster_context.options
+    options = opts.o
 
     currency = "stake"  # Does this need to be a parameter?
 
@@ -238,7 +263,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
                 print("Error: --gentx-files must be supplied")
                 sys.exit(1)
             # First look in the supplied gentx files for the other nodes' keys
-            other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
+            other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_file_list)
             # Add those keys to our genesis, with balances we determine here (why?)
             for other_node_key in other_node_keys:
                 outputk, statusk = run_container_command(
@@ -247,7 +272,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             if options.debug:
                 print(f"Command output: {outputk}")
             # Copy the gentx json files into our network dir
-            _copy_gentx_files(options, network_dir, parameters.gentx_file_list)
+            _copy_gentx_files(network_dir, parameters.gentx_file_list)
             # Now we can run collect-gentxs
             output1, status1 = run_container_command(
                 command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
@@ -256,7 +281,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
             print(f"Generated genesis file, please copy to other nodes as required: \
                 {os.path.join(network_dir, 'config', 'genesis.json')}")
             # Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
-            _remove_persistent_peers(options, network_dir)
+            _remove_persistent_peers(network_dir)
         # In both cases we validate the genesis file now
         output2, status1 = run_container_command(
             command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
@@ -267,7 +292,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
         sys.exit(1)
 
 
-def create(context: DeploymentContext, extra_args):
+def create(deployment_context: DeploymentContext, extra_args):
     network_dir = extra_args[0]
     if network_dir is None:
         print("Error: --network-dir must be supplied")
@@ -286,15 +311,17 @@ def create(context: DeploymentContext, extra_args):
         sys.exit(1)
     # Copy the network directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
+    deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config")
     copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
     # If supplied, add the initial persistent peers to the config file
     if extra_args[1]:
         initial_persistent_peers = extra_args[1]
-        _insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
+        _insert_persistent_peers(deployment_config_dir, initial_persistent_peers)
+    # Enable CORS headers so explorers and so on can talk to the node
+    _enable_cors(deployment_config_dir)
     # Copy the data directory contents into our deployment
     # TODO: change this to work with non local paths
-    deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
+    deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data")
     copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
 
 

stack_orchestrator/data/stacks/mainnet-laconic/stack.yml

@@ -2,14 +2,15 @@ version: "1.0"
 name: mainnet-laconic
 description: "Mainnet laconic node"
 repos:
-  - cerc-io/laconicd
-  - lirewine/debug
-  - lirewine/crypto
-  - lirewine/gem
-  - lirewine/sdk
-  - cerc-io/laconic-sdk
-  - cerc-io/laconic-registry-cli
-  - cerc-io/laconic-console
+  - git.vdb.to/cerc-io/laconicd
+  - github.com/lirewine/debug
+  - github.com/lirewine/crypto
+  - github.com/lirewine/gem
+  - github.com/lirewine/sdk
+  - git.vdb.to/cerc-io/laconic-sdk
+  - git.vdb.to/cerc-io/laconic-registry-cli
+  - git.vdb.to/cerc-io/laconic-console
+  - github.com/ping-pub/explorer
 npms:
   - laconic-sdk
   - laconic-registry-cli
@@ -23,7 +24,8 @@ containers:
   - cerc/laconic-registry-cli
   - cerc/webapp-base
   - cerc/laconic-console-host
+  - cerc/ping-pub
 pods:
   - mainnet-laconicd
   - fixturenet-laconic-console
-
+  - laconic-explorer

stack_orchestrator/deploy/deploy.py

@@ -26,8 +26,15 @@ import click
 from pathlib import Path
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
-from stack_orchestrator.util import resolve_compose_file
+from stack_orchestrator.util import (
+    get_stack_path,
+    include_exclude_check,
+    get_parsed_stack_config,
+    global_options2,
+    get_dev_root_path,
+    stack_is_in_deployment,
+    resolve_compose_file,
+)
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -60,6 +67,7 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
     if deploy_to is None:
         deploy_to = "compose"
 
+    stack = get_stack_path(stack)
     ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
     # Subcommand is executed now, by the magic of click
 
@@ -274,16 +282,12 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)
 
 # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
 def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
-
     dev_root_path = get_dev_root_path(ctx)
 
-    # TODO: huge hack, fix this
-    # If the caller passed a path for the stack file, then we know that we can get the compose files
-    # from the same directory
-    deployment = False
-    if isinstance(stack, os.PathLike):
-        compose_dir = stack.parent.joinpath("compose")
-        deployment = True
+    # TODO: hack, this should be encapsulated by the deployment context.
+    deployment = stack_is_in_deployment(stack)
+    if deployment:
+        compose_dir = stack.joinpath("compose")
     else:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
         compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")

stack_orchestrator/deploy/deployment.py

@@ -50,15 +50,15 @@ def command(ctx, dir):
 
 def make_deploy_context(ctx) -> DeployCommandContext:
     context: DeploymentContext = ctx.obj
-    stack_file_path = context.get_stack_file()
     env_file = context.get_env_file()
     cluster_name = context.get_cluster_id()
     if constants.deploy_to_key in context.spec.obj:
         deployment_type = context.spec.obj[constants.deploy_to_key]
     else:
         deployment_type = constants.compose_deploy_type
-    return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
-                                 deployment_type)
+    stack = context.deployment_dir
+    return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None,
+                                 cluster_name, env_file, deployment_type)
 
 
 @command.command()
@@ -123,6 +123,7 @@ def push_images(ctx):
 @click.argument('extra_args', nargs=-1)  # help: command: port <service1> <service2>
 @click.pass_context
 def port(ctx, extra_args):
+    ctx.obj = make_deploy_context(ctx)
     port_operation(ctx, extra_args)
 
 

stack_orchestrator/deploy/deployment_create.py

@@ -24,7 +24,7 @@ from secrets import token_hex
 import sys
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
+from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                      get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
                                      resolve_config_dir)
@@ -238,6 +238,11 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
                         config_dir = host_path.split("/")[2]
                         if config_dir != pod:
                             config_dirs.add(config_dir)
+        for env_file in service_info.get("env_file", []):
+            if env_file.startswith("../config"):
+                config_dir = env_file.split("/")[2]
+                if config_dir != pod:
+                    config_dirs.add(config_dir)
     return config_dirs
 
 
@@ -454,7 +459,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
     deployment_type = parsed_spec[constants.deploy_to_key]
-    stack_file = get_stack_file_path(stack_name)
+    stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name)
     parsed_stack = get_parsed_stack_config(stack_name)
     if opts.o.debug:
         print(f"parsed spec: {parsed_spec}")
@@ -467,7 +472,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     os.mkdir(deployment_dir_path)
     # Copy spec file and the stack file into the deployment dir
     copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name))
-    copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file)))
+    copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name))
     _create_deployment_file(deployment_dir_path)
     # Copy any config varibles from the spec file into an env file suitable for compose
     _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name))

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -24,7 +24,7 @@ import uuid
 
 import click
 
-from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
+from stack_orchestrator.deploy.images import remote_image_exists
 from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
                                                    build_container_image, push_container_image,
@@ -99,44 +99,62 @@ def process_app_deployment_request(
 
     deployment_record = laconic.get_record(app_deployment_crn)
     deployment_dir = os.path.join(deployment_parent_dir, fqdn)
+    # At present we use this to generate a unique but stable ID for the app's host container
+    # TODO: implement support to derive this transparently from the already-unique deployment id
+    unique_deployment_id = hashlib.md5(fqdn.encode()).hexdigest()[:16]
     deployment_config_file = os.path.join(deployment_dir, "config.env")
-    # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
-    deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
+    deployment_container_tag = "laconic-webapp/%s:local" % unique_deployment_id
     app_image_shared_tag = f"laconic-webapp/{app.id}:local"
     #   b. check for deployment directory (create if necessary)
     if not os.path.exists(deployment_dir):
         if deployment_record:
             raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." %
                             (app_deployment_crn, deployment_dir))
-        print("deploy_webapp", deployment_dir)
+        logger.log(f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}")
         deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag,
                                         f"https://{fqdn}", kube_config, image_registry, env_filename)
     elif env_filename:
         shutil.copyfile(env_filename, deployment_config_file)
 
     needs_k8s_deploy = False
+    if force_rebuild:
+        logger.log("--force-rebuild is enabled so the container will always be built now, even if nothing has changed in the app")
     # 6. build container (if needed)
-    if not deployment_record or deployment_record.attributes.application != app.id:
+    # TODO: add a comment that explains what this code is doing (not clear to me)
+    if not deployment_record or deployment_record.attributes.application != app.id or force_rebuild:
         needs_k8s_deploy = True
         # check if the image already exists
         shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
+        # Note: in the code below, calls to add_tags_to_image() won't work at present.
+        # This is because SO deployment code in general re-names the container image
+        # to be unique to the deployment. This is done transparently
+        # and so when we call add_tags_to_image() here and try to add tags to the remote image,
+        # we get the image name wrong. Accordingly I've disabled the relevant code for now.
+        # This is safe because we are running with --force-rebuild at present
         if shared_tag_exists and not force_rebuild:
             # simply add our unique tag to the existing image and we are done
-            logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
-            add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
+            logger.log(
+                f"(SKIPPED) Existing image found for this app: {app_image_shared_tag} "
+                "tagging it with: {deployment_container_tag} to use in this deployment"
+                )
+            # add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
             logger.log("Tag complete")
         else:
             extra_build_args = []  # TODO: pull from request
-            logger.log(f"Building container image {deployment_container_tag}")
+            logger.log(f"Building container image: {deployment_container_tag}")
             build_container_image(app, deployment_container_tag, extra_build_args, logger)
             logger.log("Build complete")
-            logger.log(f"Pushing container image {deployment_container_tag}")
+            logger.log(f"Pushing container image: {deployment_container_tag}")
             push_container_image(deployment_dir, logger)
             logger.log("Push complete")
             # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
-            logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
-            add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            logger.log(
+                f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
+                )
+            # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
             logger.log("Tag complete")
+    else:
+        logger.log("Requested app is already deployed, skipping build and image push")
 
     # 7. update config (if needed)
     if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:

stack_orchestrator/deploy/webapp/util.py

@@ -242,6 +242,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
 def build_container_image(app_record, tag, extra_build_args=[], logger=None):
     tmpdir = tempfile.mkdtemp()
 
+    # TODO: determine if this code could be calling into the Python git library like setup-repositories
     try:
         record_id = app_record["id"]
         ref = app_record.attributes.repository_ref
@@ -249,6 +250,16 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
         clone_dir = os.path.join(tmpdir, record_id)
 
         logger.log(f"Cloning repository {repo} to {clone_dir} ...")
+        # Set github credentials if present running a command like:
+        # git config --global url."https://${TOKEN}:@github.com/".insteadOf "https://github.com/"
+        github_token = os.environ.get("DEPLOYER_GITHUB_TOKEN")
+        if github_token:
+            logger.log("Github token detected, setting it in the git environment")
+            git_config_args = [
+                "git", "config", "--global", f"url.https://{github_token}:@github.com/.insteadOf", "https://github.com/"
+                ]
+            result = subprocess.run(git_config_args, stdout=logger.file, stderr=logger.file)
+            result.check_returncode()
         if ref:
             # TODO: Determing branch or hash, and use depth 1 if we can.
             git_env = dict(os.environ.copy())
@@ -265,6 +276,7 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
                 logger.log(f"git checkout failed.  Does ref {ref} exist?")
                 raise e
         else:
+            # TODO: why is this code different vs the branch above (run vs check_call, and no prompt disable)?
             result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
             result.check_returncode()
 
@@ -299,11 +311,12 @@ def push_container_image(deployment_dir, logger):
 
 def deploy_to_k8s(deploy_record, deployment_dir, logger):
     if not deploy_record:
-        command = "up"
+        command = "start"
     else:
         command = "update"
 
     logger.log("Deploying to k8s ...")
+    logger.log(f"Running {command} command on deployment dir: {deployment_dir}")
     result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
                             stdout=logger.file, stderr=logger.file)
     result.check_returncode()

stack_orchestrator/repos/setup_repositories.py

@@ -20,14 +20,12 @@ import os
 import sys
 from decouple import config
 import git
+from git.exc import GitCommandError
 from tqdm import tqdm
 import click
 import importlib.resources
-from pathlib import Path
-import yaml
-from stack_orchestrator.constants import stack_file_name
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit
 
 
 class GitProgress(git.RemoteProgress):
@@ -81,9 +79,13 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
     except TypeError:
         # This means that the current ref is not a branch, so possibly a tag
         # Let's try to get the tag
+        try:
             current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
-        # Note that git is assymetric -- the tag you told it to check out may not be the one
+            # Note that git is asymmetric -- the tag you told it to check out may not be the one
             # you get back here (if there are multiple tags associated with the same commit)
+        except GitCommandError:
+            # If there is no matching branch or tag checked out, just use the current SHA
+            current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha
     return current_repo_branch_or_tag, is_branch
 
 
@@ -102,7 +104,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
         full_filesystem_repo_path
         ) if is_present else (None, None)
     if not opts.o.quiet:
-        present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
+        present_text = f"already exists active {'branch' if is_branch else 'ref'}: {current_repo_branch_or_tag}" if is_present \
             else 'Needs to be fetched'
         print(f"Checking: {full_filesystem_repo_path}: {present_text}")
     # Quick check that it's actually a repo
@@ -120,7 +122,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
                         origin = git_repo.remotes.origin
                         origin.pull(progress=None if opts.o.quiet else GitProgress())
                     else:
-                        print("skipping pull because this repo checked out a tag")
+                        print("skipping pull because this repo is not on a branch")
                 else:
                     print("(git pull skipped)")
     if not is_present:
@@ -222,19 +224,9 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
 
     repos_in_scope = []
     if stack:
-        if stack_is_external(stack):
-            stack_file_path = Path(stack).joinpath(stack_file_name)
-        else:
-            # In order to be compatible with Python 3.8 we need to use this hack to get the path:
-            # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-            stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
-        if not stack_file_path.exists():
-            error_exit(f"stack {stack} does not exist")
-        with stack_file_path:
-            stack_config = yaml.safe_load(open(stack_file_path, "r"))
+        stack_config = get_parsed_stack_config(stack)
         if "repos" not in stack_config or stack_config["repos"] is None:
             warn_exit(f"stack {stack} does not define any repositories")
-            else:
         repos_in_scope = stack_config["repos"]
     else:
         repos_in_scope = all_repos

stack_orchestrator/util.py

@@ -20,6 +20,7 @@ import ruamel.yaml
 from pathlib import Path
 from dotenv import dotenv_values
 from typing import Mapping, Set, List
+from stack_orchestrator.constants import stack_file_name, deployment_file_name
 
 
 def include_exclude_check(s, include, exclude):
@@ -33,11 +34,14 @@ def include_exclude_check(s, include, exclude):
         return s not in exclude_list
 
 
-def get_stack_file_path(stack):
+def get_stack_path(stack):
+    if stack_is_external(stack):
+        stack_path = Path(stack)
+    else:
         # In order to be compatible with Python 3.8 we need to use this hack to get the path:
         # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-    stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
-    return stack_file_path
+        stack_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack)
+    return stack_path
 
 
 def get_dev_root_path(ctx):
@@ -52,21 +56,14 @@ def get_dev_root_path(ctx):
 
 # Caller can pass either the name of a stack, or a path to a stack file
 def get_parsed_stack_config(stack):
-    stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack)
-    try:
-        with stack_file_path:
-            stack_config = get_yaml().load(open(stack_file_path, "r"))
-            return stack_config
-    except FileNotFoundError as error:
+    stack_file_path = get_stack_path(stack).joinpath(stack_file_name)
+    if stack_file_path.exists():
+        return get_yaml().load(open(stack_file_path, "r"))
     # We try here to generate a useful diagnostic error
     # First check if the stack directory is present
-        stack_directory = stack_file_path.parent
-        if os.path.exists(stack_directory):
-            print(f"Error: stack.yml file is missing from stack: {stack}")
-        else:
-            print(f"Error: stack: {stack} does not exist")
-        print(f"Exiting, error: {error}")
-        sys.exit(1)
+    if stack_file_path.parent.exists():
+        error_exit(f"stack.yml file is missing from stack: {stack}")
+    error_exit(f"stack {stack} does not exist")
 
 
 def get_pod_list(parsed_stack):
@@ -87,7 +84,7 @@ def get_plugin_code_paths(stack) -> List[Path]:
     result: Set[Path] = set()
     for pod in pods:
         if type(pod) is str:
-            result.add(get_stack_file_path(stack).parent)
+            result.add(get_stack_path(stack))
         else:
             pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
             result.add(Path(os.path.join(pod_root_dir, "stack")))
@@ -199,6 +196,13 @@ def stack_is_external(stack: str):
     return Path(stack).exists() if stack is not None else False
 
 
+def stack_is_in_deployment(stack: Path):
+    if isinstance(stack, os.PathLike):
+        return stack.joinpath(deployment_file_name).exists()
+    else:
+        return False
+
+
 def get_yaml():
     # See: https://stackoverflow.com/a/45701840/1701505
     yaml = ruamel.yaml.YAML()

stack_orchestrator/version.py

@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 import click
-import importlib.resources
+from importlib import resources, metadata
 
 
 @click.command()
@@ -24,8 +24,11 @@ def command(ctx):
 
     # See: https://stackoverflow.com/a/20885799/1701505
     from stack_orchestrator import data
-    with importlib.resources.open_text(data, "build_tag.txt") as version_file:
+    if resources.is_resource(data, "build_tag.txt"):
+        with resources.open_text(data, "build_tag.txt") as version_file:
             # TODO: code better version that skips comment lines
             version_string = version_file.read().splitlines()[1]
+    else:
+        version_string = metadata.version("laconic-stack-orchestrator") + "-unknown"
 
-    print(f"Version: {version_string}")
+    print(version_string)

tests/laconic-network/run-test.sh

@@ -18,6 +18,20 @@ do
         rm -rf ${node_network_dir}
     fi
 done
+echo "Deleting any existing deployments..."
+for (( i=1 ; i<=$node_count ; i++ ));
+do
+    node_deployment_dir=${node_dir_prefix}${i}-deployment
+    node_spec_file=${node_dir_prefix}${i}-spec.yml
+    if [[ -d $node_deployment_dir ]]; then
+        echo "Deleting ${node_deployment_dir}"
+        rm -rf ${node_deployment_dir}
+    fi
+    if [[ -f $node_spec_file ]]; then
+        echo "Deleting ${node_spec_file}"
+        rm ${node_spec_file}
+    fi
+done
 
 echo "Initalizing ${node_count} nodes networks..."
 for (( i=1 ; i<=$node_count ; i++ )); 
@@ -56,3 +70,12 @@ do
     node_network_dir=${node_dir_prefix}${i}
     laconic-so --stack mainnet-laconic deploy setup --network-dir ${node_network_dir} --create-network --genesis-file ${genesis_file}
 done
+
+# Create deployments
+echo "Creating ${node_count} deployments..."
+for (( i=1 ; i<=$node_count ; i++ ));
+do
+    node_network_dir=${node_dir_prefix}${i}
+    laconic-so --stack mainnet-laconic deploy init --output ${node_network_dir}-spec.yml
+    laconic-so --stack mainnet-laconic deploy create --deployment-dir ${node_network_dir}-deployment --spec-file ${node_dir_prefix}${i}-spec.yml --network-dir ${node_network_dir}
+done

tests/webapp-test/run-webapp-test.sh

@@ -10,8 +10,11 @@ echo "Environment variables:"
 env
 # Test basic stack-orchestrator webapp
 echo "Running stack-orchestrator webapp test"
-# Bit of a hack, test the most recent package
-TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"
@@ -30,14 +33,17 @@ CHECK="SPECIAL_01234567890_TEST_STRING"
 
 set +e
 
-CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
+app_image_name="cerc/test-progressive-web-app:local"
+
+CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG ${app_image_name})
 sleep 3
 wget --tries 20 --retry-connrefused --waitretry=3 -O test.before -m http://localhost:3000
 
 docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
-CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
+echo "Running app container test"
+CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d ${app_image_name})
 sleep 3
 wget --tries 20 --retry-connrefused --waitretry=3 -O test.after -m http://localhost:3000
 
@@ -63,4 +69,18 @@ else
   echo "AFTER: PASSED"
 fi
 
+echo "Running deployment create test"
+# Note: this is not a full test -- all we're testing here is that the deploy-webapp create command doesn't crash
+test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
+fake_k8s_config_file=$CERC_REPO_BASE_DIR/kube-config.yml
+touch ${fake_k8s_config_file}
+
+$TEST_TARGET_SO deploy-webapp create --kube-config ${fake_k8s_config_file} --deployment-dir ${test_deployment_dir} --image ${app_image_name} --url https://my-test-app.example.com
+if [ -d ${test_deployment_dir} ]; then
+  echo "PASSED"
+else
+  echo "FAILED"
+  exit 1
+fi
+
 exit 0