Update version

Former-commit-id: 6ae86dbcec

app/build_containers.py

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cerc
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,12 +21,13 @@
 # TODO: display the available list of containers; allow re-build of either all or specific containers
 
 import os
+import sys
 from decouple import config
 import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from .util import include_exclude_check
+from .util import include_exclude_check, get_parsed_stack_config
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -43,6 +44,8 @@ def command(ctx, include, exclude):
     verbose = ctx.obj.verbose
     dry_run = ctx.obj.dry_run
     local_stack = ctx.obj.local_stack
+    stack = ctx.obj.stack
+    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
@@ -62,10 +65,19 @@ def command(ctx, include, exclude):
     # See: https://stackoverflow.com/a/20885799/1701505
     from . import data
     with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        containers = container_list_file.read().splitlines()
+        all_containers = container_list_file.read().splitlines()
+
+    containers_in_scope = []
+    if stack:
+        stack_config = get_parsed_stack_config(stack)
+        containers_in_scope = stack_config['containers']
+    else:
+        containers_in_scope = all_containers
 
     if verbose:
-        print(f'Containers: {containers}')
+        print(f'Containers: {containers_in_scope}')
+        if stack:
+            print(f"Stack: {stack}")
 
     # TODO: make this configurable
     container_build_env = {
@@ -96,12 +108,19 @@ def command(ctx, include, exclude):
             if verbose:
                 print(f"Executing: {build_command}")
             build_result = subprocess.run(build_command, shell=True, env=container_build_env)
-            # TODO: check result in build_result.returncode
+            if verbose:
-            print(f"Result is: {build_result}")
+                print(f"Return code is: {build_result.returncode}")
+            if build_result.returncode != 0:
+                print(f"Error running build for {container}")
+                if not continue_on_error:
+                    print("FATAL Error: container build failed and --continue-on-error not set, exiting")
+                    sys.exit(1)
+                else:
+                    print("****** Container Build Error, continuing because --continue-on-error is set")
         else:
             print("Skipped")
 
-    for container in containers:
+    for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
             process_container(container)
         else:

app/build_npms.py

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cerc
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,11 +19,12 @@
 # CERC_REPO_BASE_DIR defaults to ~/cerc
 
 import os
+import sys
 from decouple import config
 import click
 import importlib.resources
-from python_on_whales import docker
+from python_on_whales import docker, DockerException
-from .util import include_exclude_check
+from .util import include_exclude_check, get_parsed_stack_config
 
 @click.command()
 @click.option('--include', help="only build these packages")
@@ -37,6 +38,8 @@ def command(ctx, include, exclude):
     dry_run = ctx.obj.dry_run
     local_stack = ctx.obj.local_stack
     debug = ctx.obj.debug
+    stack = ctx.obj.stack
+    continue_on_error = ctx.obj.continue_on_error
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
@@ -53,10 +56,18 @@ def command(ctx, include, exclude):
     # See: https://stackoverflow.com/a/20885799/1701505
     from . import data
     with importlib.resources.open_text(data, "npm-package-list.txt") as package_list_file:
-        packages = package_list_file.read().splitlines()
+        all_packages = package_list_file.read().splitlines()
+
+    packages_in_scope = []
+    if stack:
+        stack_config = get_parsed_stack_config(stack)
+        # TODO: syntax check the input here
+        packages_in_scope = stack_config['npms']
+    else:
+        packages_in_scope = all_packages
 
     if verbose:
-        print(f'Packages: {packages}')
+        print(f'Packages: {packages_in_scope}')
 
     def build_package(package):
         if not quiet:
@@ -69,22 +80,32 @@ def command(ctx, include, exclude):
             if verbose:
                 print(f"Executing: {build_command}")
             envs = {"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"]} | ({"CERC_SCRIPT_DEBUG": "true"} if debug else {})
-            build_result = docker.run("cerc/builder-js",
+            try:
-                                      remove=True,
+                docker.run("cerc/builder-js",
-                                      interactive=True,
+                           remove=True,
-                                      tty=True,
+                           interactive=True,
-                                      user=f"{os.getuid()}:{os.getgid()}",
+                           tty=True,
-                                      envs=envs,
+                           user=f"{os.getuid()}:{os.getgid()}",
-                                      add_hosts=[("gitea.local", "host-gateway")],
+                           envs=envs,
-                                      volumes=[(repo_full_path, "/workspace")],
+                           add_hosts=[("gitea.local", "host-gateway")],
-                                      command=build_command
+                           volumes=[(repo_full_path, "/workspace")],
-                                      )
+                           command=build_command
-            # TODO: check result in build_result.returncode
+                           )
-            print(f"Result is: {build_result}")
+                # Note that although the docs say that build_result should contain
+                # the command output as a string, in reality it is always the empty string.
+                # Since we detect errors via catching exceptions below, we can safely ignore it here.
+            except DockerException as e:
+                print(f"Error executing build for {package} in container:\n {e}")
+                if not continue_on_error:
+                    print("FATAL Error: build failed and --continue-on-error not set, exiting")
+                    sys.exit(1)
+                else:
+                    print("****** Build Error, continuing because --continue-on-error is set")
+
         else:
             print("Skipped")
 
-    for package in packages:
+    for package in packages_in_scope:
         if include_exclude_check(package, include, exclude):
             build_package(package)
         else:

app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh

@@ -13,6 +13,8 @@ if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then
     echo "CERC_NPM_AUTH_TOKEN is not set" >&2
     exit 1
 fi
+# Exit on error
+set -e
 local_npm_registry_url=$1
 package_publish_version=$2
 # TODO: make this a paramater and allow a list of scopes

app/data/container-build/cerc-builder-js/build-npm-package.sh

@@ -17,6 +17,8 @@ if [[ $# -eq 2 ]]; then
 else
     package_publish_version=$( cat package.json | jq -r .version )
 fi
+# Exit on error
+set -e
 # Get the name of this package from package.json since we weren't passed that
 package_name=$( cat package.json | jq -r .name )
 local_npm_registry_url=$1
@@ -24,7 +26,7 @@ npm config set @lirewine:registry ${local_npm_registry_url}
 npm config set @cerc-io:registry ${local_npm_registry_url}
 npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN}
 # First check if the version of this package we're trying to build already exists in the registry
-package_exists=$( yarn info --json ${package_name}@${package_publish_version} | jq -r .data.dist.tarball )
+package_exists=$( yarn info --json ${package_name}@${package_publish_version} 2>/dev/null | jq -r .data.dist.tarball )
 if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then
     echo "${package_publish_version} of ${package_name} already exists in the registry, skipping build"
     exit 0

app/data/container-build/cerc-builder-js/yarn-local-registry-fixup.sh

@@ -14,14 +14,23 @@ if [[ $# -ne 2 ]]; then
     echo "Illegal number of parameters" >&2
     exit 1
 fi
+# Exit on error
+set -e
 target_package=$1
 local_npm_registry_url=$2
 # TODO: use jq rather than sed here:
 versioned_target_package=$(grep ${target_package} package.json | sed -e 's#[[:space:]]\{1,\}\"\('${target_package}'\)\":[[:space:]]\{1,\}\"\(.*\)\",#\1@\2#' )
 # Use yarn info to get URL checksums etc from the new registry
 yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null)
-# Code below parses out the values we need
+# First check if the target version actually exists. 
+# If it doesn't exist there will be no .data.dist.tarball element,
+# and jq will output the string "null"
 package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball)
+if [[ $package_tarball == "null" ]]; then
+    echo "FATAL: Target package version ($versioned_target_package) not found" >&2
+    exit 1
+fi
+# Code below parses out the values we need
 # When running inside a container, the registry can return a URL with the wrong host name due to proxying
 # so we need to check if that has happened and fix the URL if so.
 if ! [[ "${package_tarball}" =~ ^${local_npm_registry_url}.* ]]; then
@@ -33,6 +42,7 @@ package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity)
 package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum)
 package_resolved=${package_tarball}#${package_shasum}
 # Some strings need to be escaped so they work when passed to sed later
+escaped_package_integrity=$(printf '%s\n' "$package_integrity" | sed -e 's/[\/&]/\\&/g')
 escaped_package_resolved=$(printf '%s\n' "$package_resolved" | sed -e 's/[\/&]/\\&/g')
 escaped_target_package=$(printf '%s\n' "$target_package" | sed -e 's/[\/&]/\\&/g')
 if [ -n "$CERC_SCRIPT_VERBOSE" ]; then
@@ -44,4 +54,4 @@ fi
 # Use magic sed regex to replace the values in yarn.lock
 # Note: yarn.lock is not json so we can not use jq for this
 sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock
-sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${package_integrity}'/' yarn.lock
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${escaped_package_integrity}'/' yarn.lock

app/data/stacks/fixturenet-laconicd/stack.yml

@@ -1,9 +1,13 @@
 version: "1.0"
 name: fixturenet-laconicd
+description: "A laconicd fixturenet"
 repos:
   - cerc-io/laconicd
   - cerc-io/laconic-sdk
   - cerc-io/laconic-registry-cli
+npms:
+  - laconic-sdk
+  - laconic-registry-cli
 containers:
   - cerc/laconicd
   - cerc/laconic-registry-cli

app/data/version.txt

@@ -1,2 +1,2 @@
 # This file should be re-generated running: scripts/update-version-file.sh script
-v1.0.9-alpha-04a3049
+v1.0.11-alpha-edbf674

app/deploy_system.py

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cerc
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -22,7 +22,7 @@ from python_on_whales import DockerClient
 import click
 import importlib.resources
 from pathlib import Path
-from .util import include_exclude_check
+from .util import include_exclude_check, get_parsed_stack_config
 
 
 @click.command()
@@ -40,6 +40,7 @@ def command(ctx, include, exclude, cluster, command, services):
     quiet = ctx.obj.quiet
     verbose = ctx.obj.verbose
     dry_run = ctx.obj.dry_run
+    stack = ctx.obj.stack
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
@@ -56,15 +57,23 @@ def command(ctx, include, exclude, cluster, command, services):
     # See: https://stackoverflow.com/a/20885799/1701505
     from . import data
     with importlib.resources.open_text(data, "pod-list.txt") as pod_list_file:
-        pods = pod_list_file.read().splitlines()
+        all_pods = pod_list_file.read().splitlines()
+
+    pods_in_scope = []
+    if stack:
+        stack_config = get_parsed_stack_config(stack)
+        # TODO: syntax check the input here
+        pods_in_scope = stack_config['pods']
+    else:
+        pods_in_scope = all_pods
 
     if verbose:
-        print(f"Pods: {pods}")
+        print(f"Pods: {pods_in_scope}")
 
     # Construct a docker compose command suitable for our purpose
 
     compose_files = []
-    for pod in pods:
+    for pod in pods_in_scope:
         if include_exclude_check(pod, include, exclude):
             compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod}.yml")
             compose_files.append(compose_file_name)

app/util.py

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cerc
+# Copyright © 2022, 2023 Cerc
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,6 +13,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import os.path
+import sys
+import yaml
+from pathlib import Path
+
+
 def include_exclude_check(s, include, exclude):
     if include is None and exclude is None:
         return True
@@ -22,3 +28,23 @@ def include_exclude_check(s, include, exclude):
     if exclude is not None:
         exclude_list = exclude.split(",")
         return s not in exclude_list
+
+
+def get_parsed_stack_config(stack):
+    # In order to be compatible with Python 3.8 we need to use this hack to get the path:
+    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+    stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
+    try:
+        with stack_file_path:
+            stack_config = yaml.safe_load(open(stack_file_path, "r"))
+            return stack_config
+    except FileNotFoundError as error:
+        # We try here to generate a useful diagnostic error
+        # First check if the stack directory is present
+        stack_directory = stack_file_path.parent
+        if os.path.exists(stack_directory):
+            print(f"Error: stack.yml file is missing from stack: {stack}")
+        else:
+            print(f"Error: stack: {stack} does not exist")
+        print(f"Exiting, error: {error}")
+        sys.exit(1)

cli.py

@@ -27,13 +27,14 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 # TODO: this seems kind of weird and heavy on boilerplate -- check it is
 # the best Python can do for us.
 class Options(object):
-    def __init__(self, stack, quiet, verbose, dry_run, local_stack, debug):
+    def __init__(self, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error):
         self.stack = stack
         self.quiet = quiet
         self.verbose = verbose
         self.dry_run = dry_run
         self.local_stack = local_stack
         self.debug = debug
+        self.continue_on_error = continue_on_error
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
@@ -43,11 +44,12 @@ class Options(object):
 @click.option('--dry-run', is_flag=True, default=False)
 @click.option('--local-stack', is_flag=True, default=False)
 @click.option('--debug', is_flag=True, default=False)
+@click.option('--continue-on-error', is_flag=True, default=False)
 # See: https://click.palletsprojects.com/en/8.1.x/complex/#building-a-git-clone
 @click.pass_context
-def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug):
+def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error):
     """Laconic Stack Orchestrator"""
-    ctx.obj = Options(stack, quiet, verbose, dry_run, local_stack, debug)
+    ctx.obj = Options(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
 
 
 cli.add_command(setup_repositories.command, "setup-repositories")