Skip container build and actual deployment for testing

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -26,14 +26,8 @@ fi
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
-if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
-  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
 cd "${WORK_DIR}" || exit 1
 
-  ./build-webapp.sh || exit 1
-  exit 0
-fi
-
 if [ -f "next.config.mjs" ]; then
   NEXT_CONFIG_JS="next.config.mjs"
   IMPORT_OR_REQUIRE="import"

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -30,44 +30,36 @@ fi
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
 
-if [ -f "./run-webapp.sh" ]; then
+"$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
-  echo "Running webapp with run-webapp.sh ..."
+mv .next .next.old
-  cd "${WORK_DIR}" || exit 1
+mv .next-r/.next .
-  ./run-webapp.sh &
-  tpid=$!
-  wait $tpid
-else 
-  "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
-  mv .next .next.old
-  mv .next-r/.next .
 
-  if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
+if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
-    jq -e '.scripts.cerc_generate' package.json >/dev/null
+  jq -e '.scripts.cerc_generate' package.json >/dev/null
-    if [ $? -eq 0 ]; then
+  if [ $? -eq 0 ]; then
-      npm run cerc_generate > gen.out 2>&1 &
+    npm run cerc_generate > gen.out 2>&1 &
-      tail -f gen.out &
+    tail -f gen.out &
-      tpid=$!
+    tpid=$!
 
-      count=0
+    count=0
-      generate_done="false"
+    generate_done="false"
-      while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
+    while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
-        sleep 1
+      sleep 1
-        count=$((count + 1))
+      count=$((count + 1))
-        grep 'rendered as static' gen.out > /dev/null
+      grep 'rendered as static' gen.out > /dev/null
-        if [ $? -eq 0 ]; then
+      if [ $? -eq 0 ]; then
-          generate_done="true"
+        generate_done="true"
-        fi
-      done
-
-      if [ $generate_done != "true" ]; then
-        echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
-        exit 1
       fi
+    done
 
-      kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+    if [ $generate_done != "true" ]; then
-      tpid=""
+      echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
+      exit 1
     fi
-  fi
 
-  $CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}
+    kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+    tpid=""
+  fi
 fi
+
+$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}

stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from ruamel.yaml import YAML
 
 
 def create(context: DeploymentContext, extra_args):
@@ -22,12 +23,17 @@ def create(context: DeploymentContext, extra_args):
     # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment
     fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml')
 
+    with open(fixturenet_eth_compose_file, 'r') as yaml_file:
+        yaml = YAML()
+        yaml_data = yaml.load(yaml_file)
+
     new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh'
 
-    def add_geth_volume(yaml_data):
+    if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']:
-        if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']:
+        yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script)
-            yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script)
 
-    context.modify_yaml(fixturenet_eth_compose_file, add_geth_volume)
+    with open(fixturenet_eth_compose_file, 'w') as yaml_file:
+        yaml = YAML()
+        yaml.dump(yaml_data, yaml_file)
 
     return None

stack_orchestrator/data/stacks/test/stack.yml

@@ -2,6 +2,7 @@ version: "1.0"
 name: test
 description: "A test stack"
 repos:
+  - git.vdb.to/cerc-io/laconicd
   - git.vdb.to/cerc-io/test-project@test-branch
 containers:
   - cerc/test-container

stack_orchestrator/deploy/deployment_context.py

@@ -45,22 +45,20 @@ class DeploymentContext:
     def get_compose_dir(self):
         return self.deployment_dir.joinpath(constants.compose_dir_name)
 
-    def get_compose_file(self, name: str):
-        return self.get_compose_dir() / f"docker-compose-{name}.yml"
-
     def get_cluster_id(self):
         return self.id
 
-    def init(self, dir: Path):
+    def init(self, dir):
-        self.deployment_dir = dir.absolute()
+        self.deployment_dir = dir
         self.spec = Spec()
         self.spec.init_from_file(self.get_spec_file())
         self.stack = Stack(self.spec.obj["stack"])
         self.stack.init_from_file(self.get_stack_file())
         deployment_file_path = self.get_deployment_file()
         if deployment_file_path.exists():
-            obj = get_yaml().load(open(deployment_file_path, "r"))
+            with deployment_file_path:
-            self.id = obj[constants.cluster_id_key]
+                obj = get_yaml().load(open(deployment_file_path, "r"))
+                self.id = obj[constants.cluster_id_key]
         # Handle the case of a legacy deployment with no file
         # Code below is intended to match the output from _make_default_cluster_name()
         # TODO: remove when we no longer need to support legacy deployments
@@ -69,19 +67,3 @@ class DeploymentContext:
             unique_cluster_descriptor = f"{path},{self.get_stack_file()},None,None"
             hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16]
             self.id = f"{constants.cluster_name_prefix}{hash}"
-
-    def modify_yaml(self, file_path: Path, modifier_func):
-        """
-        Load a YAML from the deployment, apply a modification function, and write it back.
-        """
-        if not file_path.absolute().is_relative_to(self.deployment_dir):
-            raise ValueError(f"File is not inside deployment directory: {file_path}")
-
-        yaml = get_yaml()
-        with open(file_path, 'r') as f:
-            yaml_data = yaml.load(f)
-
-        modifier_func(yaml_data)
-
-        with open(file_path, 'w') as f:
-            yaml.dump(yaml_data, f)

stack_orchestrator/deploy/deployment_create.py

@@ -443,16 +443,18 @@ def _check_volume_definitions(spec):
 @click.command()
 @click.option("--spec-file", required=True, help="Spec file to use to create this deployment")
 @click.option("--deployment-dir", help="Create deployment files in this directory")
-@click.argument('extra_args', nargs=-1, type=click.UNPROCESSED)
+# TODO: Hack
+@click.option("--network-dir", help="Network configuration supplied in this directory")
+@click.option("--initial-peers", help="Initial set of persistent peers")
 @click.pass_context
-def create(ctx, spec_file, deployment_dir, extra_args):
+def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     deployment_command_context = ctx.obj
-    return create_operation(deployment_command_context, spec_file, deployment_dir, extra_args)
+    return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers)
 
 
 # The init command's implementation is in a separate function so that we can
 # call it from other commands, bypassing the click decoration stuff
-def create_operation(deployment_command_context, spec_file, deployment_dir, extra_args):
+def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers):
     parsed_spec = Spec(os.path.abspath(spec_file), get_parsed_deployment_spec(spec_file))
     _check_volume_definitions(parsed_spec)
     stack_name = parsed_spec["stack"]
@@ -539,7 +541,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, extr
     deployer_config_generator = getDeployerConfigGenerator(deployment_type, deployment_context)
     # TODO: make deployment_dir_path a Path above
     deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(deployment_context, extra_args)
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])
 
 
 # TODO: this code should be in the stack .py files but

stack_orchestrator/deploy/spec.py

@@ -92,8 +92,9 @@ class Spec:
         return self.obj.get(item, default)
 
     def init_from_file(self, file_path: Path):
-        self.obj = get_yaml().load(open(file_path, "r"))
+        with file_path:
-        self.file_path = file_path
+            self.obj = get_yaml().load(open(file_path, "r"))
+            self.file_path = file_path
 
     def get_image_registry(self):
         return self.obj.get(constants.image_registry_key)

stack_orchestrator/deploy/stack.py

@@ -27,4 +27,5 @@ class Stack:
         self.name = name
 
     def init_from_file(self, file_path: Path):
-        self.obj = get_yaml().load(open(file_path, "r"))
+        with file_path:
+            self.obj = get_yaml().load(open(file_path, "r"))

stack_orchestrator/deploy/webapp/deploy_webapp.py

@@ -92,6 +92,7 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist
         spec_file_name,
         deployment_dir,
         None,
+        None
     )
     # Fix up the container tag inside the deployment compose file
     _fixup_container_tag(deployment_dir, image)

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -172,6 +172,7 @@ def process_app_deployment_request(
         logger.log(
             f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}"
         )
+        # CREATES DEPLOYMENT DIR, NOT SKIPPING FOR TESTING
         deploy_webapp.create_deployment(
             ctx,
             deployment_dir,
@@ -214,21 +215,24 @@ def process_app_deployment_request(
             # add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
             logger.log("Tag complete")
         else:
-            extra_build_args = []  # TODO: pull from request
+            # SKIP BUILD
-            logger.log(f"Building container image: {deployment_container_tag}")
+            logger.log("TESTING: Skipping container build.")
-            build_container_image(
+
-                app, deployment_container_tag, extra_build_args, logger
+            # extra_build_args = []  # TODO: pull from request
-            )
+            # logger.log(f"Building container image: {deployment_container_tag}")
-            logger.log("Build complete")
+            # build_container_image(
-            logger.log(f"Pushing container image: {deployment_container_tag}")
+            #     app, deployment_container_tag, extra_build_args, logger
-            push_container_image(deployment_dir, logger)
+            # )
-            logger.log("Push complete")
+            # logger.log("Build complete")
-            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
+            # logger.log(f"Pushing container image: {deployment_container_tag}")
-            logger.log(
+            # push_container_image(deployment_dir, logger)
-                f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
+            # logger.log("Push complete")
-            )
+            # # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
-            # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            # logger.log(
-            logger.log("Tag complete")
+            #     f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
+            # )
+            # # add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            # logger.log("Tag complete")
     else:
         logger.log("Requested app is already deployed, skipping build and image push")
 
@@ -241,7 +245,9 @@ def process_app_deployment_request(
 
     # 8. update k8s deployment
     if needs_k8s_deploy:
-        deploy_to_k8s(deployment_record, deployment_dir, recreate_on_deploy, logger)
+        # SKIP DEPLOY
+        logger.log("TESTING: Skipping deployment to k8s.")
+        # deploy_to_k8s(deployment_record, deployment_dir, recreate_on_deploy, logger)
 
     logger.log("Publishing deployment to registry.")
     publish_deployment(

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -72,11 +72,12 @@ def process_app_removal_request(
     # TODO(telackey): Call the function directly.  The easiest way to build the correct click context is to
     # exec the process, but it would be better to refactor so we could just call down_operation with the
     # necessary parameters
-    down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
+    main_logger.log("TESTING: Skipping stopping deployment.")
-    if delete_volumes:
+    # down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
-        down_command.append("--delete-volumes")
+    # if delete_volumes:
-    result = subprocess.run(down_command)
+    #     down_command.append("--delete-volumes")
-    result.check_returncode()
+    # result = subprocess.run(down_command)
+    # result.check_returncode()
 
     removal_record = {
         "record": {

stack_orchestrator/util.py

@@ -180,7 +180,9 @@ def get_k8s_dir():
 def get_parsed_deployment_spec(spec_file):
     spec_file_path = Path(spec_file)
     try:
-        return get_yaml().load(open(spec_file_path, "r"))
+        with spec_file_path:
+            deploy_spec = get_yaml().load(open(spec_file_path, "r"))
+            return deploy_spec
     except FileNotFoundError as error:
         # We try here to generate a useful diagnostic error
         print(f"Error: spec file: {spec_file_path} does not exist")

tests/deploy/run-deploy-test.sh

@@ -14,13 +14,8 @@ delete_cluster_exit () {
 
 # Test basic stack-orchestrator deploy
 echo "Running stack-orchestrator deploy test"
-
+# Bit of a hack, test the most recent package
-if [ "$1" == "from-path" ]; then
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-    TEST_TARGET_SO="laconic-so"
-else
-    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-fi
-
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"