Fix missing tty parameter. (#653)

Iskay/update optimism

docs/webapp.md

@@ -0,0 +1,54 @@
+### Building and Running Webapps
+
+It is possible to build and run Next.js webapps using the `build-webapp` and `run-webapp` subcommands.
+
+To make it easier to build once and deploy into different environments and with different configuration,
+compilation and static page generation are separated in the `build-webapp` and `run-webapp` steps.
+
+This offers much more flexibilty than standard Next.js build methods, since any environment variables accessed
+via `process.env`, whether for pages or for API, will have values drawn from their runtime deployment environment,
+not their build environment. 
+
+## Building
+
+Building usually requires no additional configuration.  By default, the Next.js version specified in `package.json`
+is used, and either `yarn` or `npm` will be used automatically depending on which lock files are present.  These
+can be overidden with the build arguments `CERC_NEXT_VERSION` and `CERC_BUILD_TOOL` respectively.  For example: `--extra-build-args "--build-arg CERC_NEXT_VERSION=13.4.12"`
+
+**Example**:
+```
+$ cd ~/cerc
+$ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git
+$ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app
+...
+
+Built host container for ~/cerc/test-progressive-web-app with tag:
+
+    cerc/test-progressive-web-app:local
+
+To test locally run:
+
+    laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment.env
+
+```
+
+## Running
+
+With `run-webapp` a new container will be launched with runtime configuration provided by `--env-file` (if specified) and published on an available port.  Multiple instances can be launched with different configuration.
+
+**Example**:
+```
+# Production env
+$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/production.env
+
+Image: cerc/test-progressive-web-app:local
+ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05
+URL: http://localhost:32768
+
+# Dev env
+$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/dev.env
+
+Image: cerc/test-progressive-web-app:local
+ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c
+URL: http://localhost:32769
+```

stack_orchestrator/build/build_containers.py

@@ -27,7 +27,7 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
@@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(container,
+def process_container(stack: str,
+                      container,
                       container_build_dir: str,
                       container_build_env: dict,
                       dev_root_path: str,
@@ -69,12 +70,29 @@ def process_container(container,
                       ):
     if not quiet:
         print(f"Building: {container}")
-    build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
-    build_script_filename = os.path.join(build_dir, "build.sh")
+
+    default_container_tag = f"{container}:local"
+    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+
+    # Check if this is in an external stack
+    if stack_is_external(stack):
+        container_parent_dir = Path(stack).joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
+        # Now check if the container exists in the external stack.
+        if not temp_build_script_filename.exists():
+            # If not, revert to building an internal container
+            container_parent_dir = container_build_dir
+    else:
+        container_parent_dir = container_build_dir
+
+    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    build_script_filename = build_dir.joinpath("build.sh")
+
     if verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
-        build_command = build_script_filename
+        build_command = build_script_filename.as_posix()
     else:
         if verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
@@ -84,7 +102,7 @@ def process_container(container,
         repo_full_path = os.path.join(dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
         build_command = os.path.join(container_build_dir,
-                                     "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
+                                     "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
@@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(container, container_build_dir, container_build_env,
+            process_container(stack, container, container_build_dir, container_build_env,
                               dev_root_path, quiet, verbose, dry_run, continue_on_error)
         else:
             if verbose:

stack_orchestrator/build/build_webapp.py

@@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
 
@@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
     container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)

stack_orchestrator/constants.py

@@ -0,0 +1,16 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+stack_file_name = "stack.yml"

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -0,0 +1,77 @@
+version: '3.2'
+
+services:
+  merkl-sushiswap-v3-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=merkl-sushiswap-v3-watcher,merkl-sushiswap-v3-watcher-job-queue
+      - POSTGRES_EXTENSION=merkl-sushiswap-v3-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  merkl-sushiswap-v3-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      merkl-sushiswap-v3-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-merkl-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  merkl-sushiswap-v3-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      merkl-sushiswap-v3-watcher-db:
+        condition: service_healthy
+      merkl-sushiswap-v3-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-merkl-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+    ports:
+      - "127.0.0.1:3007:3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  merkl_sushiswap_v3_watcher_db_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -0,0 +1,77 @@
+version: '3.2'
+
+services:
+  sushiswap-v3-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=sushiswap-v3-watcher,sushiswap-v3-watcher-job-queue
+      - POSTGRES_EXTENSION=sushiswap-v3-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  sushiswap-v3-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      sushiswap-v3-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  sushiswap-v3-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      sushiswap-v3-watcher-db:
+        condition: service_healthy
+      sushiswap-v3-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+    ports:
+      - "127.0.0.1:3008:3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  sushiswap_v3_watcher_db_data:

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Initializing watcher..."
+yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1))
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -0,0 +1,99 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "merkl-sushiswap-v3-watcher-db"
+  port = 5432
+  database = "merkl-sushiswap-v3-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@merkl-sushiswap-v3-watcher-db/merkl-sushiswap-v3-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+  prefetchBlocksInMem = false
+  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Initializing watcher..."
+yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1))
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -0,0 +1,99 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "sushiswap-v3-watcher-db"
+  port = 5432
+  database = "sushiswap-v3-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = true
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+  prefetchBlocksInMem = false
+  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -28,28 +28,15 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
-
-# [Optional] Uncomment if you want to install an additional version of node using nvm
-# ARG EXTRA_NODE_VERSION=10
-# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
-
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
-COPY /scripts /scripts
+    && apt-get -y install --no-install-recommends jq gettext-base moreutils
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
-# RUN mkdir -p /config
-# COPY ./config.yml /config
-
-# Install simple web server for now (use nginx perhaps later)
-# RUN yarn global add http-server
-
 # Expose port for http
 EXPOSE 3000
 
+COPY /scripts /scripts
+
 # Default command sleeps forever so docker doesn't kill it
-CMD ["/scripts/start-serving-app.sh"]
+ENTRYPOINT ["/scripts/start-serving-app.sh"]

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp

@@ -1,4 +1,8 @@
 FROM cerc/nextjs-base:local
+
+ARG CERC_NEXT_VERSION=keep
+ARG CERC_BUILD_TOOL
+
 WORKDIR /app
 COPY . .
 RUN rm -rf node_modules build .next*

stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh

@@ -11,3 +11,19 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
 
 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+
+if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
+  cat <<EOF
+
+#################################################################
+
+Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
+
+    $CERC_CONTAINER_BUILD_TAG
+
+To test locally run:
+
+    laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
+
+EOF
+fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -8,6 +8,15 @@ WORK_DIR="${1:-./}"
 SRC_DIR="${2:-.next}"
 TRG_DIR="${3:-.next-r}"
 
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ]; then
+    CERC_BUILD_TOOL=npm
+  else
+    CERC_BUILD_TOOL=yarn
+  fi
+fi
+
 cd "${WORK_DIR}" || exit 1
 
 rm -rf "$TRG_DIR"
@@ -29,6 +38,9 @@ for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)
+    if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
+      cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
+    fi
     esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
     echo "$f: $cur_name=$cur_val"
     sed -i "s/$orig_name/$esc_val/g" $f

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -4,8 +4,17 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
@@ -20,6 +29,7 @@ if [ $? -ne 0 ]; then
 fi
 
 js-beautify next.config.dist > next.config.js
+echo "" >> next.config.js
 
 WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
 if [ -z "$WEBPACK_REQ_LINE" ]; then
@@ -58,7 +68,7 @@ if [ -n "$WEBPACK_CONF_LINE" ]; then
   cat > next.config.js.3 <<EOF
       $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
 EOF
-  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE - 1))
+  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE))
 elif [ -n "$ENV_LINE" ]; then
   head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
   cat > next.config.js.3 <<EOF
@@ -67,7 +77,7 @@ elif [ -n "$ENV_LINE" ]; then
       return config;
     },
 EOF
-  NEXT_SECTION_ADJUSTMENT=2
+  NEXT_SECTION_ADJUSTMENT=1
   NEXT_SECTION_LINE=$ENV_LINE
 else
   echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
@@ -88,7 +98,16 @@ fi
 
 cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
 
-npm install || exit 1
-npm run cerc_compile || exit 1
+CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
+
+if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
+  echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with --build-arg CERC_NEXT_VERSION)"
+  cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json
+else
+  echo "'next' version specifier '$CUR_NEXT_VERSION' (override with --build-arg CERC_NEXT_VERSION)"
+fi
+
+$CERC_BUILD_TOOL install || exit 1
+$CERC_BUILD_TOOL run cerc_compile || exit 1
 
 exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -3,7 +3,25 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+tpid=""
+
+ctrl_c() {
+    kill $tpid $(ps -ef | grep node | grep next | awk '{print $2}') 2>/dev/null
+}
+
+trap ctrl_c INT
+
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
 
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
@@ -16,19 +34,28 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
   jq -e '.scripts.cerc_generate' package.json >/dev/null
   if [ $? -eq 0 ]; then
     npm run cerc_generate > gen.out 2>&1 &
-    tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
+    tail -f gen.out &
+    tpid=$!
+
     count=0
-    while [ $count -lt 10 ]; do
+    generate_done="false"
+    while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
       sleep 1
-      ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
-      if [ $? -ne 0 ]; then 
-        break
-      else
-        count=$((count + 1))
+      count=$((count + 1))
+      grep 'rendered as static HTML' gen.out > /dev/null
+      if [ $? -eq 0 ]; then
+        generate_done="true"
       fi
     done
-    kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+
+    if [ $generate_done != "true" ]; then
+      echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
+      exit 1
+    fi
+
+    kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+    tpid=""
   fi
 fi
 
-npm start . -p ${CERC_LISTEN_PORT:-3000}
+$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-merkl-sushiswap-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack
+
+docker build -t cerc/watcher-merkl-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/merkl-sushiswap-v3-watcher-ts

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-sushiswap-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-v3-watcher-ts

stack_orchestrator/data/container-image-list.txt

@@ -57,3 +57,5 @@ cerc/nitro-contracts
 cerc/mobymask-snap
 cerc/ponder
 cerc/nitro-rpc-client
+cerc/watcher-merkl-sushiswap-v3
+cerc/watcher-sushiswap-v3

stack_orchestrator/data/pod-list.txt

@@ -43,3 +43,5 @@ nitro-contracts
 mobymask-snap
 ponder
 ipld-eth-server-payments
+merkl-sushiswap-v3
+sushiswap-v3

stack_orchestrator/data/repository-list.txt

@@ -47,3 +47,5 @@ github.com/cerc-io/go-nitro
 github.com/cerc-io/ts-nitro
 github.com/cerc-io/mobymask-snap
 github.com/cerc-io/ponder
+github.com/cerc-io/merkl-sushiswap-v3-watcher-ts
+github.com/cerc-io/sushiswap-v3-watcher-ts

stack_orchestrator/data/stacks/graph-node/README.md

@@ -59,7 +59,7 @@ ports:
 Create deployment:
 
 ```bash
-laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
+laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
 ```
 
 ## Start the stack

stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md

@@ -0,0 +1,81 @@
+# Merkl SushiSwap v3 Watcher
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 build-containers
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step:
+
+  ```bash
+  # External Filecoin (ETH RPC) endpoint to point the watcher
+  CERC_ETH_RPC_ENDPOINT=
+  ```
+
+### Deploy the stack
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
+
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+* Open the GQL playground at http://localhost:3007/graphql
+
+  ```graphql
+  {
+    _meta {
+      block {
+        number
+        timestamp
+      }
+      hasIndexingErrors
+    }
+    
+    factories {
+      id
+      poolCount
+    }
+  }
+  ```
+
+## Clean up
+
+Stop all the services running in background:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=merkl_sushiswap_v3"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
+```

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: merkl-sushiswap-v3
+description: "SushiSwap v3 watcher stack"
+repos:
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.2
+containers:
+  - cerc/watcher-merkl-sushiswap-v3
+pods:
+  - watcher-merkl-sushiswap-v3

stack_orchestrator/data/stacks/sushiswap-v3/README.md

@@ -0,0 +1,62 @@
+# SushiSwap v3 Watcher
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack sushiswap-v3 setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack sushiswap-v3 build-containers
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step:
+
+  ```bash
+  # External Filecoin (ETH RPC) endpoint to point the watcher
+  CERC_ETH_RPC_ENDPOINT=
+  ```
+
+### Deploy the stack
+
+```bash
+laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps
+
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+## Clean up
+
+Stop all the services running in background:
+
+```bash
+laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=sushiswap_v3"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3")
+```

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: sushiswap-v3
+description: "SushiSwap v3 watcher stack"
+repos:
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.2
+containers:
+  - cerc/watcher-sushiswap-v3
+pods:
+  - watcher-sushiswap-v3

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -61,9 +61,10 @@ class DockerDeployer(Deployer):
         except DockerException as e:
             raise DeployerException(e)
 
-    def run(self, image, command, user, volumes, entrypoint=None):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
         try:
-            return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint)
+            return self.docker.run(image=image, command=command, user=user, volumes=volumes,
+                                   entrypoint=entrypoint, envs=env, detach=detach, publish_all=True)
         except DockerException as e:
             raise DeployerException(e)
 

stack_orchestrator/deploy/deploy.py

@@ -155,7 +155,7 @@ def exec_operation(ctx, extra_args):
         if global_context.verbose:
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
-            ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env)
+            ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env, tty=True)
         except DeployerException:
             print("container command returned error exit status")
 

stack_orchestrator/deploy/deployer.py

@@ -44,7 +44,7 @@ class Deployer(ABC):
         pass
 
     @abstractmethod
-    def run(self, image, command, user, volumes, entrypoint):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
         pass
 
 

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -120,7 +120,7 @@ class K8sDeployer(Deployer):
         log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
         return log_stream_from_string(log_data)
 
-    def run(self, image, command, user, volumes, entrypoint=None):
+    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False):
         # We need to figure out how to do this -- check why we're being called first
         pass
 

stack_orchestrator/deploy/run_webapp.py

@@ -0,0 +1,59 @@
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# Builds webapp containers
+
+# env vars:
+# CERC_REPO_BASE_DIR defaults to ~/cerc
+
+# TODO: display the available list of containers; allow re-build of either all or specific containers
+
+import hashlib
+import click
+
+from dotenv import dotenv_values
+from stack_orchestrator.deploy.deployer_factory import getDeployer
+
+
+@click.command()
+@click.option("--image", help="image to deploy", required=True)
+@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')")
+@click.option("--env-file", help="environment file for webapp")
+@click.pass_context
+def command(ctx, image, deploy_to, env_file):
+    '''build the specified webapp container'''
+
+    env = {}
+    if env_file:
+        env = dotenv_values(env_file)
+
+    unique_cluster_descriptor = f"{image},{env}"
+    hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()
+    cluster = f"laconic-webapp-{hash}"
+
+    deployer = getDeployer(deploy_to,
+                           deployment_dir=None,
+                           compose_files=None,
+                           compose_project_name=cluster,
+                           compose_env_file=None)
+
+    container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True)
+
+    # Make configurable?
+    webappPort = "3000/tcp"
+    # TODO: This assumes a Docker container object...
+    if webappPort in container.network_settings.ports:
+        mapping = container.network_settings.ports[webappPort][0]
+        print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""")

stack_orchestrator/main.py

@@ -20,6 +20,7 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
+from stack_orchestrator.deploy import run_webapp
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -50,6 +51,7 @@ cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(build_npms.command, "build-npms")
 cli.add_command(build_webapp.command, "build-webapp")
+cli.add_command(run_webapp.command, "run-webapp")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")

stack_orchestrator/repos/setup_repositories.py

@@ -25,7 +25,8 @@ import click
 import importlib.resources
 from pathlib import Path
 import yaml
-from stack_orchestrator.util import include_exclude_check
+from stack_orchestrator.constants import stack_file_name
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
 
 
 class GitProgress(git.RemoteProgress):
@@ -238,13 +239,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
 
     repos_in_scope = []
     if stack:
-        # In order to be compatible with Python 3.8 we need to use this hack to get the path:
-        # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
-        stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
+        if stack_is_external(stack):
+            stack_file_path = Path(stack).joinpath(stack_file_name)
+        else:
+            # In order to be compatible with Python 3.8 we need to use this hack to get the path:
+            # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
+            stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
+        if not stack_file_path.exists():
+            error_exit(f"stack {stack} does not exist")
         with stack_file_path:
             stack_config = yaml.safe_load(open(stack_file_path, "r"))
-            # TODO: syntax check the input here
-            repos_in_scope = stack_config['repos']
+            if "repos" not in stack_config:
+                error_exit(f"stack {stack} does not define any repositories")
+            else:
+                repos_in_scope = stack_config["repos"]
     else:
         repos_in_scope = all_repos
 

stack_orchestrator/util.py

@@ -150,6 +150,12 @@ def get_parsed_deployment_spec(spec_file):
         sys.exit(1)
 
 
+def stack_is_external(stack: str):
+    # Bit of a hack: if the supplied stack string represents
+    # a path that exists then we assume it must be external
+    return Path(stack).exists() if stack is not None else False
+
+
 def get_yaml():
     # See: https://stackoverflow.com/a/45701840/1701505
     yaml = ruamel.yaml.YAML()
@@ -167,3 +173,8 @@ def global_options(ctx):
 # TODO: hack
 def global_options2(ctx):
     return ctx.parent.obj
+
+
+def error_exit(s):
+    print(f"ERROR: {s}")
+    sys.exit(1)

tests/webapp-test/run-webapp-test.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 set -e
+
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
+
 # Dump environment variables for debugging
 echo "Environment variables:"
 env
@@ -24,26 +26,28 @@ git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BAS
 # Test webapp command execution
 $TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app
 
-UUID=`uuidgen`
+CHECK="SPECIAL_01234567890_TEST_STRING"
 
 set +e
 
-CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
 sleep 3
-wget -O test.before -m http://localhost:3000
+wget -t 7 -O test.before -m http://localhost:3000
 
+docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
-CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
 sleep 3
-wget -O test.after -m http://localhost:3000
+wget -t 7 -O test.after -m http://localhost:3000
 
+docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
 echo "###########################################################################"
 echo ""
 
-grep "$UUID" test.before > /dev/null
+grep "$CHECK" test.before > /dev/null
 if [ $? -ne 1 ]; then
   echo "BEFORE: FAILED"
   exit 1
@@ -51,7 +55,7 @@ else
   echo "BEFORE: PASSED"
 fi
 
-grep "$UUID" test.after > /dev/null
+grep "$CHECK" test.after > /dev/null
 if [ $? -ne 0 ]; then
   echo "AFTER: FAILED"
   exit 1