Update stack.yml

* Upgrade merkl-sushiswap-v3-watcher-ts release

* Increase blockDelayInMilliSecs for merkl-sushiswap-v3 watcher

* Upgrade sushiswap-v3-watcher-ts release

* Add sushiswap-v3 watcher to stack list

* Avoid mapping ports that are not required to be exposed

.gitea/workflows/test-k8s-deploy.yml

@@ -0,0 +1,55 @@
+name: K8s Deploy Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+    paths-ignore:
+      - '.gitea/workflows/triggers/*'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  test:
+    name: "Run deploy test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Install Go"
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: "Install Kind"
+        run: go install sigs.k8s.io/kind@v0.20.0
+      - name: "Debug Kind"
+        run: kind create cluster --retain && docker logs kind-control-plane

requirements.txt

@@ -1,4 +1,5 @@
 python-decouple>=3.8
+python-dotenv==1.0.0
 GitPython>=3.1.32
 tqdm>=4.65.0
 python-on-whales>=0.64.0

stack_orchestrator/build/build_webapp.py

@@ -65,6 +65,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
 
 
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
+    container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
                                                                     base_container.replace("/", "-"),

stack_orchestrator/data/compose/docker-compose-kubo.yml

@@ -2,7 +2,7 @@ version: "3.2"
 # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
 services:
   ipfs:
-    image: ipfs/kubo:master-2023-02-20-714a968
+    image: ipfs/kubo:v0.24.0
     restart: always
     volumes:
       - ./ipfs/import:/import

stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml

@@ -0,0 +1,8 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end:local
+    restart: always
+    ports:
+      - "3002:3002" #TODO make `3000` when using the deployment feature

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -0,0 +1,77 @@
+version: '3.2'
+
+services:
+  merkl-sushiswap-v3-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=merkl-sushiswap-v3-watcher,merkl-sushiswap-v3-watcher-job-queue
+      - POSTGRES_EXTENSION=merkl-sushiswap-v3-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  merkl-sushiswap-v3-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      merkl-sushiswap-v3-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-merkl-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  merkl-sushiswap-v3-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      merkl-sushiswap-v3-watcher-db:
+        condition: service_healthy
+      merkl-sushiswap-v3-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-merkl-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+    ports:
+      - "127.0.0.1:3007:3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  merkl_sushiswap_v3_watcher_db_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -0,0 +1,77 @@
+version: '3.2'
+
+services:
+  sushiswap-v3-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=sushiswap-v3-watcher,sushiswap-v3-watcher-job-queue
+      - POSTGRES_EXTENSION=sushiswap-v3-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  sushiswap-v3-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      sushiswap-v3-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  sushiswap-v3-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      sushiswap-v3-watcher-db:
+        condition: service_healthy
+      sushiswap-v3-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+    ports:
+      - "127.0.0.1:3008:3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  sushiswap_v3_watcher_db_data:

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Initializing watcher..."
+yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1))
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -0,0 +1,99 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "merkl-sushiswap-v3-watcher-db"
+  port = 5432
+  database = "merkl-sushiswap-v3-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@merkl-sushiswap-v3-watcher-db/merkl-sushiswap-v3-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+  prefetchBlocksInMem = false
+  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Initializing watcher..."
+yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1))
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -0,0 +1,99 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "sushiswap-v3-watcher-db"
+  port = 5432
+  database = "sushiswap-v3-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = true
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+  prefetchBlocksInMem = false
+  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
+ARG VARIANT=20-bullseye
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -28,28 +28,15 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
-
-# [Optional] Uncomment if you want to install an additional version of node using nvm
-# ARG EXTRA_NODE_VERSION=10
-# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
-
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
-COPY /scripts /scripts
+    && apt-get -y install --no-install-recommends jq gettext-base moreutils
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
-# RUN mkdir -p /config
-# COPY ./config.yml /config
-
-# Install simple web server for now (use nginx perhaps later)
-# RUN yarn global add http-server
-
 # Expose port for http
 EXPOSE 3000
 
+COPY /scripts /scripts
+
 # Default command sleeps forever so docker doesn't kill it
 CMD ["/scripts/start-serving-app.sh"]

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp

@@ -1,4 +1,8 @@
 FROM cerc/nextjs-base:local
+
+ARG CERC_NEXT_VERSION=keep
+ARG CERC_BUILD_TOOL
+
 WORKDIR /app
 COPY . .
 RUN rm -rf node_modules build .next*

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -8,6 +8,15 @@ WORK_DIR="${1:-./}"
 SRC_DIR="${2:-.next}"
 TRG_DIR="${3:-.next-r}"
 
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ]; then
+    CERC_BUILD_TOOL=npm
+  else
+    CERC_BUILD_TOOL=yarn
+  fi
+fi
+
 cd "${WORK_DIR}" || exit 1
 
 rm -rf "$TRG_DIR"
@@ -25,12 +34,12 @@ if [ -f ".env" ]; then
 fi
 
 for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o '^"CERC_RUNTIME_ENV[^\"]+"$'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)
     esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
-    echo "$cur_name=$cur_val"
+    echo "$f: $cur_name=$cur_val"
     sed -i "s/$orig_name/$esc_val/g" $f
   done
 done

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -4,27 +4,41 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
-cp next.config.js next.config.dist
+if [ ! -f "next.config.dist" ]; then
+  cp next.config.js next.config.dist
+fi
+
+which js-beautify >/dev/null
+if [ $? -ne 0 ]; then
+  npm i -g js-beautify
+fi
 
-npm i -g js-beautify
 js-beautify next.config.dist > next.config.js
+echo "" >> next.config.js
 
-npm install
-
-CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
-MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1)
-
-head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1
-
-cat > next.config.js.2 <<EOF
-const webpack = require('webpack');
+WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
+if [ -z "$WEBPACK_REQ_LINE" ]; then
+  cat > next.config.js.0 <<EOF
+    const webpack = require('webpack');
+EOF
+fi
 
+cat > next.config.js.1 <<EOF
 let envMap;
 try {
   // .env-list.json provides us a list of identifiers which should be replaced at runtime.
@@ -43,21 +57,57 @@ try {
 }
 EOF
 
-grep 'module.exports' next.config.js > next.config.js.3
+CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
+ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1)
+WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1)
+NEXT_SECTION_ADJUSTMENT=0
 
-cat > next.config.js.4 <<EOF
-  webpack: (config) => {
-    config.plugins.push(new webpack.DefinePlugin(envMap));
-    return config;
-  },
+if [ -n "$WEBPACK_CONF_LINE" ]; then
+  WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2)
+  head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2
+  cat > next.config.js.3 <<EOF
+      $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
 EOF
+  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE))
+elif [ -n "$ENV_LINE" ]; then
+  head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
+  cat > next.config.js.3 <<EOF
+    webpack: (config) => {
+      config.plugins.push(new webpack.DefinePlugin(envMap));
+      return config;
+    },
+EOF
+  NEXT_SECTION_ADJUSTMENT=1
+  NEXT_SECTION_LINE=$ENV_LINE
+else
+  echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
+  rm -f next.config.js.*
+  NEXT_SECTION_LINE=0
+fi
 
-tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5
+tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5
 
-cat next.config.js.* | js-beautify > next.config.js
+cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js
 rm next.config.js.*
 
 "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json
 
-npm run build
-rm .env-list.json
+if [ ! -f "package.dist" ]; then
+  cp package.json package.dist
+fi
+
+cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
+
+CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
+
+if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
+  echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with --build-arg CERC_NEXT_VERSION)"
+  cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json
+else
+  echo "'next' version specifier '$CUR_NEXT_VERSION' (override with --build-arg CERC_NEXT_VERSION)"
+fi
+
+$CERC_BUILD_TOOL install || exit 1
+$CERC_BUILD_TOOL run cerc_compile || exit 1
+
+exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh

@@ -20,5 +20,10 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -
   done
 done
 
+NEXT_CONF="next.config.js next.config.dist"
+for f in $NEXT_CONF; do
+    cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+done
+
 cat $TMPF | sort -u | jq  --raw-input .  | jq --slurp .
 rm -f $TMPF

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -5,9 +5,39 @@ fi
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
+
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
 
-rm -rf .next-r
 "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
-npm start .next-r -p ${CERC_LISTEN_PORT:-3000}
+mv .next .next.old
+mv .next-r/.next .
+
+if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
+  jq -e '.scripts.cerc_generate' package.json >/dev/null
+  if [ $? -eq 0 ]; then
+    npm run cerc_generate > gen.out 2>&1 &
+    tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
+    count=0
+    while [ $count -lt 10 ]; do
+      sleep 1
+      ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
+      if [ $? -ne 0 ]; then 
+        break
+      else
+        count=$((count + 1))
+      fi
+    done
+    kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+  fi
+fi
+
+$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}

stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the osmosis front end image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-merkl-sushiswap-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack
+
+docker build -t cerc/watcher-merkl-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/merkl-sushiswap-v3-watcher-ts

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-sushiswap-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-v3-watcher-ts

stack_orchestrator/data/container-image-list.txt

@@ -57,3 +57,5 @@ cerc/nitro-contracts
 cerc/mobymask-snap
 cerc/ponder
 cerc/nitro-rpc-client
+cerc/watcher-merkl-sushiswap-v3
+cerc/watcher-sushiswap-v3

stack_orchestrator/data/pod-list.txt

@@ -43,3 +43,5 @@ nitro-contracts
 mobymask-snap
 ponder
 ipld-eth-server-payments
+merkl-sushiswap-v3
+sushiswap-v3

stack_orchestrator/data/repository-list.txt

@@ -47,3 +47,5 @@ github.com/cerc-io/go-nitro
 github.com/cerc-io/ts-nitro
 github.com/cerc-io/mobymask-snap
 github.com/cerc-io/ponder
+github.com/cerc-io/merkl-sushiswap-v3-watcher-ts
+github.com/cerc-io/sushiswap-v3-watcher-ts

stack_orchestrator/data/stacks/graph-node/README.md

@@ -59,7 +59,7 @@ ports:
 Create deployment:
 
 ```bash
-laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
+laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
 ```
 
 ## Start the stack

stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md

@@ -0,0 +1,81 @@
+# Merkl SushiSwap v3 Watcher
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 build-containers
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step:
+
+  ```bash
+  # External Filecoin (ETH RPC) endpoint to point the watcher
+  CERC_ETH_RPC_ENDPOINT=
+  ```
+
+### Deploy the stack
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
+
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+* Open the GQL playground at http://localhost:3007/graphql
+
+  ```graphql
+  {
+    _meta {
+      block {
+        number
+        timestamp
+      }
+      hasIndexingErrors
+    }
+    
+    factories {
+      id
+      poolCount
+    }
+  }
+  ```
+
+## Clean up
+
+Stop all the services running in background:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=merkl_sushiswap_v3"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
+```

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: merkl-sushiswap-v3
+description: "SushiSwap v3 watcher stack"
+repos:
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.1
+containers:
+  - cerc/watcher-merkl-sushiswap-v3
+pods:
+  - watcher-merkl-sushiswap-v3

stack_orchestrator/data/stacks/osmosis/README.md

@@ -0,0 +1,35 @@
+# self-hosted osmosis
+
+Build and deploy:
+- 1) self-hosted gitea,
+- 2) an ipfs node,
+- 3) the osmosis front end,
+- 4) a laconicd chain
+
+
+```
+# support image for the gitea package registry
+laconic-so --stack build-support build-containers
+
+# todo: pre-run clone
+
+# clones and builds several things
+laconic-so --stack osmosis setup-repositories
+laconic-so --stack osmosis build-containers 
+laconic-so --stack osmosis deploy up
+```
+
+Setup a test chain:
+```
+export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
+
+laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+
+laconic-so --stack fixturenet-laconic-loaded build-containers
+
+export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
+
+laconic-so --stack fixturenet-laconic-loaded deploy up
+```
+
+then `docker exec` into the `laconicd` container and either export the private key or create a new one and send funds to it. Use that private key for `LACONIC_HOTWALLET_KEY`.

stack_orchestrator/data/stacks/osmosis/stack.yml

@@ -0,0 +1,27 @@
+version: "0.1"
+name: osmosis
+repos:
+  # these are for gitea
+  - git.vdb.to/cerc-io/hosting@names-for-so
+  - gitea.com/gitea/act_runner
+  # add the osmosis FE
+  - github.com/osmosis-labs/osmosis-frontend
+containers:
+  - cerc/act-runner
+  - cerc/act-runner-task-executor
+  # note: osmosis builds but doesn't run
+  - cerc/osmosis-front-end
+pods:
+  - name: gitea
+    repository: cerc-io/hosting
+    path: gitea
+    pre_start_command: "run-this-first.sh"
+    post_start_command: "initialize-gitea.sh"
+    # todo, e.g., mirroring all of osmosis repos: https://git.vdb.to/cerc-io/hosting/pulls/42
+  - name: act-runner
+    repository: cerc-io/hosting
+    path: act-runner
+    pre_start_command: "pre_start.sh"
+    post_start_command: "post_start.sh"
+  - osmosis-front-end
+  - kubo

stack_orchestrator/data/stacks/sushiswap-v3/README.md

@@ -0,0 +1,62 @@
+# SushiSwap v3 Watcher
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack sushiswap-v3 setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack sushiswap-v3 build-containers
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step:
+
+  ```bash
+  # External Filecoin (ETH RPC) endpoint to point the watcher
+  CERC_ETH_RPC_ENDPOINT=
+  ```
+
+### Deploy the stack
+
+```bash
+laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps
+
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+## Clean up
+
+Stop all the services running in background:
+
+```bash
+laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=sushiswap_v3"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3")
+```

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: sushiswap-v3
+description: "SushiSwap v3 watcher stack"
+repos:
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.1
+containers:
+  - cerc/watcher-sushiswap-v3
+pods:
+  - watcher-sushiswap-v3

stack_orchestrator/deploy/deploy_types.py

@@ -13,7 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from typing import List
+from typing import List, Mapping
 from dataclasses import dataclass
 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.deploy.deployer import Deployer
@@ -59,3 +59,8 @@ class LaconicStackSetupCommand:
 @dataclass
 class LaconicStackCreateCommand:
     network_dir: str
+
+
+@dataclass
+class DeployEnvVars:
+    map: Mapping[str, str]

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -19,6 +19,8 @@ from typing import Any, List, Set
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
 from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names, get_node_pv_mount_path
+from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map
+from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 
 
 class ClusterInfo:
@@ -26,11 +28,12 @@ class ClusterInfo:
     image_set: Set[str] = set()
     app_name: str = "test-app"
     deployment_name: str = "test-deployment"
+    environment_variables: DeployEnvVars
 
     def __init__(self) -> None:
         pass
 
-    def int_from_pod_files(self, pod_files: List[str]):
+    def int(self, pod_files: List[str], compose_env_file):
         self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
         # Find the set of images in the pods
         for pod_name in self.parsed_pod_yaml_map:
@@ -42,6 +45,9 @@ class ClusterInfo:
                 self.image_set.add(image)
         if opts.o.debug:
             print(f"image_set: {self.image_set}")
+        self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
+        if (opts.o.debug):
+            print(f"Env vars: {self.environment_variables.map}")
 
     def get_pvcs(self):
         result = []
@@ -97,6 +103,7 @@ class ClusterInfo:
                 container = client.V1Container(
                     name=container_name,
                     image=image,
+                    env=envs_from_environment_variables_map(self.environment_variables.map),
                     ports=[client.V1ContainerPort(container_port=80)],
                     volume_mounts=volume_mounts,
                     resources=client.V1ResourceRequirements(

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -41,7 +41,7 @@ class K8sDeployer(Deployer):
         self.deployment_dir = deployment_dir
         self.kind_cluster_name = compose_project_name
         self.cluster_info = ClusterInfo()
-        self.cluster_info.int_from_pod_files(compose_files)
+        self.cluster_info.int(compose_files, compose_env_file)
 
     def connect_api(self):
         config.load_kube_config(context=f"kind-{self.kind_cluster_name}")

stack_orchestrator/deploy/k8s/helpers.py

@@ -14,10 +14,11 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from kubernetes import client
+from dotenv import dotenv_values
 import os
 from pathlib import Path
 import subprocess
-from typing import Any, Set
+from typing import Any, Set, Mapping, List
 
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import get_yaml
@@ -194,6 +195,13 @@ def _generate_kind_port_mappings(parsed_pod_files):
     )
 
 
+def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]:
+    result = []
+    for env_var, env_val in map.items():
+        result.append(client.V1EnvVar(env_var, env_val))
+    return result
+
+
 # This needs to know:
 # The service ports for the cluster
 # The bind mounted volumes for the cluster
@@ -227,3 +235,7 @@ def generate_kind_config(deployment_dir: Path):
         f"{port_mappings_yml}\n"
         f"{mounts_yml}\n"
     )
+
+
+def env_var_map_from_file(file: Path) -> Mapping[str, str]:
+    return dotenv_values(file)

tests/k8s-deploy/run-deploy-test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+# Note: eventually this test should be folded into ../deploy/
+# but keeping it separate for now for convenience
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Dump environment variables for debugging
+echo "Environment variables:"
+env
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+# Test basic stack-orchestrator deploy
+test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
+test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
+$TEST_TARGET_SO --stack test deploy --deploy-to k8s init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+$TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+# Check the file writted by the create command in the stack now exists
+if [ ! -f "$test_deployment_dir/create-file" ]; then
+    echo "deploy create test: create output file not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+# And has the right content
+create_file_content=$(<$test_deployment_dir/create-file)
+if [ ! "$create_file_content" == "create-command-output-data"  ]; then
+    echo "deploy create test: create output file contents not correct"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create output file test: passed"
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+# TODO: add a check to see if the container is up
+# Sleep because k8s not up yet
+sleep 30
+# Check logs command works
+log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+    echo "deployment logs test: passed"
+else
+    echo "deployment logs test: FAILED"
+    exit 1
+fi
+# Check the config variable CERC_TEST_PARAM_1 was passed correctly
+if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
+    echo "deployment config test: passed"
+else
+    echo "deployment config test: FAILED"
+    exit 1
+fi
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"

tests/webapp-test/run-webapp-test.sh

@@ -24,7 +24,7 @@ git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BAS
 # Test webapp command execution
 $TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app
 
-UUID=`uuidgen`
+CHECK="SPECIAL_01234567890_TEST_STRING"
 
 set +e
 
@@ -34,7 +34,7 @@ wget -O test.before -m http://localhost:3000
 
 docker remove -f $CONTAINER_ID
 
-CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -d cerc/test-progressive-web-app:local)
 sleep 3
 wget -O test.after -m http://localhost:3000
 
@@ -43,7 +43,7 @@ docker remove -f $CONTAINER_ID
 echo "###########################################################################"
 echo ""
 
-grep "$UUID" test.before > /dev/null
+grep "$CHECK" test.before > /dev/null
 if [ $? -ne 1 ]; then
   echo "BEFORE: FAILED"
   exit 1
@@ -51,7 +51,7 @@ else
   echo "BEFORE: PASSED"
 fi
 
-grep "$UUID" test.after > /dev/null
+grep "$CHECK" test.after > /dev/null
 if [ $? -ne 0 ]; then
   echo "AFTER: FAILED"
   exit 1