tightening up

Reviewed-on: #815
Co-authored-by: David Boreham <david@bozemanpass.com>
Co-committed-by: David Boreham <david@bozemanpass.com>

.gitea/workflows/test-external-stack.yml

@@ -0,0 +1,58 @@
+name: External Stack Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-external-stack'
+      - '.gitea/workflows/test-external-stack.yml'
+      - 'tests/external-stack/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '8 19 * * *'
+
+jobs:
+  test:
+    name: "Run external stack test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run external stack tests"
+        run: ./tests/external-stack/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/triggers/test-external-stack

@@ -0,0 +1,2 @@
+Change this file to trigger running the external-stack CI job
+trigger

stack_orchestrator/build/build_containers.py

@@ -71,7 +71,7 @@ def process_container(build_context: BuildContext) -> bool:
 
     # Check if this is in an external stack
     if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).joinpath("container-build")
+        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
         temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
         temp_build_script_filename = temp_build_dir.joinpath("build.sh")
         # Now check if the container exists in the external stack.

stack_orchestrator/data/compose/docker-compose-fixturenet-lotus.yml

@@ -5,7 +5,7 @@ services:
     hostname: lotus-miner
     env_file:
       - ../config/fixturenet-lotus/lotus-env.env
-    image: cerc/lotus:local
+    image: cerc/lotus-fixturenet:local
     volumes:
       - ../config/fixturenet-lotus/setup-miner.sh:/docker-entrypoint-scripts.d/setup-miner.sh
       - ../config/fixturenet-lotus/fund-account.sh:/fund-account.sh
@@ -26,7 +26,7 @@ services:
     environment:
       # Use 0.0.0.0 so that calls can be made from outside the container
       - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus:local
+    image: cerc/lotus-fixturenet:local
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
       - lotus_node_1_params:/var/tmp/filecoin-proof-parameters
@@ -42,10 +42,10 @@ services:
       - lotus-miner
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
     ports:
-      - "1234"
-      - "2345"
-      - "3456"
-      - "1777"
+      - "1235"
+      - "2346"
+      - "3457"
+      - "1778"
 
   lotus-node-2:
     hostname: lotus-node-2
@@ -54,7 +54,7 @@ services:
     environment:
       # Use 0.0.0.0 so that calls can be made from outside the container
       - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
-    image: cerc/lotus:local
+    image: cerc/lotus-fixturenet:local
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
       - lotus_node_2_params:/var/tmp/filecoin-proof-parameters
@@ -70,10 +70,10 @@ services:
       - lotus-miner
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-node.sh"]
     ports:
-      - "1234"
-      - "2345"
-      - "3456"
-      - "1777"
+      - "1236"
+      - "2347"
+      - "3458"
+      - "1779"
 
 volumes:
   lotus_miner_params:

stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -6,6 +6,7 @@ services:
       - ../config/fixturenet-eth/fixturenet-eth.env
     environment:
       RUN_BOOTNODE: "true"
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_bootnode_geth_data:/root/ethdata

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana:10.2.2
+    image: grafana/grafana:10.2.3
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -16,8 +16,12 @@ services:
       postgres_pass: password
       postgres_db: graph-node
       ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
+      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
       GRAPH_LOG: debug
       ETHEREUM_REORG_THRESHOLD: 3
+      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
+      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
+      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
     entrypoint: ["bash", "-c"]
     # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
     command: |

stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml

@@ -53,8 +53,6 @@ services:
       --nodiscover
       --maxpeers=0
       --rollup.disabletxpoolgossip=true
-      --override.canyon="${OVERRIDE_CANYON:-0}"
-      --override.ecotone="${OVERRIDE_ECOTONE:-1716843599}"
     env_file: 
       - ../config/mainnet-blast/${NETWORK:-mainnet}.config
     depends_on:
@@ -69,14 +67,13 @@ services:
       - "7300"
     command: >
       op-node
-      --l1="${L1_RPC:-https://eth-mainnet-1.vdb.to/}"
-      --l1.beacon="${L1_RPC_BEACON:-https://eth-mainnet-1.vdb.to/}"
+      --l1="https://eth-mainnet-1.vdb.to/"
+      --metrics.enabled
       --l1.rpckind="any"
       --l1.trustrpc=true
-      --l2="${L2_RPC:-http://blast-geth:8551}"
+      --l2="http://blast-geth:8551"
       --l2.jwt-secret=/blast/jwt.txt
       --rollup.config="/blast/rollup.json"
-      --metrics.enabled
     depends_on:
       - blast-geth
     env_file: 

stack_orchestrator/data/compose/docker-compose-mainnet-lotus.yml

@@ -0,0 +1,23 @@
+volumes:
+  parameters:
+  lotus-repo:
+  lotus_shared:
+
+services:
+  lotus:
+    environment:
+      - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http
+      - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/1235,/ip6/::/tcp/1235,/ip4/0.0.0.0/udp/1235/quic-v1,/ip6/::/udp/1235/quic-v1,/ip4/0.0.0.0/udp/1235/quic-v1/webtransport,/ip6/::/udp/1235/quic-v1/webtransport
+    image: cerc/lotus-mainnet:local
+    volumes:
+      - parameters:/var/tmp/filecoin-proof-parameters
+      - lotus-repo:/var/lib/lotus
+      - lotus_shared:/root/.lotus
+    ports:
+      - "1234"
+      - "1235"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 30s
+    entrypoint: ["sh", "/docker-lotus-entrypoint.sh"]

stack_orchestrator/data/config/fixturenet-eth/fixturenet-eth.env

@@ -22,4 +22,7 @@ CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
 CERC_STATEDIFF_WORKERS=2
 
 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
-CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+
+# Used by Lighthouse
+SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -102,6 +102,17 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
     fi
   fi
 
+  # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  else
+    sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  fi
+
   # Allocate genesis accounts (cosmos formatted addresses)
   laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
 

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -65,3 +65,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: postgres-exporter:9187
+
+  - job_name: laconicd
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      # Add laconicd REST endpoint target with host and port (1317)
+      # - targets: ['example-host:1317']
+    params:
+      format: ['prometheus']

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -771,3 +771,81 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+
+      # Ajna
+      - uid: ajna_diff_external
+        title: ajna_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -1,26 +1,23 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+FROM ethpandaops/ethereum-genesis-generator:3.0.0 AS ethgen
 
 FROM golang:1.20-alpine as builder
 
-RUN apk add --no-cache python3 py3-pip
-
-COPY genesis /opt/genesis
+RUN apk add --no-cache python3 py3-pip make bash envsubst jq
 
 # Install ethereum-genesis-generator tools
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
 RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
-# web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==v6.15.1"
 RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
+# Install tool to generate initial block
+RUN go install github.com/cerc-io/eth-dump-genblock@b29516740fc01cf1d1d623acbfd0e9a2b6440a96
+
 # Build genesis config
-RUN apk add --no-cache make bash envsubst jq
+COPY genesis /opt/genesis
 RUN cd /opt/genesis && make genesis-el
 
 # Snag the genesis block info.
-RUN go install github.com/cerc-io/eth-dump-genblock@latest
 RUN eth-dump-genblock /opt/genesis/build/el/geth.json > /opt/genesis/build/el/genesis_block.json
 
 FROM alpine:latest

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh

@@ -9,32 +9,8 @@ mkdir -p ../build/el
 tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
 envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml
 
-ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
-homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
-eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
-eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
-eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
-byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
-constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
-petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
-istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
-berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
-london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
-merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
-
 python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
-  jq ".config.terminalTotalDifficulty=$ttd" | \
-  jq ".config.homesteadBlock=$homestead_block" | \
-  jq ".config.eip150Block=$eip150_block" | \
-  jq ".config.eip155Block=$eip155_block" | \
-  jq ".config.eip158Block=$eip158_block" | \
-  jq ".config.byzantiumBlock=$byzantium_block" | \
-  jq ".config.constantinopleBlock=$constantinople_block" | \
-  jq ".config.petersburgBlock=$petersburg_block" | \
-  jq ".config.istanbulBlock=$istanbul_block" | \
-  jq ".config.berlinBlock=$berlin_block" | \
-  jq ".config.londonBlock=$london_block" | \
-  jq ".config.mergeForkBlock=$merge_fork_block" | \
-  jq ".config.mergeNetsplitBlock=$merge_fork_block" \
+  jq 'del(.config.pragueTime)' \
   > ../build/el/geth.json
+
 python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml

@@ -10,22 +10,8 @@ el_premine_addrs: {}
 chain_id: 1212
 deposit_contract_address: "0x1212121212121212121212121212121212121212"
 genesis_timestamp: 0
-terminal_total_difficulty: 1000
-homestead_block: 1
-eip150_block: 1
-eip155_block: 1
-eip158_block: 1
-byzantium_block: 1
-constantinople_block: 1
-petersburg_block: 1
-istanbul_block: 1
-berlin_block: 1
-london_block: 1
-merge_fork_block: 1
-
-clique:
-  enabled: false
-  signers:
-    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
-    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
-    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd
+genesis_delay: 0
+deneb_fork_epoch: 0
+# note: only needed as workaround https://github.com/ethpandaops/ethereum-genesis-generator/pull/105
+electra_fork_epoch: 0
+slot_duration_in_seconds: 3

stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -6,7 +6,7 @@ fi
 
 ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
-NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+NETRESTRICT=`ip addr | grep -w inet | grep -v '127.0' | awk '{print $2}'`
 CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
 CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"
 
@@ -102,6 +102,13 @@ else
       fi
     fi
 
+    OTHER_OPTS=""
+    # miner options were removed in v1.12
+    GETH_VERSION=$(geth --version | grep -io '[0-9][0-9a-z.-]*')
+    if echo -e "$GETH_VERSION\n1.12" | sort -Vc; then
+      OTHER_OPTS="--miner.threads=1"
+    fi
+
     $START_CMD \
       --datadir="${CERC_ETH_DATADIR}" \
       --bootnodes="${ENODE}" \
@@ -126,12 +133,12 @@ else
       --cache.preimages \
       --syncmode=full \
       --mine \
-      --miner.threads=1 \
       --metrics \
       --metrics.addr="0.0.0.0" \
       --verbosity=${CERC_GETH_VERBOSITY:-3} \
       --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
       --miner.etherbase="${ETHERBASE}" \
+      ${OTHER_OPTS} \
       ${STATEDIFF_OPTS} \
       &
 

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile

@@ -1,5 +1,4 @@
 FROM cerc/lighthouse-cli:local AS lcli
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-eth-genesis:local AS fnetgen
 
 FROM cerc/lighthouse:local
@@ -12,16 +11,13 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-COPY genesis /opt/testnet
-COPY run-cl.sh /opt/testnet/run.sh
-
 COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
-COPY --from=ethgen /apps /apps
 COPY --from=fnetgen /opt/genesis/el /opt/testnet/el
 COPY --from=fnetgen /opt/genesis/build/el /opt/testnet/build/el
 
+COPY genesis /opt/testnet
+COPY run-cl.sh /opt/testnet/run.sh
+
 RUN cd /opt/testnet && make genesis-cl
 
 # Work around some bugs in lcli where the default path is always used.

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh

@@ -10,7 +10,6 @@ set -Eeuo pipefail
 source ./vars.env
 
 SUBSCRIBE_ALL_SUBNETS=
-DEBUG_LEVEL=${DEBUG_LEVEL:-debug}
 
 # Get positional arguments
 data_dir=$DATADIR/node_${NODE_NUMBER}

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh

@@ -9,8 +9,6 @@ set -Eeuo pipefail
 
 source ./vars.env
 
-DEBUG_LEVEL=${1:-info}
-
 echo "Starting bootnode"
 
 # Clean up existing ENR dir to avoid node connectivity issues on a restart

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# See https://github.com/sigp/lighthouse/scripts/local_testnet/setup.sh
 #
 # Deploys the deposit contract and makes deposits for $VALIDATOR_COUNT insecure deterministic validators.
 # Produces a testnet specification and a genesis state where the genesis time
@@ -24,23 +25,26 @@ echo "(Note: errors of the form 'WARN: Scrypt parameters are too weak...' below
 lcli \
   new-testnet \
   --spec $SPEC_PRESET \
-  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --testnet-dir $TESTNET_DIR \
+  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --min-genesis-active-validator-count $GENESIS_VALIDATOR_COUNT \
   --validator-count $VALIDATOR_COUNT \
   --min-genesis-time $GENESIS_TIME \
   --genesis-delay $GENESIS_DELAY \
   --genesis-fork-version $GENESIS_FORK_VERSION \
   --altair-fork-epoch $ALTAIR_FORK_EPOCH \
-  --bellatrix-fork-epoch $MERGE_FORK_EPOCH \
+  --bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \
+  --capella-fork-epoch $CAPELLA_FORK_EPOCH \
+  --deneb-fork-epoch $DENEB_FORK_EPOCH \
   --eth1-id $ETH1_CHAIN_ID \
   --eth1-block-hash $ETH1_BLOCK_HASH \
   --eth1-follow-distance 1 \
   --seconds-per-slot $SECONDS_PER_SLOT \
   --seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \
+  --interop-genesis-state \
   --force
 
-echo Specification generated at $TESTNET_DIR.
+echo Specification and genesis.ssz generated at $TESTNET_DIR.
 echo "Generating $VALIDATOR_COUNT validators concurrently... (this may take a while)"
 
 lcli \
@@ -50,13 +54,3 @@ lcli \
   --node-count $BN_COUNT
 
 echo Validators generated with keystore passwords at $DATADIR.
-echo "Building genesis state... (this might take a while)"
-
-lcli \
-  interop-genesis \
-  --spec $SPEC_PRESET \
-  --genesis-time $GENESIS_TIME \
-  --testnet-dir $TESTNET_DIR \
-  $GENESIS_VALIDATOR_COUNT
-
-echo Created genesis state in $TESTNET_DIR

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh

@@ -8,8 +8,6 @@ set -Eeuo pipefail
 
 source ./vars.env
 
-DEBUG_LEVEL=info
-
 BUILDER_PROPOSALS=
 
 # Get options

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env

@@ -25,7 +25,9 @@ BOOTNODE_PORT=${BOOTNODE_PORT:-4242}
 
 # Hard fork configuration
 ALTAIR_FORK_EPOCH=${ALTAIR_FORK_EPOCH:-0}
-MERGE_FORK_EPOCH=${MERGE_FORK_EPOCH:-0}
+BELLATRIX_FORK_EPOCH=${BELLATRIX_FORK_EPOCH:-0}
+CAPELLA_FORK_EPOCH=${CAPELLA_FORK_EPOCH:-0}
+DENEB_FORK_EPOCH=${DENEB_FORK_EPOCH:-0}
 
 # Spec version (mainnet or minimal)
 SPEC_PRESET=${SPEC_PRESET:-mainnet}
@@ -51,3 +53,6 @@ ETH1_TTD=${ETH1_TTD:-`cat $ETH1_GENESIS_JSON | jq -r '.config.terminalTotalDiffi
 ETH1_DEPOSIT_CONTRACT_ADDRESS=${ETH1_DEPOSIT_CONTRACT_ADDRESS:-`cat $ETH1_CONFIG_YAML | grep 'deposit_contract_address' | awk '{ print $2 }' | sed 's/"//g'`}
 ETH1_DEPOSIT_CONTRACT_BLOCK=${ETH1_DEPOSIT_CONTRACT_BLOCK:-0x0}
 SUGGESTED_FEE_RECIPIENT=`cat ../build/el/accounts.csv | head -1 | cut -d',' -f2`
+
+# --debug-level
+DEBUG_LEVEL=${LIGHTHOUSE_DEBUG_LEVEL:-debug}

stack_orchestrator/data/container-build/cerc-fixturenet-lotus/Dockerfile

@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.19.12-bullseye AS lotus-builder
+FROM golang:1.21.7-bullseye AS lotus-builder
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
@@ -29,7 +29,6 @@ RUN set -eux; \
     rustup --version; \
     cargo --version; \
     rustc --version;
-
 COPY ./ /opt/filecoin
 WORKDIR /opt/filecoin
 
@@ -38,13 +37,13 @@ WORKDIR /opt/filecoin
 ### make configurable filecoin-ffi build
 ARG FFI_BUILD_FROM_SOURCE=0
 ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
-
+RUN ls /opt/filecoin
 RUN make clean deps
 
 ARG RUSTFLAGS=""
 ARG GOFLAGS=""
 
-#RUN make buildall
+#RUN make lotus
 RUN make 2k
 
 #####################################
@@ -75,7 +74,7 @@ COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
 #COPY scripts/docker-lotus-entrypoint.sh /
 #COPY myscripts/setup-node.sh /docker-entrypoint-scripts.d/setup-node.sh
 
-ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest
+ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://forest-archive.chainsafe.dev/latest/mainnet/
 ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT}
 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_PATH /var/lib/lotus

stack_orchestrator/data/container-build/cerc-fixturenet-lotus/build.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
-# Build cerc/lotus
+# Build cerc/fixturenet-lotus
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 # Use a release version tag to match the modified Dockerfile replaced in next step
-git -C ${CERC_REPO_BASE_DIR}/lotus checkout v1.23.3
+git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
 
 # Replace repo's Dockerfile with modified one
 cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile
 
-docker build -t cerc/lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus
+docker build -t cerc/fixturenet-lotus:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus

stack_orchestrator/data/container-build/cerc-lighthouse/Dockerfile

@@ -1,9 +1,10 @@
-ARG TAG_SUFFIX="-modern"
-FROM sigp/lighthouse:v4.3.0${TAG_SUFFIX}
+FROM sigp/lighthouse:v5.1.2
 
-RUN apt-get update; apt-get install bash netcat curl less jq wget -y;
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get -y install bash netcat curl less jq wget \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /root/
+WORKDIR /root
 ADD start-lighthouse.sh .
 
 ENTRYPOINT [ "./start-lighthouse.sh" ]

stack_orchestrator/data/container-build/cerc-lighthouse/build.sh

@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/lighthouse:local ${build_command_args} --build-arg TAG_SUFFIX="" ${SCRIPT_DIR}
+docker build -t cerc/lighthouse:local ${build_command_args} ${SCRIPT_DIR}

stack_orchestrator/data/container-build/cerc-mainnet-lotus/Dockerfile

@@ -0,0 +1,124 @@
+#####################################
+FROM golang:1.21.7-bullseye AS lotus-builder
+MAINTAINER Lotus Development Team
+
+RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
+
+ENV XDG_CACHE_HOME="/tmp"
+
+### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.63.0
+
+RUN set -eux; \
+    dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in \
+        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
+        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
+        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+COPY ./ /opt/filecoin
+WORKDIR /opt/filecoin
+
+ARG FFI_BUILD_FROM_SOURCE=0
+ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
+
+#RUN make clean deps
+
+ARG RUSTFLAGS=""
+ARG GOFLAGS=""
+
+RUN make clean deps && \
+    make lotus lotus-shed lotus-stats && \
+    install -C ./lotus /usr/local/bin/lotus && \
+    install -C ./lotus-shed /usr/local/bin/lotus-shed && \
+    install -C ./lotus-stats /usr/local/bin/lotus-stats
+
+#####################################
+FROM ubuntu:20.04 AS lotus-base
+MAINTAINER Lotus Development Team
+
+# Base resources
+COPY --from=lotus-builder /etc/ssl/certs                           /etc/ssl/certs
+COPY --from=lotus-builder /lib/*/libdl.so.2         /lib/
+COPY --from=lotus-builder /lib/*/librt.so.1         /lib/
+COPY --from=lotus-builder /lib/*/libgcc_s.so.1      /lib/
+COPY --from=lotus-builder /lib/*/libutil.so.1       /lib/
+COPY --from=lotus-builder /usr/lib/*/libltdl.so.7   /lib/
+COPY --from=lotus-builder /usr/lib/*/libnuma.so.1   /lib/
+COPY --from=lotus-builder /usr/lib/*/libhwloc.so.*  /lib/
+COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/
+
+RUN useradd -r -u 532 -U fc \
+ && mkdir -p /etc/OpenCL/vendors \
+ && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+#####################################
+FROM lotus-base AS lotus
+MAINTAINER Lotus Development Team
+
+COPY --from=lotus-builder \
+    /usr/local/bin/lotus \
+    /usr/local/bin/lotus-shed \
+    /usr/local/bin/
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_PATH /var/lib/lotus
+ENV DOCKER_LOTUS_IMPORT_WALLET ""
+
+RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
+
+VOLUME /var/lib/lotus
+VOLUME /var/tmp/filecoin-proof-parameters
+
+USER fc
+
+EXPOSE 1234
+
+
+CMD ["-help"]
+
+#####################################
+FROM lotus-base AS lotus-all-in-one
+
+# Install netcat for healthcheck
+RUN apt-get update && apt-get install -y netcat && apt-get install -y iproute2
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_PATH /var/lib/lotus
+
+ARG DOCKER_LOTUS_IMPORT_SNAPSHOT=https://forest-archive.chainsafe.dev/latest/mainnet/
+ENV DOCKER_LOTUS_IMPORT_SNAPSHOT=${DOCKER_LOTUS_IMPORT_SNAPSHOT}
+
+COPY --from=lotus-builder /opt/filecoin/lotus          /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-shed     /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
+COPY scripts/docker-lotus-entrypoint.sh /docker-lotus-entrypoint.sh
+RUN chmod +x /docker-lotus-entrypoint.sh
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+
+EXPOSE 1234
+EXPOSE 1235
+

stack_orchestrator/data/container-build/cerc-mainnet-lotus/build.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build cerc/mainnet-lotus
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+git stash
+
+# Use a release version tag to match the modified Dockerfile replaced in next step
+git -C ${CERC_REPO_BASE_DIR}/lotus checkout master
+
+# Replace repo's Dockerfile with modified one
+cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile
+
+docker build -t cerc/lotus-mainnet:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus

stack_orchestrator/data/container-build/cerc-test-container/build.sh

@@ -2,4 +2,4 @@
 # Build cerc/test-container
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
+docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

stack_orchestrator/data/container-image-list.txt

@@ -41,7 +41,8 @@ cerc/pocket
 cerc/watcher-azimuth
 cerc/ipld-eth-state-snapshot
 cerc/watcher-gelato
-cerc/lotus
+cerc/mainnet-lotus
+cerc/fixturenet-lotus
 cerc/go-opera
 cerc/lasso
 cerc/reth

stack_orchestrator/data/pod-list.txt

@@ -27,6 +27,7 @@ fixturenet-pocket
 watcher-azimuth
 watcher-gelato
 fixturenet-lotus
+mainnet-lotus
 mainnet-go-opera
 lasso
 reth

stack_orchestrator/data/stacks/fixturenet-lotus/stack.yml

@@ -2,8 +2,8 @@ version: "1.0"
 name: fixturenet-lotus
 description: "A lotus fixturenet"
 repos:
-  - github.com/filecoin-project/lotus
+  - git.vdb.to/cerc-io/lotus
 containers:
-  - cerc/lotus
+  - cerc/fixturenet-lotus
 pods:
   - fixturenet-lotus

stack_orchestrator/data/stacks/fixturenet-sushiswap-subgraph/stack.yml

@@ -13,7 +13,7 @@ repos:
   - github.com/sushiswap/subgraphs
 containers:
   # fixturenet-lotus image
-  - cerc/lotus
+  - cerc/lotus-fixturenet
   # graph-node image
   - cerc/graph-node
   # sushiswap contract deployment images

stack_orchestrator/data/stacks/graph-node/README.md

@@ -76,6 +76,19 @@ export ETH_RPC_PORT=
 # The etherum network(s) graph-node will connect to
 # Set this to a space-separated list of the networks where each entry has the form NAME:URL
 export ETH_NETWORKS=
+
+# Optional:
+
+# Timeout for ETH RPC requests in seconds (default: 180s)
+export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=
+
+# Number of times to retry ETH RPC requests (default: 10)
+export GRAPH_ETHEREUM_REQUEST_RETRIES=
+
+# Maximum number of blocks to scan for triggers in each request (default: 2000)
+export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
+
+# Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```
 
 Example env file:
@@ -85,6 +98,10 @@ export ETH_RPC_HOST=filecoin.chainup.net
 export ETH_RPC_PORT=443
 
 export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
+
+export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=360
+export GRAPH_ETHEREUM_REQUEST_RETRIES=5
+export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```
 
 Set the environment variables:

stack_orchestrator/data/stacks/mainnet-lotus/README.md

@@ -0,0 +1,26 @@
+# Lotus stack
+
+## Clone required repositories
+```
+$ laconic-so --stack mainnet-lotus setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack mainnet-lotus build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack mainnet-lotus deploy init --map-ports-to-host any-same --output lotus-spec.yml
+```
+Update the URLs in the generated spec file to use your systems' actual IP addresses.
+```
+$ laconic-so --stack mainnet-lotus deploy create --deployment-dir lotus-deployment --spec-file lotus-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir lotus-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir lotus-deployment logs
+```

stack_orchestrator/data/stacks/mainnet-lotus/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: mainnet-lotus
+description: "A lotus stack"
+repos:
+  - git.vdb.to/cerc-io/lotus
+containers:
+  - cerc/mainnet-lotus
+pods:
+  - mainnet-lotus

stack_orchestrator/data/stacks/monitoring/README.md

@@ -4,6 +4,7 @@
 * Comes with the following built-in exporters / dashboards:
   * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
   * Watchers dashboard
+  * laconicd dashboard
   * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
   * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
   * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
@@ -99,6 +100,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
       - targets:
         - <HTTP_ENDPOINT_1>
         - <HTTP_ENDPOINT_2>
+        - <LACONICD_GQL_ENDPOINT>
   ```
 
 * Postgres (in-stack exporter):
@@ -116,6 +118,16 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
     ```
   * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)
 
+* laconicd: update the `laconicd` job with a laconicd node's REST endpoint host and port:
+
+  ```yml
+  ...
+  - job_name: laconicd
+  static_configs:
+    - targets: ['example-host:1317']
+  ...
+  ```
+
 Note: Use `host.docker.internal` as host to access ports on the host machine
 
 ### Grafana Config

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -46,6 +46,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     static_configs:
       - targets:
         - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
+        - <LACONICD_GQL_ENDPOINT>
+  ...
+  - job_name: laconicd
+  static_configs:
+    - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
   ...
   - job_name: azimuth
     scrape_interval: 10s
@@ -98,6 +103,16 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
         labels:
           instance: 'merkl_sushiswap'
           chain: 'filecoin'
+
+  - job_name: ajna
+    scrape_interval: 20s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+        labels:
+          instance: 'ajna'
+          chain: 'filecoin'
   ```
 
 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.

stack_orchestrator/data/stacks/sushiswap/stack.yml

@@ -11,7 +11,7 @@ repos:
   - github.com/vulcanize/uniswap-v3-info
 containers:
   ## fixturenet-lotus image
-  - cerc/lotus
+  - cerc/lotus-fixturenet
   ## sushiswap images
   - cerc/sushiswap-v3-core
   - cerc/sushiswap-v3-periphery

stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml

@@ -2,10 +2,10 @@ version: "1.0"
 name: webapp-deployer-backend
 description: "Deployer for webapps"
 repos:
-  - git.vdb.to/telackey/webapp-deployment-status-api
+  - git.vdb.to/cerc-io/webapp-deployment-status-api
 containers:
   - cerc/webapp-deployer-backend
 pods:
   - name: webapp-deployer-backend
-    repository: git.vdb.to/telackey/webapp-deployment-status-api
+    repository: git.vdb.to/cerc-io/webapp-deployment-status-api
     path: ./

stack_orchestrator/deploy/deploy.py

@@ -27,6 +27,7 @@ from pathlib import Path
 from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
+from stack_orchestrator.util import resolve_compose_file
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -324,7 +325,10 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
         pod_path = pod["path"]
         if include_exclude_check(pod_name, include, exclude):
             if pod_repository is None or pod_repository == "internal":
-                compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
+                if deployment:
+                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
+                else:
+                    compose_file_name = resolve_compose_file(stack, pod_name)
             else:
                 if deployment:
                     compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
@@ -336,6 +340,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                     if pod_post_start_command is not None:
                         post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
                 else:
+                    # TODO: fix this code for external stack with scripts
                     pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
                     compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod.get("pre_start_command")

stack_orchestrator/deploy/deploy_util.py

@@ -16,7 +16,7 @@
 import os
 from typing import List, Any
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping
-from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
+from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_pod_list, resolve_compose_file
 from stack_orchestrator.opts import opts
 
 
@@ -27,7 +27,7 @@ def _container_image_from_service(stack: str, service: str):
     pods = get_pod_list(parsed_stack)
     yaml = get_yaml()
     for pod in pods:
-        pod_file_path = os.path.join(get_compose_file_dir(), f"docker-compose-{pod}.yml")
+        pod_file_path = resolve_compose_file(stack, pod)
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "services" in parsed_pod_file:
             services = parsed_pod_file["services"]

stack_orchestrator/deploy/deployment_create.py

@@ -26,7 +26,8 @@ from stack_orchestrator import constants
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
-                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file)
+                                     get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
+                                     resolve_config_dir)
 from stack_orchestrator.deploy.spec import Spec
 from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
@@ -43,7 +44,7 @@ def _get_ports(stack):
     pods = get_pod_list(parsed_stack)
     yaml = get_yaml()
     for pod in pods:
-        pod_file_path = get_pod_file_path(parsed_stack, pod)
+        pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "services" in parsed_pod_file:
             for svc_name, svc in parsed_pod_file["services"].items():
@@ -79,7 +80,7 @@ def _get_named_volumes(stack):
         return ret
 
     for pod in pods:
-        pod_file_path = get_pod_file_path(parsed_stack, pod)
+        pod_file_path = get_pod_file_path(stack, parsed_stack, pod)
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
@@ -480,10 +481,9 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     os.mkdir(destination_compose_dir)
     destination_pods_dir = deployment_dir_path.joinpath("pods")
     os.mkdir(destination_pods_dir)
-    data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
     yaml = get_yaml()
     for pod in pods:
-        pod_file_path = get_pod_file_path(parsed_stack, pod)
+        pod_file_path = get_pod_file_path(stack_name, parsed_stack, pod)
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod)
         destination_pod_dir = destination_pods_dir.joinpath(pod)
@@ -497,7 +497,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
         config_dirs = {pod}
         config_dirs = config_dirs.union(extra_config_dirs)
         for config_dir in config_dirs:
-            source_config_dir = data_dir.joinpath("config", config_dir)
+            source_config_dir = resolve_config_dir(stack_name, config_dir)
             if os.path.exists(source_config_dir):
                 destination_config_dir = deployment_dir_path.joinpath("config", config_dir)
                 # If the same config dir appears in multiple pods, it may already have been copied

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -39,11 +39,12 @@ def process_app_deployment_request(
     app_deployment_request,
     deployment_record_namespace,
     dns_record_namespace,
-    dns_suffix,
+    default_dns_suffix,
     deployment_parent_dir,
     kube_config,
     image_registry,
     force_rebuild,
+    fqdn_policy,
     logger
 ):
     logger.log("BEGIN - process_app_deployment_request")
@@ -56,14 +57,15 @@ def process_app_deployment_request(
     requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
     logger.log(f"Determined requested name: {requested_name}")
 
-    # HACK
     if "." in requested_name:
-        raise Exception("Only unqualified hostnames allowed at this time.")
-
-    fqdn = f"{requested_name}.{dns_suffix}"
+        if "allow" == fqdn_policy or "preexisting" == fqdn_policy:
+            fqdn = requested_name
+        else:
+            raise Exception(f"{requested_name} is invalid: only unqualified hostnames are allowed.")
+    else:
+        fqdn = f"{requested_name}.{default_dns_suffix}"
 
     # 3. check ownership of existing dnsrecord vs this request
-    # TODO: Support foreign DNS
     dns_crn = f"{dns_record_namespace}/{fqdn}"
     dns_record = laconic.get_record(dns_crn)
     if dns_record:
@@ -75,7 +77,9 @@ def process_app_deployment_request(
             logger.log(f"Matched DnsRecord ownership: {matched_owner}")
         else:
             raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
-                            (dns_record.id, app_deployment_request.id))
+                            (dns_crn, app_deployment_request.id))
+    elif "preexisting" == fqdn_policy:
+        raise Exception(f"No pre-existing DnsRecord {dns_crn} could be found for request {app_deployment_request.id}.")
 
     # 4. get build and runtime config from request
     env_filename = None
@@ -191,6 +195,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
 @click.option("--state-file", help="File to store state about previously seen requests.")
 @click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
 @click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world")
+@click.option("--fqdn-policy", help="How to handle requests with an FQDN: prohibit, allow, preexisting", default="prohibit")
 @click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
 @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
 @click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@@ -201,7 +206,7 @@ def dump_known_requests(filename, requests, status="SEEN"):
 @click.pass_context
 def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,  # noqa: C901
             request_id, discover, state_file, only_update_state,
-            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
+            dns_suffix, fqdn_policy, record_namespace_dns, record_namespace_deployments, dry_run,
             include_tags, exclude_tags, force_rebuild, log_dir):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -220,6 +225,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
             print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
             sys.exit(2)
 
+    if fqdn_policy not in ["prohibit", "allow", "preexisting"]:
+        print("--fqdn-policy must be one of 'prohibit', 'allow', or 'preexisting'", file=sys.stderr)
+        sys.exit(2)
+
     # Split CSV and clean up values.
     include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
     exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
@@ -247,7 +256,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
     requests_by_name = {}
     skipped_by_name = {}
     for r in requests:
-        # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
+        if r.id in previous_requests and previous_requests[r.id].get("status", "") != "RETRY":
+            print(f"Skipping request {r.id}, we've already seen it.")
+            continue
+
         app = laconic.get_record(r.attributes.application)
         if not app:
             print("Skipping request %s, cannot locate app." % r.id)
@@ -334,6 +346,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                     kube_config,
                     image_registry,
                     force_rebuild,
+                    fqdn_policy,
                     logger
                 )
                 status = "DEPLOYED"

stack_orchestrator/main.py

@@ -17,6 +17,7 @@ import click
 
 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.repos import setup_repositories
+from stack_orchestrator.repos import fetch_stack
 from stack_orchestrator.build import build_containers, fetch_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
@@ -50,6 +51,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
     ctx.obj = command_options
 
 
+cli.add_command(fetch_stack.command, "fetch-stack")
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
 cli.add_command(fetch_containers.command, "fetch-containers")

stack_orchestrator/repos/fetch_stack.py

@@ -0,0 +1,45 @@
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# env vars:
+# CERC_REPO_BASE_DIR defaults to ~/cerc
+
+
+import click
+import os
+
+from decouple import config
+from git import exc
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.repos.setup_repositories import process_repo
+from stack_orchestrator.util import error_exit
+
+
+@click.command()
+@click.argument('stack-locator')
+@click.option('--git-ssh', is_flag=True, default=False)
+@click.option('--check-only', is_flag=True, default=False)
+@click.option('--pull', is_flag=True, default=False)
+@click.pass_context
+def command(ctx, stack_locator, git_ssh, check_only, pull):
+    '''optionally resolve then git clone a repository containing one or more stack definitions'''
+    dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
+    if not opts.o.quiet:
+        print(f"Dev Root is: {dev_root_path}")
+    try:
+        process_repo(pull, check_only, git_ssh, dev_root_path, None, stack_locator)
+    except exc.GitCommandError as error:
+        error_exit(f"\n******* git command returned error exit status:\n{error}")

stack_orchestrator/repos/setup_repositories.py

@@ -26,6 +26,7 @@ import importlib.resources
 from pathlib import Path
 import yaml
 from stack_orchestrator.constants import stack_file_name
+from stack_orchestrator.opts import opts
 from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
 
 
@@ -87,8 +88,8 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
 
 
 # TODO: fix the messy arg list here
-def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
-    if verbose:
+def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully_qualified_repo):
+    if opts.o.verbose:
         print(f"Processing repo: {fully_qualified_repo}")
     repo_host, repo_path, repo_branch = host_and_path_for_repo(fully_qualified_repo)
     git_ssh_prefix = f"git@{repo_host}:"
@@ -100,7 +101,7 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
     (current_repo_branch_or_tag, is_branch) = _get_repo_current_branch_or_tag(
         full_filesystem_repo_path
         ) if is_present else (None, None)
-    if not quiet:
+    if not opts.o.quiet:
         present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
             else 'Needs to be fetched'
         print(f"Checking: {full_filesystem_repo_path}: {present_text}")
@@ -111,25 +112,25 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
             sys.exit(1)
         else:
             if pull:
-                if verbose:
+                if opts.o.verbose:
                     print(f"Running git pull for {full_filesystem_repo_path}")
                 if not check_only:
                     if is_branch:
                         git_repo = git.Repo(full_filesystem_repo_path)
                         origin = git_repo.remotes.origin
-                        origin.pull(progress=None if quiet else GitProgress())
+                        origin.pull(progress=None if opts.o.quiet else GitProgress())
                     else:
                         print("skipping pull because this repo checked out a tag")
                 else:
                     print("(git pull skipped)")
     if not is_present:
         # Clone
-        if verbose:
+        if opts.o.verbose:
             print(f'Running git clone for {full_github_repo_path} into {full_filesystem_repo_path}')
-        if not dry_run:
+        if not opts.o.dry_run:
             git.Repo.clone_from(full_github_repo_path,
                                 full_filesystem_repo_path,
-                                progress=None if quiet else GitProgress())
+                                progress=None if opts.o.quiet else GitProgress())
         else:
             print("(git clone skipped)")
     # Checkout the requested branch, if one was specified
@@ -150,13 +151,13 @@ def process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_pa
             current_repo_branch_or_tag and (
                 current_repo_branch_or_tag != branch_to_checkout)
                 ):
-            if not quiet:
+            if not opts.o.quiet:
                 print(f"switching to branch {branch_to_checkout} in repo {repo_path}")
             git_repo = git.Repo(full_filesystem_repo_path)
             # git checkout works for both branches and tags
             git_repo.git.checkout(branch_to_checkout)
         else:
-            if verbose:
+            if opts.o.verbose:
                 print(f"repo {repo_path} is already on branch/tag {branch_to_checkout}")
 
 
@@ -182,36 +183,18 @@ def parse_branches(branches_string):
 @click.option('--check-only', is_flag=True, default=False)
 @click.option('--pull', is_flag=True, default=False)
 @click.option("--branches", help="override branches for repositories")
-@click.option('--branches-file', help="checkout branches specified in this file")
 @click.pass_context
-def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches_file):
+def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
     '''git clone the set of repositories required to build the complete system from source'''
 
-    quiet = ctx.obj.quiet
-    verbose = ctx.obj.verbose
-    dry_run = ctx.obj.dry_run
-    stack = ctx.obj.stack
+    quiet = opts.o.quiet
+    verbose = opts.o.verbose
+    stack = opts.o.stack
 
     branches_array = []
 
-    # TODO: branches file needs to be re-worked in the context of stacks
-    if branches_file:
-        if branches:
-            print("Error: can't specify both --branches and --branches-file")
-            sys.exit(1)
-        else:
-            if verbose:
-                print(f"loading branches from: {branches_file}")
-            with open(branches_file) as branches_file_open:
-                branches_array = branches_file_open.read().splitlines()
-
-    print(f"branches: {branches}")
     if branches:
-        if branches_file:
-            print("Error: can't specify both --branches and --branches-file")
-            sys.exit(1)
-        else:
-            branches_array = parse_branches(branches)
+        branches_array = parse_branches(branches)
 
     if branches_array and verbose:
         print(f"Branches are: {branches_array}")
@@ -271,7 +254,6 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
 
     for repo in repos:
         try:
-            process_repo(verbose, quiet, dry_run, pull, check_only, git_ssh, dev_root_path, branches_array, repo)
+            process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, repo)
         except git.exc.GitCommandError as error:
-            print(f"\n******* git command returned error exit status:\n{error}")
-            sys.exit(1)
+            error_exit(f"\n******* git command returned error exit status:\n{error}")

stack_orchestrator/util.py

@@ -94,10 +94,38 @@ def get_plugin_code_paths(stack) -> List[Path]:
     return list(result)
 
 
-def get_pod_file_path(parsed_stack, pod_name: str):
+# # Find a config directory, looking first in any external stack
+# and if not found there, internally
+def resolve_config_dir(stack, config_dir_name: str):
+    if stack_is_external(stack):
+        # First try looking in the external stack for the compose file
+        config_base = Path(stack).parent.parent.joinpath("config")
+        proposed_dir = config_base.joinpath(config_dir_name)
+        if proposed_dir.exists():
+            return proposed_dir
+        # If we don't find it fall through to the internal case
+    config_base = get_internal_config_dir()
+    return config_base.joinpath(config_dir_name)
+
+
+# Find a compose file, looking first in any external stack
+# and if not found there, internally
+def resolve_compose_file(stack, pod_name: str):
+    if stack_is_external(stack):
+        # First try looking in the external stack for the compose file
+        compose_base = Path(stack).parent.parent.joinpath("compose")
+        proposed_file = compose_base.joinpath(f"docker-compose-{pod_name}.yml")
+        if proposed_file.exists():
+            return proposed_file
+        # If we don't find it fall through to the internal case
+    compose_base = get_internal_compose_file_dir()
+    return compose_base.joinpath(f"docker-compose-{pod_name}.yml")
+
+
+def get_pod_file_path(stack, parsed_stack, pod_name: str):
     pods = parsed_stack["pods"]
     if type(pods[0]) is str:
-        result = os.path.join(get_compose_file_dir(), f"docker-compose-{pod_name}.yml")
+        result = resolve_compose_file(stack, pod_name)
     else:
         for pod in pods:
             if pod["name"] == pod_name:
@@ -131,7 +159,7 @@ def pod_has_scripts(parsed_stack, pod_name: str):
     return result
 
 
-def get_compose_file_dir():
+def get_internal_compose_file_dir():
     # TODO: refactor to use common code with deploy command
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     data_dir = Path(__file__).absolute().parent.joinpath("data")
@@ -139,7 +167,7 @@ def get_compose_file_dir():
     return source_compose_dir
 
 
-def get_config_file_dir():
+def get_internal_config_dir():
     # TODO: refactor to use common code with deploy command
     data_dir = Path(__file__).absolute().parent.joinpath("data")
     source_config_dir = data_dir.joinpath("config")

tests/external-stack/run-test.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+# Dump environment variables for debugging
+echo "Environment variables:"
+env
+
+if [ "$1" == "from-path" ]; then
+    TEST_TARGET_SO="laconic-so"
+else
+    TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+fi
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
+# Test basic stack-orchestrator deploy
+echo "Running stack-orchestrator external stack deploy test"
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+# Clone the external test stack
+$TEST_TARGET_SO fetch-stack git.vdb.to/cerc-io/test-external-stack
+stack_name="$CERC_REPO_BASE_DIR/test-external-stack/stack-orchestrator/stacks/test-external-stack"
+TEST_TARGET_SO_STACK="$TEST_TARGET_SO --stack ${stack_name}"
+# Test bringing the test container up and down
+# with and without volume removal
+$TEST_TARGET_SO_STACK setup-repositories
+$TEST_TARGET_SO_STACK build-containers
+# Test deploy command execution
+$TEST_TARGET_SO_STACK deploy setup $CERC_REPO_BASE_DIR
+# Check that we now have the expected output directory
+container_output_dir=$CERC_REPO_BASE_DIR/container-output-dir
+if [ ! -d "$container_output_dir" ]; then
+    echo "deploy setup test: output directory not present"
+    echo "deploy setup test: FAILED"
+    exit 1
+fi
+if [ ! -f "$container_output_dir/output-file" ]; then
+    echo "deploy setup test: output file not present"
+    echo "deploy setup test: FAILED"
+    exit 1
+fi
+output_file_content=$(<$container_output_dir/output-file)
+if [ ! "$output_file_content" == "output-data"  ]; then
+    echo "deploy setup test: output file contents not correct"
+    echo "deploy setup test: FAILED"
+    exit 1
+fi
+# Check that we now have the expected output file
+$TEST_TARGET_SO_STACK deploy up
+# Test deploy port command
+deploy_port_output=$( $TEST_TARGET_SO_STACK deploy port test 80 )
+if [[ "$deploy_port_output" =~ ^0.0.0.0:[1-9][0-9]* ]]; then
+    echo "Deploy port test: passed"
+else
+    echo "Deploy port test: FAILED"
+    exit 1
+fi
+$TEST_TARGET_SO_STACK deploy down
+# The next time we bring the container up the volume will be old (from the previous run above)
+$TEST_TARGET_SO_STACK deploy up
+log_output_1=$( $TEST_TARGET_SO_STACK deploy logs )
+if [[ "$log_output_1" == *"filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    exit 1
+fi
+$TEST_TARGET_SO_STACK deploy down --delete-volumes
+# Now when we bring the container up the volume will be new again
+$TEST_TARGET_SO_STACK deploy up
+log_output_2=$( $TEST_TARGET_SO_STACK deploy logs )
+if [[ "$log_output_2" == *"filesystem is fresh"* ]]; then
+    echo "Delete volumes test: passed"
+else
+    echo "Delete volumes test: FAILED"
+    exit 1
+fi
+$TEST_TARGET_SO_STACK deploy down --delete-volumes
+# Basic test of creating a deployment
+test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
+test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
+$TEST_TARGET_SO_STACK deploy init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED,CERC_TEST_PARAM_3=FAST
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+$TEST_TARGET_SO_STACK deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+# Check the file writted by the create command in the stack now exists
+if [ ! -f "$test_deployment_dir/create-file" ]; then
+    echo "deploy create test: create output file not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+# And has the right content
+create_file_content=$(<$test_deployment_dir/create-file)
+if [ ! "$create_file_content" == "create-command-output-data"  ]; then
+    echo "deploy create test: create output file contents not correct"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+
+# Add a config file to be picked up by the ConfigMap before starting.
+echo "dbfc7a4d-44a7-416d-b5f3-29842cc47650" > $test_deployment_dir/data/test-config/test_config
+
+echo "deploy create output file test: passed"
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+# Check logs command works
+log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_3" == *"filesystem is fresh"* ]]; then
+    echo "deployment logs test: passed"
+else
+    echo "deployment logs test: FAILED"
+    exit 1
+fi
+# Check the config variable CERC_TEST_PARAM_1 was passed correctly
+if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
+    echo "deployment config test: passed"
+else
+    echo "deployment config test: FAILED"
+    exit 1
+fi
+# Check the config variable CERC_TEST_PARAM_2 was passed correctly from the compose file
+if [[ "$log_output_3" == *"Test-param-2: CERC_TEST_PARAM_2_VALUE"* ]]; then
+    echo "deployment compose config test: passed"
+else
+    echo "deployment compose config test: FAILED"
+    exit 1
+fi
+# Check the config variable CERC_TEST_PARAM_3 was passed correctly
+if [[ "$log_output_3" == *"Test-param-3: FAST"* ]]; then
+    echo "deployment config test: passed"
+else
+    echo "deployment config test: FAILED"
+    exit 1
+fi
+
+# Check that the ConfigMap is mounted and contains the expected content.
+log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_4" == *"/config/test_config:"* ]] && [[ "$log_output_4" == *"dbfc7a4d-44a7-416d-b5f3-29842cc47650"* ]]; then
+    echo "deployment ConfigMap test: passed"
+else
+    echo "deployment ConfigMap test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+log_output_5=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_5" == *"filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    delete_cluster_exit
+fi
+
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"