update fixturenet-plugeth test

add fixturenet-laconicd test

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -0,0 +1,55 @@
+name: Fixturenet-Laconicd-Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+
+jobs:
+  test:
+    name: "Run an Laconicd fixturenet test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Update'
+        run: apt-get update
+      - name: 'Setup jq'
+        run: apt-get install jq -y
+      - name: 'Check jq'
+        run: |
+          which jq
+          jq --version
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+

.github/workflows/fixturenet-laconicd.yml

@@ -0,0 +1,30 @@
+name: Fixturenet-Laconicd Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-laconicd-test'
+
+jobs:
+  test:
+    name: "Run fixturenet-laconicd test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-laconicd tests"
+        run: ./tests/fixturenet-laconicd/run-test.sh

.github/workflows/fixturenet-plugeth.yml

@@ -0,0 +1,30 @@
+name: Fixturenet-Plugeth Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.github/workflows/triggers/fixturenet-plugeth-test'
+
+jobs:
+  test:
+    name: "Run fixturenet-plugeth test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-plugeth tests"
+        run: ./tests/fixturenet-plugeth/run-test.sh

.github/workflows/triggers/fixturenet-laconicd-test

@@ -0,0 +1,3 @@
+Change this file to trigger running the fixturenet-laconicd-test CI job
+
+trigger

.github/workflows/triggers/fixturenet-plugeth-test

@@ -0,0 +1,3 @@
+Change this file to trigger running the fixturenet-plugeth-test CI job
+
+trigger

app/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml

@@ -0,0 +1,29 @@
+version: "3.2"
+
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      ipld-eth-db:
+        condition: service_healthy
+    image: cerc/ipld-eth-db:local
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+
+  ipld-eth-db:
+    image: timescale/timescaledb:2.8.1-pg14
+    restart: always
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+    volumes:
+      - mainnet_eth_ipld_eth_db:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    ports:
+      - "5432"
+volumes:
+  mainnet_eth_ipld_eth_db:

app/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml

@@ -0,0 +1,24 @@
+version: "3.7"
+services:
+  ipld-eth-server:
+    restart: always
+    depends_on:
+      ipld-eth-db:
+        condition: service_healthy
+    image: cerc/ipld-eth-server:local
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+      - ../config/mainnet-eth-ipld-eth-server/srv.env
+    volumes:
+      - ../config/mainnet-eth-ipld-eth-server/config.toml:/app/config.toml:ro
+    ports:
+      - "8081"
+      - "8082"
+      - "8090"
+      - "40001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8081"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s

app/data/compose/docker-compose-mainnet-eth-keycloak.yml

@@ -6,7 +6,7 @@ services:
     env_file:
       - ../config/mainnet-eth-keycloak/keycloak.env
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      test: ["CMD", "nc", "-v", "localhost", "35432"]
       interval: 30s
       timeout: 10s
       retries: 10
@@ -14,7 +14,7 @@ services:
     volumes:
       - mainnet_eth_keycloak_db:/var/lib/postgresql/data
     ports:
-      - 5432
+      - 35432
 
   keycloak:
     image: cerc/keycloak:local

app/data/compose/docker-compose-mainnet-eth-plugeth.yml

@@ -0,0 +1,72 @@
+
+services:
+
+  mainnet-eth-geth-1:
+    restart: always
+    hostname: mainnet-eth-geth-1
+    cap_add:
+      - SYS_PTRACE
+    image: cerc/plugeth-with-plugins:local
+    entrypoint: /bin/sh
+    command: -c "/opt/run-geth.sh"
+    env_file:
+      - ../config/mainnet-eth-ipld-eth-db/db.env
+      - ../config/mainnet-eth-plugeth/geth.env
+    volumes:
+      - mainnet_eth_plugeth_geth_1_data:/data
+      - mainnet_eth_plugeth_config_data:/etc/mainnet-eth
+      - ../config/mainnet-eth-plugeth/scripts/run-geth.sh:/opt/run-geth.sh
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8545"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    ports:
+      # http api
+      - "8545"
+      # ws api
+      - "8546"
+      # ws el
+      - "8551"
+      # p2p
+      - "30303"
+      - "30303/udp"
+      # debugging
+      - "40000"
+      # metrics
+      - "6060"
+
+  mainnet-eth-lighthouse-1:
+    restart: always
+    hostname: mainnet-eth-lighthouse-1
+    healthcheck:
+      test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 30s
+    environment:
+      LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
+    env_file:
+      - ../config/mainnet-eth-plugeth/lighthouse.env
+    image: cerc/lighthouse:local
+    entrypoint: /bin/sh
+    command: -c "/opt/run-lighthouse.sh"
+    volumes:
+      - mainnet_eth_plugeth_lighthouse_1_data:/data
+      - mainnet_eth_plugeth_config_data:/etc/mainnet-eth
+      - ../config/mainnet-eth-plugeth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh
+    ports:
+      # api
+      - "5052"
+      # metrics
+      - "5054"
+      # p2p
+      - "9000"
+      - "9000/udp"
+
+volumes:
+  mainnet_eth_plugeth_config_data:
+  mainnet_eth_plugeth_geth_1_data:
+  mainnet_eth_plugeth_lighthouse_1_data:

app/data/config/mainnet-eth-ipld-eth-db/db.env

@@ -0,0 +1,15 @@
+DATABASE_HOSTNAME="ipld-eth-db"
+DATABASE_NAME="cerc"
+DATABASE_PASSWORD="CHANGEME"
+DATABASE_PORT=5432
+DATABASE_USER="vdbm"
+
+POSTGRES_DB="${DATABASE_NAME}"
+POSTGRES_PASSWORD="${DATABASE_PASSWORD}"
+POSTGRES_USER="${DATABASE_USER}"
+
+CERC_STATEDIFF_DB_HOST="${DATABASE_HOSTNAME}"
+CERC_STATEDIFF_DB_NAME="${DATABASE_NAME}"
+CERC_STATEDIFF_DB_PASSWORD="${DATABASE_PASSWORD}"
+CERC_STATEDIFF_DB_PORT=${DATABASE_PORT}
+CERC_STATEDIFF_DB_USER="${DATABASE_USER}"

app/data/config/mainnet-eth-ipld-eth-server/config.toml

@@ -0,0 +1,33 @@
+[database]
+    name     = "" # $DATABASE_NAME
+    hostname = "" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD
+
+[log]
+    level = "info" # $LOG_LEVEL
+
+[server]
+    ipc = false
+    ipcPath = "" # $SERVER_IPC_PATH
+    ws = false
+    wsPath = "0.0.0.0:8080" # $SERVER_WS_PATH
+    http = true
+    httpPath = "0.0.0.0:8081" # $SERVER_HTTP_PATH
+    graphql = false # $SERVER_GRAPHQL
+    graphqlPath = "0.0.0.0:8082" # $SERVER_GRAPHQL_PATH
+
+[ethereum]
+    chainConfig = "" # ETH_CHAIN_CONFIG
+    chainID = "1" # $ETH_CHAIN_ID
+    rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "mainnet-eth-geth-1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
+    stateDiffTimeout = "4m" # $ETH_STATEDIFF_TIMEOUT
+    forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS
+    proxyOnError = true # $ETH_PROXY_ON_ERROR
+    nodeID = "" # $ETH_NODE_ID
+    clientName = "" # $ETH_CLIENT_NAME
+    genesisBlock = "" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID

app/data/config/mainnet-eth-ipld-eth-server/srv.env

@@ -0,0 +1,27 @@
+CERC_REMOTE_DEBUG="false"
+
+LOG_LEVEL="debug"
+
+ETH_CHAIN_ID=1
+ETH_CLIENT_NAME="Geth"
+ETH_FORWARD_ETH_CALLS="false"
+ETH_FORWARD_GET_STORAGE_AT="false"
+ETH_GENESIS_BLOCK="0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+ETH_HTTP_PATH="mainnet-eth-geth-1:8545"
+ETH_NETWORK_ID=1
+ETH_NODE_ID=1112
+ETH_PROXY_ON_ERROR="true"
+ETH_RPC_GAS_CAP=1000000000000
+ETH_SUPPORTS_STATEDIFF="true"
+ETH_STATEDIFF_TIMEOUT=4m
+
+SERVER_HTTP_PATH=0.0.0.0:8081
+SERVER_GRAPHQL="false"
+SERVER_GRAPHQLPATH=0.0.0.0:8082
+
+METRICS="true"
+PROM_HTTP="true"
+PROM_HTTP_ADDR="0.0.0.0"
+PROM_HTTP_PORT="8090"
+
+VDB_COMMAND="serve"

app/data/config/mainnet-eth-keycloak/keycloak.env

@@ -1,8 +1,11 @@
 POSTGRES_DB=keycloak
 POSTGRES_USER=keycloak
 POSTGRES_PASSWORD=keycloak
+# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
+PGPORT=35432
 KC_DB=postgres
 KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_PORT=${PGPORT}
 KC_DB_URL_DATABASE=${POSTGRES_DB}
 KC_DB_USERNAME=${POSTGRES_USER}
 KC_DB_PASSWORD=${POSTGRES_PASSWORD}

app/data/config/mainnet-eth-keycloak/nginx.example

@@ -15,42 +15,49 @@ server {
 }
 
 upstream geth-pool {
-  keepalive 100;
-  hash $user_id consistent;
-  server server-a:8545;
-  server server-b:8545;
-  server server-c:8545;
+  server server-a:8545 max_fails=10 fail_timeout=2s;
+  server server-c:8545 max_fails=10 fail_timeout=2s backup;
+  server server-b:8545 max_fails=10 fail_timeout=2s backup;
+  keepalive 200;
 }
 
-# self-reg happens on one server for clarity
 upstream reg-ui-pool {
-  keepalive 100;
+  keepalive 2;
   server server-a:8085;
 }
 
 upstream reg-api-pool {
-  keepalive 100;
+  keepalive 2;
   server server-a:8086;
 }
 
-# auth uses server-a if available
+# auth uses the reg server when available
 upstream auth-pool {
-  keepalive 100;
+  keepalive 10;
   server server-a:8080;
   server server-b:8080 backup;
   server server-c:8080 backup;
 }
 
-log_format upstreamlog '[$time_local] $remote_addr $user_id - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time';
-proxy_cache_path                /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m;
+
+log_format upstreamlog '[$time_local] $msec $remote_addr $user_id - $server_name($host) to $upstream_addr: $request $status upstream_response_time $upstream_response_time request_time $request_time';
+proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m;
+
 server {
     listen 443 ssl http2;
     server_name my.example.com;
+    keepalive_requests 500000;
+    keepalive_timeout 90s;
+    http2_max_requests 5000000;
+    http2_max_concurrent_streams 1024;
+    http2_idle_timeout 3m;
+    http2_recv_timeout 30s;
     access_log /var/log/nginx/my.example.com-access.log upstreamlog;
     error_log /var/log/nginx/my.example.com-error.log;
 
     ssl_certificate /etc/nginx/ssl/my.example.com/cert.pem;
     ssl_certificate_key /etc/nginx/ssl/my.example.com/key.pem;
+    ssl_session_cache shared:SSL:10m;
 
     error_page 500 502 503 504 /50x.html;
     location = /50x.html {
@@ -60,7 +67,6 @@ server {
     #rewrite ^/?$ /newuser/;
     rewrite ^/?$ https://www.example.com/;
 
-
     # geth-pool ETH API
     location ~ ^/v1/eth/?([^/]*)$ {
         set $apiKey $1;
@@ -71,8 +77,8 @@ server {
         auth_request_set $user_id $sent_http_x_user_id;
         rewrite /.*$ / break;
 
-        client_max_body_size 3m;
-        client_body_buffer_size 3m;
+        client_max_body_size       3m;
+        client_body_buffer_size    3m;
         proxy_buffer_size          32k;
         proxy_buffers              16 32k;
         proxy_busy_buffers_size    96k;
@@ -80,8 +86,10 @@ server {
         proxy_pass  http://geth-pool;
         proxy_set_header X-Original-Remote-Addr $remote_addr;
         proxy_set_header X-User-Id $user_id;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
     }
-    
+
     # keycloak
     location = /auth {
         internal;
@@ -95,6 +103,8 @@ server {
         proxy_set_header X-Original-URI $request_uri;
         proxy_set_header X-Original-Remote-Addr $remote_addr;
         proxy_set_header X-Original-Host $host;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
     }
 
     location /newuser/ {

app/data/config/mainnet-eth-plugeth/geth.env

@@ -0,0 +1,75 @@
+# Enable remote debugging using dlv
+CERC_REMOTE_DEBUG=false
+
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Simple toggle to choose either a 'full' node or an 'archive' node
+# (controls the values of --syncmode --gcmode --snapshot)
+CERC_GETH_MODE_QUICK_SET=archive
+
+# Path to plugeth plugins.
+CERC_PLUGINS_DIR="/usr/local/lib/plugeth"
+
+# Will turn on statediffing automatically if CERC_STATEDIFF_DB_HOST exists (see ../mainnet-eth-ipld-eth-db/db.env).
+CERC_RUN_STATEDIFF="detect"
+
+# The minimum necessary verion of the DB to enable statediffing.
+CERC_STATEDIFF_DB_GOOSE_MIN_VER=18
+
+# Whether all statediff-related DB statements should be logged (useful for debugging).
+CERC_STATEDIFF_DB_LOG_STATEMENTS=false
+
+# The number of concurrent workers to process state diff objects
+CERC_STATEDIFF_WORKERS=16
+
+# Each statediffing node should have a unique node ID.
+CERC_STATEDIFF_DB_NODE_ID=1111
+
+# Optional custom node name.
+# GETH_NODE_NAME=""
+
+# Specify any other geth CLI options.
+GETH_OPTS=""
+
+# --cache
+GETH_CACHE=1024
+
+# --cache.database
+GETH_CACHE_DB=50
+
+# --cache.gc
+GETH_CACHE_GC=25
+
+# --cache.trie
+GETH_CACHE_TRIE=15
+
+# --datadir
+GETH_DATADIR="/data"
+
+# --http.api
+GETH_HTTP_API="eth,web3,net"
+
+# --authrpc.jwtsecret
+GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret"
+
+# --maxpeers
+GETH_MAX_PEERS=100
+
+# --rpc.evmtimeout
+GETH_RPC_EVMTIMEOUT=0
+
+# --rpc.gascap
+GETH_RPC_GASCAP=0
+
+# --txlookuplimit
+GETH_TXLOOKUPLIMIT=0
+
+# --verbosity
+GETH_VERBOSITY=3
+
+# --log.vmodule
+GETH_VMODULE="rpc/*=4"
+
+# --ws.api
+GETH_WS_API="eth,web3,net"

app/data/config/mainnet-eth-plugeth/lighthouse.env

@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000

app/data/config/mainnet-eth-plugeth/scripts/run-geth.sh

@@ -0,0 +1,121 @@
+#!/bin/sh
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+START_CMD="geth"
+if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then
+    START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
+fi
+
+# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
+cleanup() {
+    echo "Signal received, cleaning up..."
+
+    # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process)
+    pkill -P ${geth_pid}
+    sleep 2
+    kill $(jobs -p)
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' SIGINT SIGTERM
+
+MODE_FLAGS=""
+if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then
+  MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}"
+else
+  MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-full} --snapshot=${GETH_SNAPSHOT:-true}"
+fi
+
+if [[ "${CERC_RUN_STATEDIFF}" == "detect" ]] && [[ -n "$CERC_STATEDIFF_DB_HOST" ]]; then
+  dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short)
+  dig_status_code=$?
+  if [[ $dig_status_code = 0 && -n $dig_result ]]; then
+    echo "Statediff DB at $CERC_STATEDIFF_DB_HOST"
+    CERC_RUN_STATEDIFF="true"
+  else
+    echo "No statediff DB available."
+    CERC_RUN_STATEDIFF="false"
+  fi
+fi
+
+STATEDIFF_OPTS=""
+if [[ "${CERC_RUN_STATEDIFF}" == "true" ]]; then
+  ready=0
+  echo "Waiting for statediff DB..."
+  while [ $ready -eq 0 ]; do
+    sleep 1
+    export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD"
+    result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \
+      -p "$CERC_STATEDIFF_DB_PORT" \
+      -U "$CERC_STATEDIFF_DB_USER" \
+      -d "$CERC_STATEDIFF_DB_NAME" \
+      -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }')
+    if [ -n "$result" ]; then
+      echo "DB ready..."
+      if [[ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]]; then
+        ready=1
+      else
+        echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)"
+      fi
+    fi
+  done
+
+  STATEDIFF_OPTS="--statediff \
+  --statediff.db.host=$CERC_STATEDIFF_DB_HOST \
+  --statediff.db.name=$CERC_STATEDIFF_DB_NAME \
+  --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \
+  --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \
+  --statediff.db.port=$CERC_STATEDIFF_DB_PORT \
+  --statediff.db.user=$CERC_STATEDIFF_DB_USER \
+  --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \
+  --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \
+  --statediff.waitforsync=${CERC_STATEDIFF_WAIT_FO_SYNC:-true} \
+  --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \
+  --statediff.writing=${CERC_STATEDIFF_WRITING:-true}"
+
+  if [[ -d "${CERC_PLUGINS_DIR}" ]]; then
+    # With plugeth, we separate the statediff options by prefixing with ' -- '
+    STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}"
+  fi
+fi
+
+$START_CMD \
+  $MODE_FLAGS \
+  --datadir="${GETH_DATADIR}"\
+  --identity="${GETH_NODE_NAME}" \
+  --maxpeers=${GETH_MAX_PEERS} \
+  --cache=${GETH_CACHE} \
+  --cache.gc=${GETH_CACHE_GC} \
+  --cache.database=${GETH_CACHE_DB} \
+  --cache.trie=${GETH_CACHE_TRIE} \
+  --authrpc.addr='0.0.0.0' \
+  --authrpc.vhosts='*'  \
+  --authrpc.jwtsecret="${GETH_JWTSECRET}" \
+  --http \
+  --http.addr='0.0.0.0' \
+  --http.api="${GETH_HTTP_API}" \
+  --http.vhosts='*' \
+  --metrics \
+  --metrics.addr='0.0.0.0' \
+  --ws \
+  --ws.addr='0.0.0.0' \
+  --ws.api="${GETH_WS_API}" \
+  --rpc.gascap=${GETH_RPC_GASCAP} \
+  --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \
+  --txlookuplimit=${GETH_TXLOOKUPLIMIT} \
+  --verbosity=${GETH_VERBOSITY} \
+  --log.vmodule="${GETH_VMODULE}" \
+  ${STATEDIFF_OPTS} \
+  ${GETH_OPTS} &
+
+geth_pid=$!
+wait $geth_pid
+
+if [[ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]]; then
+  while [[ 1 -eq 1 ]]; do
+    sleep 60
+  done
+fi

app/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
+
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
+  --disable-deposit-contract-sync \
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS

app/data/config/mainnet-eth/geth.env

@@ -25,7 +25,7 @@ GETH_CACHE_GC=25
 
 # --cache.trie
 GETH_CACHE_TRIE=15
-j
+
 # --datadir
 GETH_DATADIR="/data"
 

app/data/container-build/cerc-act-runner-task-executor/build.sh

@@ -2,4 +2,4 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/gitea/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}

app/data/container-build/cerc-plugeth-with-plugins/Dockerfile

@@ -0,0 +1,22 @@
+# Using the same golang image as used to build plugeth: https://git.vdb.to/cerc-io/plugeth/src/branch/statediff/Dockerfile
+FROM golang:1.20-alpine3.18 as delve
+
+# Add delve so that we can do remote debugging.
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+FROM cerc/plugeth-statediff:local as statediff
+FROM cerc/plugeth:local as plugeth
+
+FROM alpine:3.18
+
+# Install tools often used in scripting, like bash, wget, and jq.
+RUN apk add --no-cache ca-certificates bash wget curl python3 bind-tools postgresql-client jq
+
+COPY --from=delve /go/bin/dlv /usr/local/bin/
+COPY --from=plugeth /usr/local/bin/geth /usr/local/bin/
+
+# Place all plugeth plugins in /usr/local/lib/plugeth
+COPY --from=statediff  /usr/local/lib/statediff.so /usr/local/lib/plugeth/
+
+EXPOSE 8545 8546 8551 6060 30303 30303/udp 40000
+ENTRYPOINT ["geth"]

app/data/container-build/cerc-plugeth-with-plugins/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/cerc-plugeth-with-plugins
+set -x
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/plugeth-with-plugins:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR

app/data/stacks/act-runner/README.md

@@ -0,0 +1,13 @@
+# act-runner stack
+
+## Example
+
+```
+$ laconic-so --stack act-runner deploy init --output act-runner-1.yml --config CERC_GITEA_RUNNER_REGISTRATION_TOKEN=FOO
+$ laconic-so --stack act-runner deploy create --spec-file act-runner-1.yml --deployment-dir ~/opt/deployments/act-runner-1
+$ laconic-so deployment --dir ~/opt/deployments/act-runner-1 up
+
+$ laconic-so --stack act-runner deploy init --output act-runner-2.yml --config CERC_GITEA_RUNNER_REGISTRATION_TOKEN=BAR
+$ laconic-so --stack act-runner deploy create --spec-file act-runner-2.yml --deployment-dir ~/opt/deployments/act-runner-2
+$ laconic-so deployment --dir ~/opt/deployments/act-runner-2 up
+```

app/data/stacks/act-runner/stack.yml

@@ -0,0 +1,15 @@
+version: "1.1"
+name: act-runner
+description: "Local act-runner"
+repos:
+  - git.vdb.to/cerc-io/hosting
+  - gitea.com/gitea/act_runner
+containers:
+  - cerc/act-runner
+  - cerc/act-runner-task-executor
+pods:
+  - name: act-runner
+    repository: cerc-io/hosting
+    path: act-runner
+    pre_start_command: "pre_start.sh"
+    post_start_command: "post_start.sh"

app/data/stacks/chain-chunker/stack.yml

@@ -6,8 +6,10 @@ repos:
   - git.vdb.to/cerc-io/eth-statediff-service@v5
   - git.vdb.to/cerc-io/ipld-eth-db@v5
   - git.vdb.to/cerc-io/ipld-eth-server@v5
+  - git.vdb.to/cerc-io/plugeth@statediff
 containers:
   - cerc/ipld-eth-state-snapshot
   - cerc/eth-statediff-service
   - cerc/ipld-eth-db
   - cerc/ipld-eth-server
+  - cerc/plugeth

app/data/stacks/mainnet-eth-plugeth/README.md

@@ -0,0 +1,141 @@
+# mainnet-eth-plugeth
+
+Deploys a "head-tracking" mainnet Ethereum stack comprising a [plugeth](https://git.vdb.to/cerc-io/plugeth) execution layer node and a [lighthouse](https://github.com/sigp/lighthouse) consensus layer node, with [plugeth-statediff](https://git.vdb.to/cerc-io/plugeth-statediff) for statediffing, [ipld-eth-db](https://git.vdb.to/cerc-io/ipld-eth-db) for storage, and [ipld-eth-server](https://git.vdb.to/cerc-io/ipld-eth-server) for indexed ETH IPLD objects.
+
+## Clone required repositories
+
+```
+$ laconic-so --stack mainnet-eth-plugeth setup-repositories
+```
+
+## Build containers
+
+```
+$ laconic-so --stack mainnet-eth-plugeth build-containers
+```
+
+## Create a deployment
+
+```
+$ laconic-so --stack mainnet-eth-plugeth deploy init --map-ports-to-host any-same --output mainnet-eth-plugeth-spec.yml
+$ laconic-so --stack mainnet-eth-plugeth deploy create --spec-file mainnet-eth-plugeth-spec.yml --deployment-dir mainnet-eth-plugeth-deployment
+```
+## Start the stack
+```
+$ laconic-so deployment --dir mainnet-eth-plugeth-deployment start
+```
+Display stack status:
+```
+$ laconic-so deployment --dir mainnet-eth-plugeth-deployment ps
+Running containers:
+id: f39608eca04d72d6b0f1f3acefc5ebb52908da06e221d20c7138f7e3dff5e423, name: laconic-ef641b4d13eb61ed561b19be67063241-foundry-1, ports:
+id: 4052b1eddd886ae0d6b41f9ff22e68a70f267b2bfde10f4b7b79b5bd1eeddcac, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-geth-1-1, ports: 30303/tcp, 30303/udp, 0.0.0.0:49184->40000/tcp, 0.0.0.0:49185->6060/tcp, 0.0.0.0:49186->8545/tcp, 8546/tcp
+id: ac331232e597944b621b3b8942ace5dafb14524302cab338ff946c7f6e5a1d52, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1, ports: 0.0.0.0:49187->8001/tcp
+```
+See stack logs:
+```
+$ laconic-so deployment --dir mainnet-eth-plugeth-deployment logs
+time="2023-07-25T09:46:29-06:00" level=warning msg="The \"CERC_SCRIPT_DEBUG\" variable is not set. Defaulting to a blank string."
+laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1  | Jul 25 15:45:13.362 INFO Logging to file                         path: "/var/lighthouse-data-dir/beacon/logs/beacon.log"
+laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1  | Jul 25 15:45:13.365 INFO Lighthouse started                      version: Lighthouse/v4.1.0-693886b
+laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1  | Jul 25 15:45:13.365 INFO Configured for network                  name: mainnet
+laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1  | Jul 25 15:45:13.366 INFO Data directory initialised              datadir: /var/lighthouse-data-dir
+laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1  | Jul 25 15:45:13.366 INFO Deposit contract                        address: 0x00000000219ab540356cbb839cbe05303d7705fa, deploy_block: 11184524
+laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1  | Jul 25 15:45:13.424 INFO Starting checkpoint sync                remote_url: https://beaconstate.ethstaker.cc/, service: beacon
+```
+## Monitoring stack sync progress
+Both go-ethereum and lighthouse will engage in an initial chain sync phase that will last up to several hours depending on hardware performance and network capacity.
+Syncing can be monitored by looking for these log messages:
+```
+Jul 24 12:34:17.001 INFO Downloading historical blocks           est_time: 5 days 11 hrs, speed: 14.67 slots/sec, distance: 6932481 slots (137 weeks 3 days), service: slot_notifier
+INFO [07-24|12:14:52.493] Syncing beacon headers                   downloaded=145,920 left=17,617,968 eta=1h23m32.815s
+INFO [07-24|12:33:15.238] Syncing: chain download in progress      synced=1.86% chain=148.94MiB headers=368,640@95.03MiB bodies=330,081@40.56MiB receipts=330,081@13.35MiB eta=37m54.505s
+INFO [07-24|12:35:13.028] Syncing: state download in progress      synced=1.32% state=4.64GiB   accounts=2,850,314@677.57MiB slots=18,663,070@3.87GiB  codes=26662@111.14MiB eta=3h18m0.699s
+```
+Once synced up these log messages will be observed:
+```
+INFO Synced                                  slot: 6952515, block: 0x5bcb…f6d9, epoch: 217266, finalized_epoch: 217264, finalized_root: 0x6342…2c5c, exec_hash: 0x8d8c…2443 (verified), peers: 31, service: slot_notifier
+INFO [07-25|03:04:48.941] Imported new potential chain segment     number=17,767,316 hash=84f6e7..bc2cb0 blocks=1  txs=137  mgas=16.123  elapsed=57.087ms     mgasps=282.434 dirty=461.46MiB
+INFO [07-25|03:04:49.042] Chain head was updated                   number=17,767,316 hash=84f6e7..bc2cb0 root=ca58b2..8258c1 elapsed=2.480111ms
+```
+## Clean up
+
+Stop the stack:
+```
+$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop
+```
+This leaves data volumes in place, allowing the stack to be subsequently re-started.
+To permanently *delete* the stack's data volumes run:
+```
+$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop --delete-data-volumes
+```
+After deleting the volumes, any subsequent re-start will begin chain sync from cold.
+
+## Ports
+It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be
+customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while
+port `40000` has been left with random assignment:
+```
+$ cat mainnet-eth-plugeth-spec.yml
+stack: mainnet-eth-plugeth
+ports:
+  mainnet-eth-plugeth-geth-1:
+   - '10.10.10.10:8545:8545'
+   - '40000'
+  mainnet-eth-plugeth-lighthouse-1:
+   - '10.10.10.10:5052:5052'
+volumes:
+  mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data
+  mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data
+  mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data
+```
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
+## Data volumes
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory:
+```
+$ cat mainnet-eth-plugeth-spec.yml
+stack: mainnet-eth-plugeth
+ports:
+  mainnet-eth-plugeth-geth-1:
+   - '10.10.10.10:8545:8545'
+   - '40000'
+  mainnet-eth-plugeth-lighthouse-1:
+   - '10.10.10.10:5052:5052'
+volumes:
+  mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data
+  mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data
+  mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data
+```
+A synced-up stack will consume around 900GB of data volume space:
+```
+$ sudo du -h mainnet-eth-plugeth-deployment/data/
+150M    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/freezer_db
+25G     mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/chain_db
+16K     mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/network
+368M    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/logs
+26G     mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon
+26G     mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data
+8.0K    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_config_data
+4.0K    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/keystore
+527G    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient/chain
+527G    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient
+859G    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata
+4.8M    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/nodes
+242M    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/ethash
+669M    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/triecache
+860G    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth
+860G    mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data
+885G    mainnet-eth-plugeth-deployment/data/
+```

app/data/stacks/mainnet-eth-plugeth/deploy/commands.py

@@ -0,0 +1,32 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from secrets import token_hex
+
+
+def init(ctx):
+    return None
+
+
+def setup(ctx):
+    return None
+
+
+def create(ctx, extra_args):
+    # Generate the JWT secret and save to its config file
+    secret = token_hex(32)
+    jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_plugeth_config_data", "jwtsecret")
+    with open(jwt_file_path, 'w+') as jwt_file:
+        jwt_file.write(secret)

app/data/stacks/mainnet-eth-plugeth/stack.yml

@@ -0,0 +1,29 @@
+version: "1.2"
+name: mainnet-eth
+description: "Ethereum Mainnet"
+repos:
+  - git.vdb.to/cerc-io/plugeth@statediff
+  - git.vdb.to/cerc-io/plugeth-statediff
+  - git.vdb.to/cerc-io/lighthouse
+  - git.vdb.to/cerc-io/ipld-eth-db@v5
+  - git.vdb.to/cerc-io/ipld-eth-server@v5
+  - git.vdb.to/cerc-io/keycloak-reg-api
+  - git.vdb.to/cerc-io/keycloak-reg-ui
+containers:
+  - cerc/plugeth-statediff
+  - cerc/plugeth
+  - cerc/plugeth-with-plugins
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/ipld-eth-db
+  - cerc/ipld-eth-server
+  - cerc/keycloak
+  - cerc/webapp-base
+  - cerc/keycloak-reg-api
+  - cerc/keycloak-reg-ui
+pods:
+  - mainnet-eth-plugeth
+  - mainnet-eth-ipld-eth-db
+  - mainnet-eth-ipld-eth-server
+  - mainnet-eth-keycloak
+  - mainnet-eth-metrics

app/data/stacks/package-registry/stack.yml

@@ -13,3 +13,8 @@ pods:
     path: gitea
     pre_start_command: "run-this-first.sh"
     post_start_command: "initialize-gitea.sh"
+  - name: act-runner
+    repository: cerc-io/hosting
+    path: act-runner
+    pre_start_command: "pre_start.sh"
+    post_start_command: "post_start.sh"

app/deploy/compose/deploy_docker.py

@@ -48,9 +48,9 @@ class DockerDeployer(Deployer):
         except DockerException as e:
             raise DeployerException(e)
 
-    def execute(self, service, command, envs):
+    def execute(self, service, command, tty, envs):
         try:
-            return self.docker.compose.execute(service=service, command=command, envs=envs)
+            return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs)
         except DockerException as e:
             raise DeployerException(e)
 

app/deploy/deploy.py

@@ -151,7 +151,7 @@ def exec_operation(ctx, extra_args):
         if global_context.verbose:
             print(f"Running compose exec {service_name} {command_to_exec}")
         try:
-            ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env)
+            ctx.obj.deployer.execute(service_name, command_to_exec, tty=True, envs=container_exec_env)
         except DeployerException:
             print("container command returned error exit status")
 
@@ -307,7 +307,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                 compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
             else:
                 if deployment:
-                    compose_file_name = os.path.join(compose_dir, "docker-compose.yml")
+                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod["pre_start_command"]
                     pod_post_start_command = pod["post_start_command"]
                     script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
@@ -317,7 +317,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                         post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
                 else:
                     pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
-                    compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
+                    compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod["pre_start_command"]
                     pod_post_start_command = pod["post_start_command"]
                     if pod_pre_start_command is not None:

app/deploy/deployer.py

@@ -35,7 +35,7 @@ class Deployer(ABC):
         pass
 
     @abstractmethod
-    def execute(self, service_name, command, envs):
+    def execute(self, service_name, command, tty, envs):
         pass
 
     @abstractmethod

app/deploy/deployment_create.py

@@ -22,7 +22,7 @@ import random
 from shutil import copy, copyfile, copytree
 import sys
 from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
-                      get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
+                      get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths)
 from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
 
 
@@ -106,9 +106,10 @@ def _fixup_pod_file(pod, spec, compose_dir):
                 pod["services"][container_name]["ports"] = container_ports
 
 
-def _commands_plugin_path(ctx: DeployCommandContext):
-    plugin_path = get_plugin_code_path(ctx.stack)
-    return plugin_path.joinpath("deploy", "commands.py")
+def _commands_plugin_paths(ctx: DeployCommandContext):
+    plugin_paths = get_plugin_code_paths(ctx.stack)
+    ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
+    return ret
 
 
 # See: https://stackoverflow.com/a/54625079/1701505
@@ -120,15 +121,23 @@ def call_stack_deploy_init(deploy_command_context):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deploy_command_context)
-    if python_file_path.exists():
-        spec = util.spec_from_file_location("commands", python_file_path)
-        imported_stack = util.module_from_spec(spec)
-        spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "init"):
-            return imported_stack.init(deploy_command_context)
-    else:
-        return None
+    python_file_paths = _commands_plugin_paths(deploy_command_context)
+
+    ret = None
+    init_done = False
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "init"):
+                if not init_done:
+                    ret = imported_stack.init(deploy_command_context)
+                    init_done = True
+                else:
+                    # TODO: remove this restriction
+                    print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.")
+    return ret
 
 
 # TODO: fold this with function above
@@ -136,16 +145,14 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deploy_command_context)
-    print(f"Path: {python_file_path}")
-    if python_file_path.exists():
-        spec = util.spec_from_file_location("commands", python_file_path)
-        imported_stack = util.module_from_spec(spec)
-        spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "setup"):
-            return imported_stack.setup(deploy_command_context, parameters, extra_args)
-    else:
-        return None
+    python_file_paths = _commands_plugin_paths(deploy_command_context)
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "setup"):
+                imported_stack.setup(deploy_command_context, parameters, extra_args)
 
 
 # TODO: fold this with function above
@@ -153,15 +160,14 @@ def call_stack_deploy_create(deployment_context, extra_args):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deployment_context.command_context)
-    if python_file_path.exists():
-        spec = util.spec_from_file_location("commands", python_file_path)
-        imported_stack = util.module_from_spec(spec)
-        spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "create"):
-            return imported_stack.create(deployment_context, extra_args)
-    else:
-        return None
+    python_file_paths = _commands_plugin_paths(deployment_context.command_context)
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "create"):
+                imported_stack.create(deployment_context, extra_args)
 
 
 # Inspect the pod yaml to find config files referenced in subdirectories
@@ -255,7 +261,7 @@ def init(ctx, config, output, map_ports_to_host):
     config_variables = _parse_config_variables(config)
     if config_variables:
         # Implement merge, since update() overwrites
-        orig_config = spec_file_content["config"]
+        orig_config = spec_file_content.get("config", {})
         new_config = config_variables["config"]
         merged_config = {**new_config, **orig_config}
         spec_file_content.update({"config": merged_config})
@@ -336,7 +342,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
         if global_options(ctx).debug:
             print(f"extra config dirs: {extra_config_dirs}")
         _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
-        with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file:
+        with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file:
             yaml.dump(parsed_pod_file, output_file)
         # Copy the config files for the pod, if any
         config_dirs = {pod}

app/deploy/k8s/cluster_info.py

@@ -0,0 +1,84 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from kubernetes import client
+from typing import Any, List, Set
+
+from app.opts import opts
+from app.util import get_yaml
+
+
+class ClusterInfo:
+    parsed_pod_yaml_map: Any = {}
+    image_set: Set[str] = set()
+    app_name: str = "test-app"
+    deployment_name: str = "test-deployment"
+
+    def __init__(self) -> None:
+        pass
+
+    def int_from_pod_files(self, pod_files: List[str]):
+        for pod_file in pod_files:
+            with open(pod_file, "r") as pod_file_descriptor:
+                parsed_pod_file = get_yaml().load(pod_file_descriptor)
+                self.parsed_pod_yaml_map[pod_file] = parsed_pod_file
+        if opts.o.debug:
+            print(f"parsed_pod_yaml_map: {self.parsed_pod_yaml_map}")
+        # Find the set of images in the pods
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
+            services = pod["services"]
+            for service_name in services:
+                service_info = services[service_name]
+                image = service_info["image"]
+                self.image_set.add(image)
+        if opts.o.debug:
+            print(f"image_set: {self.image_set}")
+
+    def get_deployment(self):
+        containers = []
+        for pod_name in self.parsed_pod_yaml_map:
+            pod = self.parsed_pod_yaml_map[pod_name]
+            services = pod["services"]
+            for service_name in services:
+                container_name = service_name
+                service_info = services[service_name]
+                image = service_info["image"]
+                container = client.V1Container(
+                    name=container_name,
+                    image=image,
+                    ports=[client.V1ContainerPort(container_port=80)],
+                    resources=client.V1ResourceRequirements(
+                        requests={"cpu": "100m", "memory": "200Mi"},
+                        limits={"cpu": "500m", "memory": "500Mi"},
+                    ),
+                )
+                containers.append(container)
+        template = client.V1PodTemplateSpec(
+            metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
+            spec=client.V1PodSpec(containers=containers),
+        )
+        spec = client.V1DeploymentSpec(
+            replicas=1, template=template, selector={
+                "matchLabels":
+                {"app": self.app_name}})
+
+        deployment = client.V1Deployment(
+            api_version="apps/v1",
+            kind="Deployment",
+            metadata=client.V1ObjectMeta(name=self.deployment_name),
+            spec=spec,
+        )
+        return deployment

app/deploy/k8s/deploy_k8s.py

@@ -14,33 +14,86 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from kubernetes import client, config
+
 from app.deploy.deployer import Deployer
+from app.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
+from app.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string
+from app.deploy.k8s.cluster_info import ClusterInfo
+from app.opts import opts
 
 
 class K8sDeployer(Deployer):
     name: str = "k8s"
+    core_api: client.CoreV1Api
+    apps_api: client.AppsV1Api
+    kind_cluster_name: str
+    cluster_info : ClusterInfo
 
     def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
-        config.load_kube_config()
-        self.client = client.CoreV1Api()
+        if (opts.o.debug):
+            print(f"Compose files: {compose_files}")
+            print(f"Project name: {compose_project_name}")
+            print(f"Env file: {compose_env_file}")
+        self.kind_cluster_name = compose_project_name
+        self.cluster_info = ClusterInfo()
+        self.cluster_info.int_from_pod_files(compose_files)
+
+    def connect_api(self):
+        config.load_kube_config(context=f"kind-{self.kind_cluster_name}")
+        self.core_api = client.CoreV1Api()
+        self.apps_api = client.AppsV1Api()
 
     def up(self, detach, services):
-        pass
+        # Create the kind cluster
+        create_cluster(self.kind_cluster_name)
+        self.connect_api()
+        # Ensure the referenced containers are copied into kind
+        load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
+        # Process compose files into a Deployment
+        deployment = self.cluster_info.get_deployment()
+        # Create the k8s objects
+        resp = self.apps_api.create_namespaced_deployment(
+            body=deployment, namespace="default"
+        )
+
+        if opts.o.debug:
+            print("Deployment created.\n")
+            print(f"{resp.metadata.namespace} {resp.metadata.name} \
+                  {resp.metadata.generation} {resp.spec.template.spec.containers[0].image}")
 
     def down(self, timeout, volumes):
-        pass
+        # Delete the k8s objects
+        # Destroy the kind cluster
+        destroy_cluster(self.kind_cluster_name)
 
     def ps(self):
-        pass
+        self.connect_api()
+        # Call whatever API we need to get the running container list
+        ret = self.core_api.list_pod_for_all_namespaces(watch=False)
+        if ret.items:
+            for i in ret.items:
+                print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
+        ret = self.core_api.list_node(pretty=True, watch=False)
+        return []
 
     def port(self, service, private_port):
+        # Since we handle the port mapping, need to figure out where this comes from
+        # Also look into whether it makes sense to get ports for k8s
         pass
 
-    def execute(self, service_name, command, envs):
+    def execute(self, service_name, command, tty, envs):
+        # Call the API to execute a command in a running container
         pass
 
     def logs(self, services, tail, follow, stream):
-        pass
+        self.connect_api()
+        pods = pods_in_deployment(self.core_api, "test-deployment")
+        if len(pods) > 1:
+            print("Warning: more than one pod in the deployment")
+        k8s_pod_name = pods[0]
+        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+        return log_stream_from_string(log_data)
 
     def run(self, image, command, user, volumes, entrypoint=None):
+        # We need to figure out how to do this -- check why we're being called first
         pass

app/deploy/k8s/helpers.py

@@ -0,0 +1,57 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from kubernetes import client
+import subprocess
+from typing import Set
+
+from app.opts import opts
+
+
+def _run_command(command: str):
+    if opts.o.debug:
+        print(f"Running: {command}")
+    result = subprocess.run(command, shell=True)
+    if opts.o.debug:
+        print(f"Result: {result}")
+
+
+def create_cluster(name: str):
+    _run_command(f"kind create cluster --name {name}")
+
+
+def destroy_cluster(name: str):
+    _run_command(f"kind delete cluster --name {name}")
+
+
+def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
+    for image in image_set:
+        _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+
+
+def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
+    pods = []
+    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
+    if opts.o.debug:
+        print(f"pod_response: {pod_response}")
+    for pod_info in pod_response.items:
+        pod_name = pod_info.metadata.name
+        pods.append(pod_name)
+    return pods
+
+
+def log_stream_from_string(s: str):
+    # Note response has to be UTF-8 encoded because the caller expects to decode it
+    yield ("ignore", s.encode())

app/opts.py

@@ -0,0 +1,20 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from app.command_types import CommandOptions
+
+
+class opts:
+    o: CommandOptions = None

app/util.py

@@ -79,16 +79,16 @@ def get_pod_list(parsed_stack):
     return result
 
 
-def get_plugin_code_path(stack):
+def get_plugin_code_paths(stack):
     parsed_stack = get_parsed_stack_config(stack)
     pods = parsed_stack["pods"]
-    # TODO: Hack
-    pod = pods[0]
-    if type(pod) is str:
-        result = get_stack_file_path(stack).parent
-    else:
-        pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
-        result = Path(os.path.join(pod_root_dir, "stack"))
+    result = []
+    for pod in pods:
+        if type(pod) is str:
+            result.append(get_stack_file_path(stack).parent)
+        else:
+            pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
+            result.append(Path(os.path.join(pod_root_dir, "stack")))
     return result
 
 

cli.py

@@ -22,6 +22,7 @@ from app.build import build_npms
 from app.deploy import deploy
 from app import version
 from app.deploy import deployment
+from app import opts
 from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@@ -39,7 +40,9 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 @click.pass_context
 def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error):
     """Laconic Stack Orchestrator"""
-    ctx.obj = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
+    command_options = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
+    opts.opts.o = command_options
+    ctx.obj = command_options
 
 
 cli.add_command(setup_repositories.command, "setup-repositories")

tests/fixturenet-eth-plugeth/run-test.sh

@@ -24,21 +24,58 @@ echo "Deploying the cluster"
 $TEST_TARGET_SO --stack $CERC_STACK_NAME deploy up
 # Verify that the fixturenet is up and running
 $TEST_TARGET_SO --stack $CERC_STACK_NAME deploy ps
-$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
-initial_block_number=$($TEST_TARGET_SO --stack fixturenet-plugeth-tx deploy exec foundry "cast block-number")
-# Check that the block number increases some time later
-sleep 12
-subsequent_block_number=$($TEST_TARGET_SO  --stack $CERC_STACK_NAME deploy exec foundry "cast block-number")
+# $TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
+
+timeout=900 # 15 minutes
+echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
+start_time=$(date +%s)
+elapsed_time=0
+initial_block_number=0
+while [ "$initial_block_number" -eq 0 ] && [ $elapsed_time -lt $timeout ]; do
+  sleep 10
+  echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
+  initial_block_number=$($TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec foundry "cast block-number")
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+done
+
+subsequent_block_number=$initial_block_number
+
+# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
+if [[ $initial_block_number -gt 0 ]]; then
+  timeout=300
+  echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
+  start_time=$(date +%s)
+  elapsed_time=0
+  # wait for 5 blocks or timeout
+  while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ] && [ $elapsed_time -lt $timeout ]; do
+    sleep 10
+    echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
+    subsequent_block_number=$($TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec foundry "cast block-number")
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+  done
+fi
+
+# will return 0 if either of the above loops timed out
 block_number_difference=$((subsequent_block_number - initial_block_number))
+
+echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
+echo "Initial block height: $initial_block_number"
+echo "Subsequent block height: $subsequent_block_number"
+
 # Block height difference should be between 1 and some small number
-if [[ $block_number_difference -gt 1 && $block_number_difference -lt 10 ]]; then
+if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
   echo "Test passed"
   test_result=0
 else
   echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}"
+  echo "Logs from stack:"
+  $TEST_TARGET_SO --stack $CERC_STACK_NAME deploy logs
   test_result=1
 fi
-$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy down
+
+$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy down --delete-volumes
 echo "Removing cloned repositories"
 rm -rf $CERC_REPO_BASE_DIR
 exit $test_result

tests/fixturenet-laconicd/run-test.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconicd fixturenet test"
+env
+cat /etc/hosts
+# Bit of a hack, test the most recent package
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Set a new unique repo dir
+export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
+echo "$(date +"%Y-%m-%d %T"): Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "$(date +"%Y-%m-%d %T"): Version reported is: ${reported_version_string}"
+
+echo "$(date +"%Y-%m-%d %T"): Cloning laconicd repositories into: $CERC_REPO_BASE_DIR"
+$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories
+
+echo "$(date +"%Y-%m-%d %T"): Building containers"
+$TEST_TARGET_SO --stack fixturenet-laconicd build-containers
+echo "$(date +"%Y-%m-%d %T"): Starting stack"
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
+echo "$(date +"%Y-%m-%d %T"): Stack started"
+# Verify that the fixturenet is up and running
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
+
+timeout=900 # 15 minutes
+echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
+start_time=$(date +%s)
+elapsed_time=0
+initial_block_number=null
+while [ "$initial_block_number" == "null" ]  && [ $elapsed_time -lt $timeout ]; do
+  sleep 10
+  echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
+  initial_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+done
+
+subsequent_block_number=$initial_block_number
+
+# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
+if [[ $initial_block_number != "null" ]]; then
+  timeout=300
+  echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
+  start_time=$(date +%s)
+  elapsed_time=0
+  # wait for 5 blocks or timeout
+  while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ]  && [ $elapsed_time -lt $timeout ]; do
+    sleep 10
+    echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
+    subsequent_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+  done
+fi
+
+# will return 0 if either of the above loops timed out
+block_number_difference=$((subsequent_block_number - initial_block_number))
+
+echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
+echo "Initial block height: $initial_block_number"
+echo "Subsequent block height: $subsequent_block_number"
+
+# Block height difference should be between 1 and some small number
+if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
+  echo "Test passed"
+  test_result=0
+else
+  echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}"
+  echo "Logs from stack:"
+  $TEST_TARGET_SO --stack fixturenet-laconicd deploy logs
+  test_result=1
+fi
+
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
+echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
+rm -rf $CERC_REPO_BASE_DIR
+echo "$(date +"%Y-%m-%d %T"): Test finished"
+exit $test_result