fix linter errors

* Support the case where webpack config is already present next.config.js

* Update scripts for experimental-compile/experimental-generate

.gitea/workflows/test-k8s-deploy.yml

@@ -0,0 +1,55 @@
+name: K8s Deploy Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+    paths-ignore:
+      - '.gitea/workflows/triggers/*'
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  test:
+    name: "Run deploy test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: Start dockerd # Also needed until we can incorporate into the executor
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: "Install Go"
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: "Install Kind"
+        run: go install sigs.k8s.io/kind@v0.20.0
+      - name: "Debug Kind"
+        run: kind create cluster --retain && docker logs kind-control-plane

.github/workflows/test-webapp.yml

@@ -0,0 +1,29 @@
+name: Webapp Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run webapp test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run webapp tests"
+        run: ./tests/webapp-test/run-webapp-test.sh

requirements.txt

@@ -1,4 +1,5 @@
 python-decouple>=3.8
+python-dotenv==1.0.0
 GitPython>=3.1.32
 tqdm>=4.65.0
 python-on-whales>=0.64.0

stack_orchestrator/build/build_webapp.py

@@ -65,6 +65,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
 
 
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
+    container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
                                                                     base_container.replace("/", "-"),

stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml

@@ -6,8 +6,8 @@ services:
   # Deploys the L1 smart contracts (outputs to volume l1_deployment)
   fixturenet-optimism-contracts:
     restart: on-failure
-    hostname: fixturenet-optimism-contracts
     image: cerc/optimism-contracts:local
+    hostname: fixturenet-optimism-contracts
     env_file:
       - ../config/fixturenet-optimism/l1-params.env
     environment:
@@ -17,27 +17,49 @@ services:
       CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL}
       CERC_L1_ADDRESS: ${CERC_L1_ADDRESS}
       CERC_L1_PRIV_KEY: ${CERC_L1_PRIV_KEY}
-      CERC_L1_ADDRESS_2: ${CERC_L1_ADDRESS_2}
-      CERC_L1_PRIV_KEY_2: ${CERC_L1_PRIV_KEY_2}
-    # Waits for L1 endpoint to be up before running the script
-    command: |
-      "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh"
     volumes:
       - ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh
-      - ../config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts
-      - ../config/optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts
-      - ../config/optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts
-      - ../config/fixturenet-optimism/optimism-contracts/update-config.js:/app/packages/contracts-bedrock/update-config.js
-      - ../config/fixturenet-optimism/optimism-contracts/run.sh:/app/packages/contracts-bedrock/run.sh
+      - ../config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh:/app/packages/contracts-bedrock/deploy-contracts.sh
       - l2_accounts:/l2-accounts
-      - l1_deployment:/app/packages/contracts-bedrock
+      - l1_deployment:/l1-deployment
+      - l2_config:/l2-config
+    # Waits for L1 endpoint to be up before running the contract deploy script
+    command: |
+      "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./deploy-contracts.sh"
+
+  # Initializes and runs the L2 execution client  (outputs to volume l2_geth_data)
+  op-geth:
+    restart: always
+    image: cerc/optimism-l2geth:local
+    hostname: op-geth
+    depends_on:
+      op-node:
+        condition: service_started
+    volumes:
+      - ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh
+      - l2_config:/l2-config:ro
+      - l2_accounts:/l2-accounts:ro
+      - l2_geth_data:/datadir
+    entrypoint: "sh"
+    command: "/run-op-geth.sh"
+    ports:
+      - "8545"
+      - "8546"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost:8545"]
+      interval: 30s
+      timeout: 10s
+      retries: 100
+      start_period: 10s
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  # Generates the config files required for L2 (outputs to volume l2_config)
-  op-node-l2-config-gen:
-    restart: on-failure
+  # Runs the L2 consensus client (Sequencer node)
+  # Generates the L2 config files if not already present (outputs to volume l2_config)
+  op-node:
+    restart: always
     image: cerc/optimism-op-node:local
+    hostname: op-node
     depends_on:
       fixturenet-optimism-contracts:
         condition: service_completed_successfully
@@ -47,61 +69,19 @@ services:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_L1_RPC: ${CERC_L1_RPC}
     volumes:
-      - ../config/fixturenet-optimism/generate-l2-config.sh:/app/generate-l2-config.sh
-      - l1_deployment:/contracts-bedrock:ro
-      - l2_config:/app
-    command: ["sh", "/app/generate-l2-config.sh"]
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  # Initializes and runs the L2 execution client  (outputs to volume l2_geth_data)
-  op-geth:
-    restart: always
-    image: cerc/optimism-l2geth:local
-    depends_on:
-      op-node-l2-config-gen:
-        condition: service_started
-    volumes:
-      - ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh
-      - l2_config:/op-node:ro
+      - ../config/fixturenet-optimism/run-op-node.sh:/run-op-node.sh
+      - l1_deployment:/l1-deployment:ro
+      - l2_config:/l2-config
       - l2_accounts:/l2-accounts:ro
-      - l2_geth_data:/datadir
     entrypoint: "sh"
-    command: "/run-op-geth.sh"
+    command: "/run-op-node.sh"
     ports:
-      - "0.0.0.0:8545:8545"
-      - "0.0.0.0:8546:8546"
-    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost:8545"]
-      interval: 30s
-      timeout: 10s
-      retries: 10
-      start_period: 10s
-
-  # Runs the L2 consensus client (Sequencer node)
-  op-node:
-    restart: always
-    image: cerc/optimism-op-node:local
-    depends_on:
-      op-geth:
-        condition: service_healthy
-    env_file:
-      - ../config/fixturenet-optimism/l1-params.env
-    environment:
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_L1_RPC: ${CERC_L1_RPC}
-    volumes:
-      - ../config/fixturenet-optimism/run-op-node.sh:/app/run-op-node.sh
-      - l2_config:/op-node-data:ro
-      - l2_accounts:/l2-accounts:ro
-    command: ["sh", "/app/run-op-node.sh"]
-    ports:
-      - "0.0.0.0:8547:8547"
+      - "8547"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost:8547"]
       interval: 30s
       timeout: 10s
-      retries: 10
+      retries: 100
       start_period: 10s
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -110,6 +90,7 @@ services:
   op-batcher:
     restart: always
     image: cerc/optimism-op-batcher:local
+    hostname: op-batcher
     depends_on:
       op-node:
         condition: service_healthy
@@ -129,7 +110,7 @@ services:
     command: |
       "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-batcher.sh"
     ports:
-      - "127.0.0.1:8548:8548"
+      - "8548"
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -137,25 +118,29 @@ services:
   op-proposer:
     restart: always
     image: cerc/optimism-op-proposer:local
+    hostname: op-proposer
     depends_on:
       op-node:
         condition: service_healthy
+      op-geth:
+        condition: service_healthy
     env_file:
       - ../config/fixturenet-optimism/l1-params.env
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_L1_RPC: ${CERC_L1_RPC}
+      CERC_L1_CHAIN_ID: ${CERC_L1_CHAIN_ID}
     volumes:
       - ../config/network/wait-for-it.sh:/wait-for-it.sh
       - ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh
-      - l1_deployment:/contracts-bedrock:ro
+      - l1_deployment:/l1-deployment:ro
       - l2_accounts:/l2-accounts:ro
     entrypoint: ["sh", "-c"]
     # Waits for L1 endpoint to be up before running the proposer
     command: |
       "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-proposer.sh"
     ports:
-      - "127.0.0.1:8560:8560"
+      - "8560"
     extra_hosts:
       - "host.docker.internal:host-gateway"
 

stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh

@@ -1,37 +0,0 @@
-#!/bin/sh
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
-
-# Check existing config if it exists
-if [ -f /app/jwt.txt ] && [ -f /app/rollup.json ]; then
-  echo "Found existing L2 config, cross-checking with L1 deployment config"
-
-  SOURCE_L1_CONF=$(cat /contracts-bedrock/deploy-config/getting-started.json)
-  EXP_L1_BLOCKHASH=$(echo "$SOURCE_L1_CONF" | jq -r '.l1StartingBlockTag')
-  EXP_BATCHER=$(echo "$SOURCE_L1_CONF" | jq -r '.batchSenderAddress')
-
-  GEN_L2_CONF=$(cat /app/rollup.json)
-  GEN_L1_BLOCKHASH=$(echo "$GEN_L2_CONF" | jq -r '.genesis.l1.hash')
-  GEN_BATCHER=$(echo "$GEN_L2_CONF" | jq -r '.genesis.system_config.batcherAddr')
-
-  if [ "$EXP_L1_BLOCKHASH" = "$GEN_L1_BLOCKHASH" ] && [ "$EXP_BATCHER" = "$GEN_BATCHER" ]; then
-    echo "Config cross-checked, exiting"
-    exit 0
-  fi
-
-  echo "Existing L2 config doesn't match the L1 deployment config, please clear L2 config volume before starting"
-  exit 1
-fi
-
-op-node genesis l2 \
-  --deploy-config /contracts-bedrock/deploy-config/getting-started.json \
-  --deployment-dir /contracts-bedrock/deployments/getting-started/ \
-  --outfile.l2 /app/genesis.json \
-  --outfile.rollup /app/rollup.json \
-  --l1-rpc $CERC_L1_RPC
-
-openssl rand -hex 32 > /app/jwt.txt

stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh

@@ -0,0 +1,172 @@
+#!/bin/bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
+CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
+
+CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
+
+export DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
+# Optional create2 salt for deterministic deployment of contract implementations
+export IMPL_SALT=$(openssl rand -hex 32)
+
+echo "Using L1 RPC endpoint ${CERC_L1_RPC}"
+
+# Exit if a deployment already exists (on restarts)
+if [ -d "/l1-deployment/$DEPLOYMENT_CONTEXT" ]; then
+  echo "Deployment directory /l1-deployment/$DEPLOYMENT_CONTEXT, checking OptimismPortal deployment"
+
+  OPTIMISM_PORTAL_ADDRESS=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/OptimismPortal.json | jq -r .address)
+  contract_code=$(cast code $OPTIMISM_PORTAL_ADDRESS --rpc-url $CERC_L1_RPC)
+
+  if [ -z "${contract_code#0x}" ]; then
+    echo "Error: A deployment directory was found in the volume, but no contract code was found on-chain at the associated address. Please clear L1 deployment volume before restarting."
+    exit 1
+  else
+    echo "Deployment found, exiting (successfully)."
+    exit 0
+  fi
+fi
+
+wait_for_block() {
+  local block="$1"  # Block to wait for
+  local timeout="$2"  # Max time to wait in seconds
+
+  echo "Waiting for block $block."
+  i=0
+  loops=$(($timeout/10))
+  while [ -z "$block_result" ] && [[ "$i" -lt "$loops" ]]; do
+    sleep 10
+    echo "Checking..."
+    block_result=$(cast block $block --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true)
+    i=$(($i + 1))
+  done
+}
+
+# We need four accounts and their private keys for the deployment: Admin, Proposer, Batcher, and Sequencer
+# If $CERC_L1_ADDRESS and $CERC_L1_PRIV_KEY have been set, we'll assign it to Admin and generate/fund the remaining three accounts from it
+# If not, we'll assume the L1 is the stack's own fixturenet-eth and use the pre-funded accounts/keys from $CERC_L1_ACCOUNTS_CSV_URL
+if [ -n "$CERC_L1_ADDRESS" ] && [ -n "$CERC_L1_PRIV_KEY" ]; then
+  wallet1=$(cast wallet new)
+  wallet2=$(cast wallet new)
+  wallet3=$(cast wallet new)
+  # Admin
+  ADMIN=$CERC_L1_ADDRESS
+  ADMIN_KEY=$CERC_L1_PRIV_KEY
+  # Proposer
+  PROPOSER=$(echo "$wallet1" | awk '/Address:/{print $2}')
+  PROPOSER_KEY=$(echo "$wallet1" | awk '/Private key:/{print $3}')
+  # Batcher
+  BATCHER=$(echo "$wallet2" | awk '/Address:/{print $2}')
+  BATCHER_KEY=$(echo "$wallet2" | awk '/Private key:/{print $3}')
+  # Sequencer
+  SEQ=$(echo "$wallet3" | awk '/Address:/{print $2}')
+  SEQ_KEY=$(echo "$wallet3" | awk '/Private key:/{print $3}')
+  
+  echo "Funding accounts."
+  wait_for_block 1 300
+  cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 5ether $PROPOSER --private-key $ADMIN_KEY
+  cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 10ether $BATCHER --private-key $ADMIN_KEY
+  cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 2ether $SEQ --private-key $ADMIN_KEY
+else
+  curl -o accounts.csv $CERC_L1_ACCOUNTS_CSV_URL
+  # Admin
+  ADMIN=$(awk -F ',' 'NR == 1 {print $2}' accounts.csv)
+  ADMIN_KEY=$(awk -F ',' 'NR == 1 {print $3}' accounts.csv)
+  # Proposer
+  PROPOSER=$(awk -F ',' 'NR == 2 {print $2}' accounts.csv)
+  PROPOSER_KEY=$(awk -F ',' 'NR == 2 {print $3}' accounts.csv)
+  # Batcher
+  BATCHER=$(awk -F ',' 'NR == 3 {print $2}' accounts.csv)
+  BATCHER_KEY=$(awk -F ',' 'NR == 3 {print $3}' accounts.csv)
+  # Sequencer
+  SEQ=$(awk -F ',' 'NR == 4 {print $2}' accounts.csv)
+  SEQ_KEY=$(awk -F ',' 'NR == 4 {print $3}' accounts.csv)
+fi
+
+echo "Using accounts:"
+echo -e "Admin: $ADMIN\nProposer: $PROPOSER\nBatcher: $BATCHER\nSequencer: $SEQ"
+
+# These accounts will be needed by other containers, so write them to a shared volume
+echo "Writing accounts/private keys to volume l2_accounts."
+accounts_json=$(jq -n \
+  --arg Admin "$ADMIN" --arg AdminKey "$ADMIN_KEY" \
+  --arg Proposer "$PROPOSER" --arg ProposerKey "$PROPOSER_KEY" \
+  --arg Batcher "$BATCHER" --arg BatcherKey "$BATCHER_KEY" \
+  --arg Seq "$SEQ" --arg SeqKey "$SEQ_KEY" \
+  '{Admin: $Admin, AdminKey: $AdminKey, Proposer: $Proposer, ProposerKey: $ProposerKey, Batcher: $Batcher, BatcherKey: $BatcherKey, Seq: $Seq, SeqKey: $SeqKey}')
+echo "$accounts_json" > "/l2-accounts/accounts.json"
+
+# Get a finalized L1 block to set as the starting point for the L2 deployment
+# If the chain is a freshly created fixturenet-eth, a finalized block won't be available for many minutes; rather than wait, we can use block 1
+echo "Checking L1 for finalized block..."
+finalized=$(cast block finalized --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true)
+
+if [ -n "$finalized" ]; then
+  # finalized block was found
+  start_block=$finalized
+else
+  # assume fresh chain and use block 1 instead
+  echo "No finalized block. Using block 1 instead."
+  # wait for 20 or so blocks to be safe
+  wait_for_block 24 300
+  start_block=$(cast block 1 --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true)
+fi
+
+if [ -z "$start_block" ]; then
+  echo "Unable to query chain for starting block. Exiting..."
+  exit 1
+fi
+
+BLOCKHASH=$(echo $start_block | awk -F ' ' '{print $2}')
+HEIGHT=$(echo $start_block | awk -F ' ' '{print $4}')
+TIMESTAMP=$(echo $start_block | awk -F ' ' '{print $6}')
+
+echo "Using block as deployment point:"
+echo "Height: $HEIGHT"
+echo "Hash: $BLOCKHASH"
+echo "Timestamp: $TIMESTAMP"
+
+# Fill out the deployment template (./deploy-config/getting-started.json) with our values:
+echo "Writing deployment config."
+deploy_config_file="deploy-config/$DEPLOYMENT_CONTEXT.json"
+cp deploy-config/getting-started.json $deploy_config_file
+sed -i "s/\"l1ChainID\": .*/\"l1ChainID\": $DEPLOYMENT_CONTEXT,/g" $deploy_config_file
+sed -i "s/ADMIN/$ADMIN/g" $deploy_config_file
+sed -i "s/PROPOSER/$PROPOSER/g" $deploy_config_file
+sed -i "s/BATCHER/$BATCHER/g" $deploy_config_file
+sed -i "s/SEQUENCER/$SEQ/g" $deploy_config_file
+sed -i "s/BLOCKHASH/$BLOCKHASH/g" $deploy_config_file
+sed -i "s/TIMESTAMP/$TIMESTAMP/g" $deploy_config_file
+
+mkdir -p deployments/$DEPLOYMENT_CONTEXT
+
+# Deployment requires the create2 deterministic proxy contract be published on L1 at address 0x4e59b44847b379578588920ca78fbf26c0b4956c
+# See: https://github.com/Arachnid/deterministic-deployment-proxy
+echo "Deploying create2 proxy contract..."
+echo "Funding deployment signer address"
+deployment_signer="0x3fab184622dc19b6109349b94811493bf2a45362"
+cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 0.5ether $deployment_signer --private-key $ADMIN_KEY
+echo "Deploying contract..."
+raw_bytes="0xf8a58085174876e800830186a08080b853604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf31ba02222222222222222222222222222222222222222222222222222222222222222a02222222222222222222222222222222222222222222222222222222222222222"
+
+cast publish --rpc-url $CERC_L1_RPC $raw_bytes
+
+# Create the L2 deployment
+echo "Deploying L1 Optimism contracts..."
+forge script scripts/Deploy.s.sol:Deploy --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC
+forge script scripts/Deploy.s.sol:Deploy --sig 'sync()' --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC
+
+echo "*************************************"
+echo "Done deploying contracts."
+
+# Copy files needed by other containers to the appropriate shared volumes
+echo "Copying deployment artifacts volume l1_deployment and deploy-config to volume l2_config"
+cp -a /app/packages/contracts-bedrock/deployments/$DEPLOYMENT_CONTEXT /l1-deployment
+cp /app/packages/contracts-bedrock/deploy-config/$DEPLOYMENT_CONTEXT.json /l2-config
+openssl rand -hex 32 > /l2-config/l2-jwt.txt
+
+echo "Deployment successful. Exiting"

stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh

@@ -1,131 +0,0 @@
-#!/bin/bash
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
-CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
-
-CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
-
-echo "Using L1 RPC endpoint ${CERC_L1_RPC}"
-
-IMPORT_1="import './verify-contract-deployment'"
-IMPORT_2="import './rekey-json'"
-IMPORT_3="import './send-balance'"
-
-# Append mounted tasks to tasks/index.ts file if not present
-if ! grep -Fxq "$IMPORT_1" tasks/index.ts; then
-  echo "$IMPORT_1" >> tasks/index.ts
-  echo "$IMPORT_2" >> tasks/index.ts
-  echo "$IMPORT_3" >> tasks/index.ts
-fi
-
-# Update the chainId in the hardhat config
-sed -i "/getting-started/ {n; s/.*chainId.*/      chainId: $CERC_L1_CHAIN_ID,/}" hardhat.config.ts
-
-# Exit if a deployment already exists (on restarts)
-# Note: fixturenet-eth-geth currently starts fresh on a restart
-if [ -d "deployments/getting-started" ]; then
-  echo "Deployment directory deployments/getting-started found, checking SystemDictator deployment"
-
-  # Read JSON file into variable
-  SYSTEM_DICTATOR_DETAILS=$(cat deployments/getting-started/SystemDictator.json)
-
-  # Parse JSON into variables
-  SYSTEM_DICTATOR_ADDRESS=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.address')
-  SYSTEM_DICTATOR_TXHASH=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.transactionHash')
-
-  if yarn hardhat verify-contract-deployment --contract "${SYSTEM_DICTATOR_ADDRESS}" --transaction-hash "${SYSTEM_DICTATOR_TXHASH}"; then
-    echo "Deployment verfication successful, exiting"
-    exit 0
-  else
-    echo "Deployment verfication failed, please clear L1 deployment volume before starting"
-    exit 1
-  fi
-fi
-
-# Generate the L2 account addresses
-yarn hardhat rekey-json --output /l2-accounts/keys.json
-
-# Read JSON file into variable
-KEYS_JSON=$(cat /l2-accounts/keys.json)
-
-# Parse JSON into variables
-ADMIN_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Admin.address')
-ADMIN_PRIV_KEY=$(echo "$KEYS_JSON" | jq -r '.Admin.privateKey')
-PROPOSER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Proposer.address')
-BATCHER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Batcher.address')
-SEQUENCER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Sequencer.address')
-
-# Get the private keys of L1 accounts
-if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \
-  l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \
-  [ "$l1_accounts_response" -eq 200 ];
-then
-  echo "Fetching L1 account credentials using provided URL"
-  mkdir -p /geth-accounts
-  wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL"
-
-  CERC_L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2)
-  CERC_L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3)
-  CERC_L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv)
-  CERC_L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv)
-else
-  echo "Couldn't fetch L1 account credentials, using them from env"
-fi
-
-# Send balances to the above L2 addresses
-yarn hardhat send-balance --to "${ADMIN_ADDRESS}" --amount 2 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
-yarn hardhat send-balance --to "${PROPOSER_ADDRESS}" --amount 5 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
-yarn hardhat send-balance --to "${BATCHER_ADDRESS}" --amount 1000 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
-
-echo "Balances sent to L2 accounts"
-
-# Select a finalized L1 block as the starting point for roll ups
-until FINALIZED_BLOCK=$(cast block finalized --rpc-url "$CERC_L1_RPC"); do
-    echo "Waiting for a finalized L1 block to exist, retrying after 10s"
-    sleep 10
-done
-
-L1_BLOCKNUMBER=$(echo "$FINALIZED_BLOCK" | awk '/number/{print $2}')
-L1_BLOCKHASH=$(echo "$FINALIZED_BLOCK" | awk '/hash/{print $2}')
-L1_BLOCKTIMESTAMP=$(echo "$FINALIZED_BLOCK" | awk '/timestamp/{print $2}')
-
-echo "Selected L1 block ${L1_BLOCKNUMBER} as the starting block for roll ups"
-
-# Update the deployment config
-sed -i 's/"l2OutputOracleStartingTimestamp": TIMESTAMP/"l2OutputOracleStartingTimestamp": '"$L1_BLOCKTIMESTAMP"'/g' deploy-config/getting-started.json
-jq --arg chainid "$CERC_L1_CHAIN_ID" '.l1ChainID = ($chainid | tonumber)' deploy-config/getting-started.json > tmp.json && mv tmp.json deploy-config/getting-started.json
-
-node update-config.js deploy-config/getting-started.json "$ADMIN_ADDRESS" "$PROPOSER_ADDRESS" "$BATCHER_ADDRESS" "$SEQUENCER_ADDRESS" "$L1_BLOCKHASH"
-
-echo "Updated the deployment config"
-
-# Create a .env file
-echo "L1_RPC=$CERC_L1_RPC" > .env
-echo "PRIVATE_KEY_DEPLOYER=$ADMIN_PRIV_KEY" >> .env
-
-echo "Deploying the L1 smart contracts, this will take a while..."
-
-# Deploy the L1 smart contracts
-yarn hardhat deploy --network getting-started --tags l1
-
-echo "Deployed the L1 smart contracts"
-
-# Read Proxy contract's JSON and get the address
-PROXY_JSON=$(cat deployments/getting-started/Proxy__OVM_L1StandardBridge.json)
-PROXY_ADDRESS=$(echo "$PROXY_JSON" | jq -r '.address')
-
-# Send balance to the above Proxy contract in L1 for reflecting balance in L2
-# First account
-yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started
-# Second account
-yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY_2}" --network getting-started
-
-echo "Balance sent to Proxy L2 contract"
-echo "Use following accounts for transactions in L2:"
-echo "${CERC_L1_ADDRESS}"
-echo "${CERC_L1_ADDRESS_2}"
-echo "Done"

stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js

@@ -1,36 +0,0 @@
-const fs = require('fs')
-
-// Get the command-line argument
-const configFile = process.argv[2]
-const adminAddress = process.argv[3]
-const proposerAddress = process.argv[4]
-const batcherAddress = process.argv[5]
-const sequencerAddress = process.argv[6]
-const blockHash = process.argv[7]
-
-// Read the JSON file
-const configData = fs.readFileSync(configFile)
-const configObj = JSON.parse(configData)
-
-// Update the finalSystemOwner property with the ADMIN_ADDRESS value
-configObj.finalSystemOwner =
-  configObj.portalGuardian =
-  configObj.controller =
-  configObj.l2OutputOracleChallenger =
-  configObj.proxyAdminOwner =
-  configObj.baseFeeVaultRecipient =
-  configObj.l1FeeVaultRecipient =
-  configObj.sequencerFeeVaultRecipient =
-  configObj.governanceTokenOwner =
-    adminAddress
-
-configObj.l2OutputOracleProposer = proposerAddress
-
-configObj.batchSenderAddress = batcherAddress
-
-configObj.p2pSequencerAddress = sequencerAddress
-
-configObj.l1StartingBlockTag = blockHash
-
-// Write the updated JSON object back to the file
-fs.writeFileSync(configFile, JSON.stringify(configObj, null, 2))

stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+# To facilitate deploying the Optimism contracts, a few additional arguments have been added to the geth start command
+# Otherwise this script is unchanged from the image's default startup script
+
+ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
+NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
+NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
+CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"
+
+cd /opt/testnet/build/el
+python3 -m http.server 9898 &
+cd $HOME
+
+START_CMD="geth"
+if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
+    START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
+fi
+
+# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
+cleanup() {
+    echo "Signal received, cleaning up..."
+
+    # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process)
+    pkill -P ${geth_pid}
+    sleep 2
+    kill $(jobs -p)
+
+    wait
+    echo "Done"
+}
+trap 'cleanup' SIGINT SIGTERM
+
+if [ "true" == "$RUN_BOOTNODE" ]; then
+    $START_CMD \
+      --datadir="${CERC_ETH_DATADIR}" \
+      --nodekeyhex="${BOOTNODE_KEY}" \
+      --nodiscover \
+      --ipcdisable \
+      --networkid=${NETWORK_ID} \
+      --netrestrict="${NETRESTRICT}" \
+      &
+
+    geth_pid=$!
+else
+    cd /opt/testnet/accounts
+    ./import_keys.sh
+
+    echo -n "$JWT" > /opt/testnet/build/el/jwtsecret
+
+    if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then
+      dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short)
+      dig_status_code=$?
+      if [[ $dig_status_code = 0 && -n $dig_result ]]; then
+        echo "Statediff DB at $CERC_STATEDIFF_DB_HOST"
+        CERC_RUN_STATEDIFF="true"
+      else
+        echo "No statediff DB available."
+        CERC_RUN_STATEDIFF="false"
+      fi
+    fi
+
+    STATEDIFF_OPTS=""
+    if [ "$CERC_RUN_STATEDIFF" == "true" ]; then
+      ready=0
+      echo "Waiting for statediff DB..."
+      while [ $ready -eq 0 ]; do
+        sleep 1
+        export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD"
+        result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \
+          -p "$CERC_STATEDIFF_DB_PORT" \
+          -U "$CERC_STATEDIFF_DB_USER" \
+          -d "$CERC_STATEDIFF_DB_NAME" \
+          -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }')
+        if [ -n "$result" ]; then
+          echo "DB ready..."
+          if [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then
+            ready=1
+          else
+            echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)"
+          fi
+        fi
+      done
+      STATEDIFF_OPTS="--statediff \
+      --statediff.db.host=$CERC_STATEDIFF_DB_HOST \
+      --statediff.db.name=$CERC_STATEDIFF_DB_NAME \
+      --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \
+      --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \
+      --statediff.db.port=$CERC_STATEDIFF_DB_PORT \
+      --statediff.db.user=$CERC_STATEDIFF_DB_USER \
+      --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \
+      --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \
+      --statediff.waitforsync=true \
+      --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \
+      --statediff.writing=true"
+
+      if [ -d "${CERC_PLUGINS_DIR}" ]; then
+        # With plugeth, we separate the statediff options by prefixing with ' -- '
+        STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}"
+      fi
+    fi
+
+    # unlock account[0]
+    echo $ACCOUNT_PASSWORD > "$CERC_ETH_DATADIR/password"
+
+    $START_CMD \
+      --datadir="${CERC_ETH_DATADIR}" \
+      --bootnodes="${ENODE}" \
+      --allow-insecure-unlock \
+      --password="${CERC_ETH_DATADIR}/password" \
+      --unlock="$ETHERBASE" \
+      --rpc.allow-unprotected-txs \
+      --http \
+      --http.addr="0.0.0.0" \
+      --http.vhosts="*" \
+      --http.api="${CERC_GETH_HTTP_APIS:-eth,web3,net,admin,personal,debug,statediff}" \
+      --http.corsdomain="*" \
+      --authrpc.addr="0.0.0.0" \
+      --authrpc.vhosts="*" \
+      --authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \
+      --ws \
+      --ws.addr="0.0.0.0" \
+      --ws.origins="*" \
+      --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \
+      --http.corsdomain="*" \
+      --networkid="${NETWORK_ID}" \
+      --netrestrict="${NETRESTRICT}" \
+      --gcmode archive \
+      --txlookuplimit=0 \
+      --cache.preimages \
+      --syncmode=full \
+      --mine \
+      --miner.threads=1 \
+      --metrics \
+      --metrics.addr="0.0.0.0" \
+      --verbosity=${CERC_GETH_VERBOSITY:-3} \
+      --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
+      --miner.etherbase="${ETHERBASE}" \
+      ${STATEDIFF_OPTS} \
+      &
+
+    geth_pid=$!
+fi
+
+wait $geth_pid
+
+if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then
+  while [ 1 -eq 1 ]; do
+    sleep 60
+  done
+fi

stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh

@@ -6,22 +6,14 @@ fi
 
 CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
 
-# Get Batcher key from keys.json
-BATCHER_KEY=$(jq -r '.Batcher.privateKey' /l2-accounts/keys.json | tr -d '"')
+# Start op-batcher
+L2_RPC="http://op-geth:8545"
+ROLLUP_RPC="http://op-node:8547"
+BATCHER_KEY=$(cat /l2-accounts/accounts.json | jq -r .BatcherKey)
 
-cleanup() {
-    echo "Signal received, cleaning up..."
-    kill ${batcher_pid}
-
-    wait
-    echo "Done"
-}
-trap 'cleanup' INT TERM
-
-# Run op-batcher
 op-batcher \
-  --l2-eth-rpc=http://op-geth:8545 \
-  --rollup-rpc=http://op-node:8547 \
+  --l2-eth-rpc=$L2_RPC \
+  --rollup-rpc=$ROLLUP_RPC \
   --poll-interval=1s \
   --sub-safety-margin=6 \
   --num-confirmations=1 \
@@ -32,8 +24,4 @@ op-batcher \
   --rpc.enable-admin \
   --max-channel-duration=1 \
   --l1-eth-rpc=$CERC_L1_RPC \
-  --private-key=$BATCHER_KEY \
-  &
-
-batcher_pid=$!
-wait $batcher_pid
+  --private-key="${BATCHER_KEY#0x}"

stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh

@@ -4,61 +4,36 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-# TODO: Add in container build or use other tool
-echo "Installing jq"
-apk update && apk add jq
+l2_genesis_file="/l2-config/genesis.json"
 
-# Get Sequencer key from keys.json
-SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"')
+# Check for genesis file; if necessary, wait on op-node to generate
+timeout=300 # 5 minutes
+start_time=$(date +%s)
+elapsed_time=0
+echo "Checking for L2 genesis file at location $l2_genesis_file"
+while [ ! -f "$l2_genesis_file" ] && [ $elapsed_time -lt $timeout ]; do
+  echo "Waiting for L2 genesis file to be generated..."
+  sleep 10
+  current_time=$(date +%s)
+  elapsed_time=$((current_time - start_time))
+done
 
-# Initialize op-geth if datadir/geth not found
-if [ -f /op-node/jwt.txt ] && [ -d datadir/geth ]; then
-  echo "Found existing datadir, checking block signer key"
-
-  BLOCK_SIGNER_KEY=$(cat datadir/block-signer-key)
-
-  if [ "$SEQUENCER_KEY" = "$BLOCK_SIGNER_KEY" ]; then
-    echo "Sequencer and block signer keys match, skipping initialization"
-  else
-    echo "Sequencer and block signer keys don't match, please clear L2 geth data volume before starting"
-    exit 1
-  fi
-else
-  echo "Initializing op-geth"
-
-  mkdir -p datadir
-  echo "pwd" > datadir/password
-  echo $SEQUENCER_KEY > datadir/block-signer-key
-
-  geth account import --datadir=datadir --password=datadir/password datadir/block-signer-key
-
-  while [ ! -f "/op-node/jwt.txt" ]
-  do
-    echo "Config files not created. Checking after 5 seconds."
-    sleep 5
-  done
-
-  echo "Config files created by op-node, proceeding with the initialization..."
-
-  geth init --datadir=datadir /op-node/genesis.json
-  echo "Node Initialized"
+if [ ! -f "$l2_genesis_file" ]; then
+  echo "L2 genesis file not found after timeout of $timeout seconds. Exiting..."
+  exit 1
 fi
 
-SEQUENCER_ADDRESS=$(jq -r '.Sequencer.address' /l2-accounts/keys.json | tr -d '"')
-echo "SEQUENCER_ADDRESS: ${SEQUENCER_ADDRESS}"
+# Initialize geth from our generated L2 genesis file (if not already initialized)
+data_dir="/datadir"
+if [ ! -d "$datadir/geth" ]; then
+  geth init --datadir=$data_dir $l2_genesis_file
+fi
 
-cleanup() {
-    echo "Signal received, cleaning up..."
-    kill ${geth_pid}
+# Start op-geth
+jwt_file="/l2-config/l2-jwt.txt"
 
-    wait
-    echo "Done"
-}
-trap 'cleanup' INT TERM
-
-# Run op-geth
 geth \
-  --datadir ./datadir \
+  --datadir=$data_dir \
   --http \
   --http.corsdomain="*" \
   --http.vhosts="*" \
@@ -77,14 +52,5 @@ geth \
   --authrpc.vhosts="*" \
   --authrpc.addr=0.0.0.0 \
   --authrpc.port=8551 \
-  --authrpc.jwtsecret=/op-node/jwt.txt \
-  --rollup.disabletxpoolgossip=true \
-  --password=./datadir/password \
-  --allow-insecure-unlock \
-  --mine \
-  --miner.etherbase=$SEQUENCER_ADDRESS \
-  --unlock=$SEQUENCER_ADDRESS \
-  &
-
-geth_pid=$!
-wait $geth_pid
+  --authrpc.jwtsecret=$jwt_file \
+  --rollup.disabletxpoolgossip=true

stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh

@@ -4,23 +4,42 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
+CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
 CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
+DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
 
-# Get Sequencer key from keys.json
-SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"')
+deploy_config_file="/l2-config/$DEPLOYMENT_CONTEXT.json"
+deployment_dir="/l1-deployment/$DEPLOYMENT_CONTEXT"
+genesis_outfile="/l2-config/genesis.json"
+rollup_outfile="/l2-config/rollup.json"
+
+# Generate L2 genesis (if not already done)
+if [ ! -f "$genesis_outfile" ] || [ ! -f "$rollup_outfile" ]; then
+	op-node genesis l2 \
+		--deploy-config $deploy_config_file \
+		--deployment-dir $deployment_dir \
+		--outfile.l2 $genesis_outfile \
+		--outfile.rollup $rollup_outfile \
+		--l1-rpc $CERC_L1_RPC
+fi
+
+# Start op-node
+SEQ_KEY=$(cat /l2-accounts/accounts.json | jq -r .SeqKey)
+jwt_file=/l2-config/l2-jwt.txt
+L2_AUTH="http://op-geth:8551"
+RPC_KIND=any # this can optionally be set to a preset for common node providers like Infura, Alchemy, etc.
 
-# Run op-node
 op-node \
-  --l2=http://op-geth:8551 \
-  --l2.jwt-secret=/op-node-data/jwt.txt \
-  --sequencer.enabled \
-  --sequencer.l1-confs=3 \
-  --verifier.l1-confs=3 \
-  --rollup.config=/op-node-data/rollup.json \
-  --rpc.addr=0.0.0.0 \
-  --rpc.port=8547 \
-  --p2p.disable \
-  --rpc.enable-admin \
-  --p2p.sequencer.key=$SEQUENCER_KEY \
-  --l1=$CERC_L1_RPC \
-  --l1.rpckind=any
+	--l2=$L2_AUTH \
+	--l2.jwt-secret=$jwt_file \
+	--sequencer.enabled \
+	--sequencer.l1-confs=5 \
+	--verifier.l1-confs=4 \
+	--rollup.config=$rollup_outfile \
+	--rpc.addr=0.0.0.0 \
+	--rpc.port=8547 \
+	--p2p.disable \
+	--rpc.enable-admin \
+	--p2p.sequencer.key="${SEQ_KEY#0x}" \
+	--l1=$CERC_L1_RPC \
+	--l1.rpckind=$RPC_KIND

stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh

@@ -5,32 +5,18 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
+CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
+DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
 
-# Read the L2OutputOracle contract address from the deployment
-L2OO_DEPLOYMENT=$(cat /contracts-bedrock/deployments/getting-started/L2OutputOracle.json)
-L2OO_ADDR=$(echo "$L2OO_DEPLOYMENT" | jq -r '.address')
+# Start op-proposer
+ROLLUP_RPC="http://op-node:8547"
+PROPOSER_KEY=$(cat /l2-accounts/accounts.json | jq -r .ProposerKey)
+L2OO_ADDR=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/L2OutputOracleProxy.json | jq -r .address)
 
-# Get Proposer key from keys.json
-PROPOSER_KEY=$(jq -r '.Proposer.privateKey' /l2-accounts/keys.json | tr -d '"')
-
-cleanup() {
-    echo "Signal received, cleaning up..."
-    kill ${proposer_pid}
-
-    wait
-    echo "Done"
-}
-trap 'cleanup' INT TERM
-
-# Run op-proposer
 op-proposer \
-  --poll-interval 12s \
-  --rpc.port 8560 \
-  --rollup-rpc http://op-node:8547 \
-  --l2oo-address $L2OO_ADDR \
-  --private-key $PROPOSER_KEY \
-  --l1-eth-rpc $CERC_L1_RPC \
-  &
-
-proposer_pid=$!
-wait $proposer_pid
+  --poll-interval=12s \
+  --rpc.port=8560 \
+  --rollup-rpc=$ROLLUP_RPC \
+  --l2oo-address="${L2OO_ADDR#0x}" \
+  --private-key="${PROPOSER_KEY#0x}" \
+  --l1-eth-rpc=$CERC_L1_RPC

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
+ARG VARIANT=20-bullseye
 FROM node:${VARIANT}
 
 ARG USERNAME=node

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -25,12 +25,12 @@ if [ -f ".env" ]; then
 fi
 
 for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o '^"CERC_RUNTIME_ENV[^\"]+"$'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)
     esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
-    echo "$cur_name=$cur_val"
+    echo "$f: $cur_name=$cur_val"
     sed -i "s/$orig_name/$esc_val/g" $f
   done
 done

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -10,21 +10,25 @@ WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
-cp next.config.js next.config.dist
+if [ ! -f "next.config.dist" ]; then
+  cp next.config.js next.config.dist
+fi
+
+which js-beautify >/dev/null
+if [ $? -ne 0 ]; then
+  npm i -g js-beautify
+fi
 
-npm i -g js-beautify
 js-beautify next.config.dist > next.config.js
 
-npm install
-
-CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
-MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1)
-
-head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1
-
-cat > next.config.js.2 <<EOF
-const webpack = require('webpack');
+WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
+if [ -z "$WEBPACK_REQ_LINE" ]; then
+  cat > next.config.js.0 <<EOF
+    const webpack = require('webpack');
+EOF
+fi
 
+cat > next.config.js.1 <<EOF
 let envMap;
 try {
   // .env-list.json provides us a list of identifiers which should be replaced at runtime.
@@ -43,21 +47,48 @@ try {
 }
 EOF
 
-grep 'module.exports' next.config.js > next.config.js.3
+CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
+ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1)
+WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1)
+NEXT_SECTION_ADJUSTMENT=0
 
-cat > next.config.js.4 <<EOF
-  webpack: (config) => {
-    config.plugins.push(new webpack.DefinePlugin(envMap));
-    return config;
-  },
+if [ -n "$WEBPACK_CONF_LINE" ]; then
+  WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2)
+  head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2
+  cat > next.config.js.3 <<EOF
+      $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
 EOF
+  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE - 1))
+elif [ -n "$ENV_LINE" ]; then
+  head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
+  cat > next.config.js.3 <<EOF
+    webpack: (config) => {
+      config.plugins.push(new webpack.DefinePlugin(envMap));
+      return config;
+    },
+EOF
+  NEXT_SECTION_ADJUSTMENT=2
+  NEXT_SECTION_LINE=$ENV_LINE
+else
+  echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
+  rm -f next.config.js.*
+  NEXT_SECTION_LINE=0
+fi
 
-tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5
+tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5
 
-cat next.config.js.* | js-beautify > next.config.js
+cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js
 rm next.config.js.*
 
 "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json
 
-npm run build
-rm .env-list.json
+if [ ! -f "package.dist" ]; then
+  cp package.json package.dist
+fi
+
+cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
+
+npm install || exit 1
+npm run cerc_compile || exit 1
+
+exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh

@@ -20,5 +20,10 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -
   done
 done
 
+NEXT_CONF="next.config.js next.config.dist"
+for f in $NEXT_CONF; do
+    cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+done
+
 cat $TMPF | sort -u | jq  --raw-input .  | jq --slurp .
 rm -f $TMPF

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -8,6 +8,27 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
 
-rm -rf .next-r
 "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
-npm start .next-r -p ${CERC_LISTEN_PORT:-3000}
+mv .next .next.old
+mv .next-r/.next .
+
+if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
+  jq -e '.scripts.cerc_generate' package.json >/dev/null
+  if [ $? -eq 0 ]; then
+    npm run cerc_generate > gen.out 2>&1 &
+    tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
+    count=0
+    while [ $count -lt 10 ]; do
+      sleep 1
+      ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
+      if [ $? -ne 0 ]; then 
+        break
+      else
+        count=$((count + 1))
+      fi
+    done
+    kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+  fi
+fi
+
+npm start . -p ${CERC_LISTEN_PORT:-3000}

stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile

@@ -17,6 +17,6 @@ WORKDIR /app
 COPY . .
 
 RUN echo "Building optimism" && \
-  yarn && yarn build
+  pnpm install && pnpm build
 
 WORKDIR /app/packages/contracts-bedrock

stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.0-alpine3.15 as builder
+FROM golang:1.21.0-alpine3.18 as builder
 
 ARG VERSION=v0.0.0
 
@@ -9,7 +9,7 @@ COPY ./op-batcher /app/op-batcher
 COPY ./op-bindings /app/op-bindings
 COPY ./op-node /app/op-node
 COPY ./op-service /app/op-service
-COPY ./op-signer /app/op-signer
+#COPY ./op-signer /app/op-signer
 COPY ./go.mod /app/go.mod
 COPY ./go.sum /app/go.sum
 
@@ -23,7 +23,7 @@ ARG TARGETOS TARGETARCH
 
 RUN make op-batcher VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
 
-FROM alpine:3.15
+FROM alpine:3.18
 
 RUN apk add --no-cache jq bash
 

stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.0-alpine3.15 as builder
+FROM golang:1.21.0-alpine3.18 as builder
 
 ARG VERSION=v0.0.0
 
@@ -21,7 +21,7 @@ ARG TARGETOS TARGETARCH
 
 RUN make op-node VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
 
-FROM alpine:3.15
+FROM alpine:3.18
 
 RUN apk add --no-cache openssl jq
 

stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.0-alpine3.15 as builder
+FROM golang:1.21.0-alpine3.18 as builder
 
 ARG VERSION=v0.0.0
 
@@ -9,7 +9,7 @@ COPY ./op-proposer /app/op-proposer
 COPY ./op-bindings /app/op-bindings
 COPY ./op-node /app/op-node
 COPY ./op-service /app/op-service
-COPY ./op-signer /app/op-signer
+#COPY ./op-signer /app/op-signer
 COPY ./go.mod /app/go.mod
 COPY ./go.sum /app/go.sum
 COPY ./.git /app/.git
@@ -22,7 +22,7 @@ ARG TARGETOS TARGETARCH
 
 RUN make op-proposer VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
 
-FROM alpine:3.15
+FROM alpine:3.18
 
 RUN apk add --no-cache jq bash
 

stack_orchestrator/data/stacks/fixturenet-optimism/README.md

@@ -12,6 +12,7 @@ Clone required repositories:
 laconic-so --stack fixturenet-optimism setup-repositories
 
 # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
+# The repositories are located in $HOME/cerc by default
 ```
 
 Build the container images:
@@ -39,20 +40,53 @@ This should create the required docker images in the local image registry:
 * `cerc/optimism-op-batcher`
 * `cerc/optimism-op-proposer`
 
-## Deploy
 
-Deploy the stack:
+## Create a deployment
 
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
 ```bash
-laconic-so --stack fixturenet-optimism deploy up
+laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml
 ```
 
-The `fixturenet-optimism-contracts` service takes a while to complete running as it:
-1. waits for the 'Merge' to happen on L1
-2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups)
-3. deploys the L1 contracts
-It may restart a few times after running into errors.
+### Ports
+It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
 
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
+
+For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `fixturenet-eth-geth-1` RPC to port 8545 and the `op-geth` RPC to port 9545 on the host.  
+
+Or, you may wish to use `any-same` for the initial mappings -- in which case you'll have to edit the spec to file to ensure the various geth instances aren't all trying to publish to host ports 8545/8546 at once.
+
+### Data volumes 
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+Once you've made any needed changes to the spec file, create a deployment from it:
+```bash
+laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment
+```
+
+## Start the stack
+Start the deployment:
+```bash
+laconic-so deployment --dir fixturenet-optimism-deployment start
+```
+1. The `fixturenet-eth` L1 chain will start up first and begin producing blocks.
+2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below).
+3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start
+4. The remaining services, `op-batcher` and `op-proposer` will start
+
+### Logs
 To list and monitor the running containers:
 
 ```bash
@@ -65,22 +99,69 @@ docker ps
 docker logs -f <CONTAINER_ID>
 ```
 
-## Clean up
+## Example: bridge some ETH from L1 to L2
 
-Stop all services running in the background:
+Send some ETH from the desired account to the `L1StandardBridgeProxy` contract on L1 to test bridging to L2.  
 
+We can use the testing account `0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F` which is pre-funded and unlocked, and the `cerc/foundry:local` container to make use of the `cast` cli.
+
+1. Note the docker network the stack is running on:
 ```bash
-laconic-so --stack fixturenet-optimism deploy down 30
+docker network ls
+# The network name will be something like laconic-[some_hash]_default
+```
+2. Set some variables:
+```bash
+L1_RPC=http://fixturenet-eth-geth-1:8545
+L2_RPC=http://op-geth:8545
+NETWORK=<the network name found above>
+DEPLOYMENT_CONTEXT=<L1 chain-id; 1212 by default>
+ACCOUNT=0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F
 ```
 
-Clear volumes created by this stack:
+If you need to check the L1 chain-id, you can use:
+```bash
+docker run --rm --network $NETWORK cerc/foundry:local "cast chain-id --rpc-url $L1_RPC"
+```
+
+3. Check the account starting balance on L2 (it should be 0):
+```bash
+docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC"
+# 0
+```
+
+4. Read the bridge contract address from the L1 deployment records in the `op-node` container:
+```bash
+# get the container id for op-node
+NODE_CONTAINER=$(docker ps --filter "name=op-node" -q)
+BRIDGE=$(docker exec $NODE_CONTAINER cat /l1-deployment/$DEPLOYMENT_CONTEXT/L1StandardBridgeProxy.json | jq -r .address)
+```
+
+5. Use cast to send some ETH to the bridge contract:
+```bash
+docker run --rm --network $NETWORK cerc/foundry:local "cast send --from $ACCOUNT --value 1ether $BRIDGE --rpc-url $L1_RPC"
+```
+
+6. Allow a couple minutes for the bridge to complete
+
+7. Check the L2 balance again (it should show the bridged funds):
+```bash
+docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC"
+# 1000000000000000000
+```
+
+## Clean up
+
+To stop all services running in the background, while preserving chain data:
 
 ```bash
-# List all relevant volumes
-docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
+laconic-so deployment --dir fixturenet-optimism-deployment stop
+```
 
-# Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
+To stop all services and also delete chain data:
+
+```bash
+laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes
 ```
 
 ## Troubleshooting

stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py

@@ -0,0 +1,39 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from ruamel.yaml import YAML
+
+
+def create(context: DeploymentContext, extra_args):
+    # Slightly modify the base fixturenet-eth compose file to replace the startup script for fixturenet-eth-geth-1
+    # We need to start geth with the flag to allow non eip-155 compliant transactions in order to publish the
+    # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment
+    fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml')
+
+    with open(fixturenet_eth_compose_file, 'r') as yaml_file:
+        yaml = YAML()
+        yaml_data = yaml.load(yaml_file)
+
+    new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh'
+
+    if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']:
+        yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script)
+
+    with open(fixturenet_eth_compose_file, 'w') as yaml_file:
+        yaml = YAML()
+        yaml.dump(yaml_data, yaml_file)
+
+    return None

stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md

@@ -1,4 +1,4 @@
-# fixturenet-optimism
+# fixturenet-optimism (L2-only)
 
 Instructions to setup and deploy L2 fixturenet using [Optimism](https://stack.optimism.io)
 
@@ -28,9 +28,43 @@ This should create the required docker images in the local image registry:
 * `cerc/optimism-op-batcher`
 * `cerc/optimism-op-proposer`
 
-## Deploy
+## Create a deployment
 
-Create and update an env file to be used in the next step ([defaults](../../config/fixturenet-optimism/l1-params.env)):
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+```bash
+laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml
+```
+### Ports
+It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
+Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+In addition, a stack-wide port mapping "recipe" can be applied at the time the 
+`laconic-so deploy init` command is run, by supplying the  desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
+| Recipe | Host Port Mapping |
+|--------|-------------------|
+| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
+| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
+| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
+| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
+| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
+
+For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `op-geth` RPC to an easy to remember port like 8545 or 9545 on the host.  
+
+### Data volumes 
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+Once you've made any needed changes to the spec file, create a deployment from it:
+```bash
+laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment
+```
+
+Finally, open the `stack.yml` file inside your deployment directory and, under the `pods:` section, remove (or comment out) the entry for `fixturenet-eth`. This will prevent the deployment from trying to spin up a new L1 chain when starting the stack.
+
+## Set chain env variables
+
+Inside the deployment directory, open the file `config.env` and add the following variables to point the stack at your L1 rpc and provide account credentials ([defaults](../../config/fixturenet-optimism/l1-params.env)):
 
   ```bash
   # External L1 endpoint
@@ -45,30 +79,29 @@ Create and update an env file to be used in the next step ([defaults](../../conf
   CERC_L1_ACCOUNTS_CSV_URL=
 
   # OR
-  # Specify the required account credentials
+  # Specify the required account credentials for the Admin account
+  # Other generated accounts will be funded from this account, so it should contain ~20 Eth
   CERC_L1_ADDRESS=
   CERC_L1_PRIV_KEY=
-  CERC_L1_ADDRESS_2=
-  CERC_L1_PRIV_KEY_2=
   ```
 
-* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port
-
-Deploy the stack:
+* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
 
+## Start the stack
+Start the deployment:
 ```bash
-laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism --env-file <PATH_TO_ENV_FILE> up
+laconic-so deployment --dir fixturenet-optimism-deployment start
 ```
+1. The stack will check for a response from the L1 endpoint specified in your env file.
+2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below).
+3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start
+4. The remaining services, `op-batcher` and `op-proposer` will start
 
-The `fixturenet-optimism-contracts` service may take a while (`~15 mins`) to complete running as it:
-1. waits for the 'Merge' to happen on L1
-2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups)
-3. deploys the L1 contracts
-
-To list down and monitor the running containers:
+### Logs
+To list and monitor the running containers:
 
 ```bash
-laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism ps
+laconic-so --stack fixturenet-optimism deploy ps
 
 # With status
 docker ps
@@ -79,20 +112,16 @@ docker logs -f <CONTAINER_ID>
 
 ## Clean up
 
-Stop all services running in the background:
+To stop all L2 services running in the background, while preserving chain data:
 
 ```bash
-laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism down 30
+laconic-so deployment --dir fixturenet-optimism-deployment stop
 ```
 
-Clear volumes created by this stack:
+To stop all L2 services and also delete chain data:
 
 ```bash
-# List all relevant volumes
-docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data"
-
-# Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data")
+laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes
 ```
 
 ## Troubleshooting

stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml

@@ -5,8 +5,8 @@ repos:
   - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5
   - git.vdb.to/cerc-io/lighthouse
   - github.com/dboreham/foundry
-  - github.com/ethereum-optimism/optimism@v1.0.4
-  - github.com/ethereum-optimism/op-geth@v1.101105.2
+  - github.com/ethereum-optimism/optimism@op-node/v1.3.0
+  - github.com/ethereum-optimism/op-geth@v1.101304.0
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse

stack_orchestrator/deploy/compose/deploy_docker.py

@@ -21,7 +21,7 @@ from stack_orchestrator.deploy.deployer import Deployer, DeployerException, Depl
 class DockerDeployer(Deployer):
     name: str = "compose"
 
-    def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
+    def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None:
         self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name,
                                    compose_env_file=compose_env_file)
 

stack_orchestrator/deploy/deploy.py

@@ -28,6 +28,7 @@ from stack_orchestrator.util import include_exclude_check, get_parsed_stack_conf
 from stack_orchestrator.deploy.deployer import Deployer, DeployerException
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.deployment_create import create as deployment_create
 from stack_orchestrator.deploy.deployment_create import init as deployment_init
 from stack_orchestrator.deploy.deployment_create import setup as deployment_setup
@@ -56,14 +57,17 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
     if deploy_to is None:
         deploy_to = "compose"
 
-    ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file, deploy_to)
+    ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
     # Subcommand is executed now, by the magic of click
 
 
-def create_deploy_context(global_context, stack, include, exclude, cluster, env_file, deployer):
+def create_deploy_context(
+        global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deployer):
     cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
+    deployment_dir = deployment_context.deployment_dir if deployment_context else None
     # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
-    deployer = getDeployer(deployer, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
+    deployer = getDeployer(deployer, deployment_dir, compose_files=cluster_context.compose_files,
+                           compose_project_name=cluster_context.cluster,
                            compose_env_file=cluster_context.env_file)
     return DeployCommandContext(stack, cluster_context, deployer)
 

stack_orchestrator/deploy/deploy_types.py

@@ -13,9 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
-from typing import List
+from typing import List, Mapping
 from dataclasses import dataclass
-from pathlib import Path
 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.deploy.deployer import Deployer
 
@@ -38,12 +37,6 @@ class DeployCommandContext:
     deployer: Deployer
 
 
-@dataclass
-class DeploymentContext:
-    deployment_dir: Path
-    command_context: DeployCommandContext
-
-
 @dataclass
 class VolumeMapping:
     host_path: str
@@ -66,3 +59,8 @@ class LaconicStackSetupCommand:
 @dataclass
 class LaconicStackCreateCommand:
     network_dir: str
+
+
+@dataclass
+class DeployEnvVars:
+    map: Mapping[str, str]

stack_orchestrator/deploy/deployer_factory.py

@@ -26,10 +26,10 @@ def getDeployerConfigGenerator(type: str):
         print(f"ERROR: deploy-to {type} is not valid")
 
 
-def getDeployer(type: str, compose_files, compose_project_name, compose_env_file):
+def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file):
     if type == "compose" or type is None:
-        return DockerDeployer(compose_files, compose_project_name, compose_env_file)
+        return DockerDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file)
     elif type == "k8s":
-        return K8sDeployer(compose_files, compose_project_name, compose_env_file)
+        return K8sDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file)
     else:
         print(f"ERROR: deploy-to {type} is not valid")

stack_orchestrator/deploy/deployment.py

@@ -18,34 +18,7 @@ from pathlib import Path
 import sys
 from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation
 from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context
-from stack_orchestrator.deploy.stack import Stack
-from stack_orchestrator.deploy.spec import Spec
-
-
-class DeploymentContext:
-    dir: Path
-    spec: Spec
-    stack: Stack
-
-    def get_stack_file(self):
-        return self.dir.joinpath("stack.yml")
-
-    def get_spec_file(self):
-        return self.dir.joinpath("spec.yml")
-
-    def get_env_file(self):
-        return self.dir.joinpath("config.env")
-
-    # TODO: implement me
-    def get_cluster_name(self):
-        return None
-
-    def init(self, dir):
-        self.dir = dir
-        self.stack = Stack()
-        self.stack.init_from_file(self.get_stack_file())
-        self.spec = Spec()
-        self.spec.init_from_file(self.get_spec_file())
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 
 
 @click.group()
@@ -77,7 +50,7 @@ def make_deploy_context(ctx):
     stack_file_path = context.get_stack_file()
     env_file = context.get_env_file()
     cluster_name = context.get_cluster_name()
-    return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file,
+    return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
                                  context.spec.obj["deploy-to"])
 
 

stack_orchestrator/deploy/deployment_context.py

@@ -0,0 +1,46 @@
+
+# Copyright © 2022, 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from pathlib import Path
+
+from stack_orchestrator.deploy.stack import Stack
+from stack_orchestrator.deploy.spec import Spec
+
+
+class DeploymentContext:
+    deployment_dir: Path
+    spec: Spec
+    stack: Stack
+
+    def get_stack_file(self):
+        return self.deployment_dir.joinpath("stack.yml")
+
+    def get_spec_file(self):
+        return self.deployment_dir.joinpath("spec.yml")
+
+    def get_env_file(self):
+        return self.deployment_dir.joinpath("config.env")
+
+    # TODO: implement me
+    def get_cluster_name(self):
+        return None
+
+    def init(self, dir):
+        self.deployment_dir = dir
+        self.spec = Spec()
+        self.spec.init_from_file(self.get_spec_file())
+        self.stack = Stack(self.spec.obj["stack"])
+        self.stack.init_from_file(self.get_stack_file())

stack_orchestrator/deploy/deployment_create.py

@@ -24,8 +24,9 @@ import sys
 from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
                                      global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
                                      get_pod_script_paths, get_plugin_code_paths)
-from stack_orchestrator.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand
 from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 
 
 def _make_default_deployment_dir():
@@ -108,8 +109,8 @@ def _fixup_pod_file(pod, spec, compose_dir):
                 pod["services"][container_name]["ports"] = container_ports
 
 
-def _commands_plugin_paths(ctx: DeployCommandContext):
-    plugin_paths = get_plugin_code_paths(ctx.stack)
+def _commands_plugin_paths(stack_name: str):
+    plugin_paths = get_plugin_code_paths(stack_name)
     ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
     return ret
 
@@ -123,7 +124,7 @@ def call_stack_deploy_init(deploy_command_context):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_paths = _commands_plugin_paths(deploy_command_context)
+    python_file_paths = _commands_plugin_paths(deploy_command_context.stack)
 
     ret = None
     init_done = False
@@ -147,7 +148,7 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_paths = _commands_plugin_paths(deploy_command_context)
+    python_file_paths = _commands_plugin_paths(deploy_command_context.stack)
     for python_file_path in python_file_paths:
         if python_file_path.exists():
             spec = util.spec_from_file_location("commands", python_file_path)
@@ -162,7 +163,7 @@ def call_stack_deploy_create(deployment_context, extra_args):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_paths = _commands_plugin_paths(deployment_context.command_context)
+    python_file_paths = _commands_plugin_paths(deployment_context.stack.name)
     for python_file_path in python_file_paths:
         if python_file_path.exists():
             spec = util.spec_from_file_location("commands", python_file_path)
@@ -311,7 +312,7 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path):
 def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     # This function fails with a useful error message if the file doens't exist
     parsed_spec = get_parsed_deployment_spec(spec_file)
-    stack_name = parsed_spec['stack']
+    stack_name = parsed_spec["stack"]
     stack_file = get_stack_file_path(stack_name)
     parsed_stack = get_parsed_stack_config(stack_name)
     if global_options(ctx).debug:
@@ -367,7 +368,8 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
     # stack member here.
     deployment_command_context = ctx.obj
     deployment_command_context.stack = stack_name
-    deployment_context = DeploymentContext(Path(deployment_dir), deployment_command_context)
+    deployment_context = DeploymentContext()
+    deployment_context.init(Path(deployment_dir))
     # Call the deployer to generate any deployer-specific files (e.g. for kind)
     deployer_config_generator = getDeployerConfigGenerator(parsed_spec["deploy-to"])
     # TODO: make deployment_dir a Path above

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -18,7 +18,9 @@ from typing import Any, List, Set
 
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files
-from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names
+from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names, get_node_pv_mount_path
+from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map
+from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 
 
 class ClusterInfo:
@@ -26,11 +28,12 @@ class ClusterInfo:
     image_set: Set[str] = set()
     app_name: str = "test-app"
     deployment_name: str = "test-deployment"
+    environment_variables: DeployEnvVars
 
     def __init__(self) -> None:
         pass
 
-    def int_from_pod_files(self, pod_files: List[str]):
+    def int(self, pod_files: List[str], compose_env_file):
         self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files)
         # Find the set of images in the pods
         for pod_name in self.parsed_pod_yaml_map:
@@ -42,6 +45,9 @@ class ClusterInfo:
                 self.image_set.add(image)
         if opts.o.debug:
             print(f"image_set: {self.image_set}")
+        self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file))
+        if (opts.o.debug):
+            print(f"Env vars: {self.environment_variables.map}")
 
     def get_pvcs(self):
         result = []
@@ -50,11 +56,12 @@ class ClusterInfo:
             print(f"Volumes: {volumes}")
         for volume_name in volumes:
             spec = client.V1PersistentVolumeClaimSpec(
-                storage_class_name="standard",
                 access_modes=["ReadWriteOnce"],
+                storage_class_name="manual",
                 resources=client.V1ResourceRequirements(
                     requests={"storage": "2Gi"}
-                )
+                ),
+                volume_name=volume_name
             )
             pvc = client.V1PersistentVolumeClaim(
                 metadata=client.V1ObjectMeta(name=volume_name,
@@ -64,6 +71,24 @@ class ClusterInfo:
             result.append(pvc)
         return result
 
+    def get_pvs(self):
+        result = []
+        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        for volume_name in volumes:
+            spec = client.V1PersistentVolumeSpec(
+                storage_class_name="manual",
+                access_modes=["ReadWriteOnce"],
+                capacity={"storage": "2Gi"},
+                host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
+            )
+            pv = client.V1PersistentVolume(
+                metadata=client.V1ObjectMeta(name=volume_name,
+                                             labels={"volume-label": volume_name}),
+                spec=spec,
+            )
+            result.append(pv)
+        return result
+
     # to suit the deployment, and also annotate the container specs to point at said volumes
     def get_deployment(self):
         containers = []
@@ -78,6 +103,7 @@ class ClusterInfo:
                 container = client.V1Container(
                     name=container_name,
                     image=image,
+                    env=envs_from_environment_variables_map(self.environment_variables.map),
                     ports=[client.V1ContainerPort(container_port=80)],
                     volume_mounts=volume_mounts,
                     resources=client.V1ResourceRequirements(

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -30,15 +30,18 @@ class K8sDeployer(Deployer):
     k8s_namespace: str = "default"
     kind_cluster_name: str
     cluster_info : ClusterInfo
+    deployment_dir: Path
 
-    def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
+    def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None:
         if (opts.o.debug):
+            print(f"Deployment dir: {deployment_dir}")
             print(f"Compose files: {compose_files}")
             print(f"Project name: {compose_project_name}")
             print(f"Env file: {compose_env_file}")
+        self.deployment_dir = deployment_dir
         self.kind_cluster_name = compose_project_name
         self.cluster_info = ClusterInfo()
-        self.cluster_info.int_from_pod_files(compose_files)
+        self.cluster_info.int(compose_files, compose_env_file)
 
     def connect_api(self):
         config.load_kube_config(context=f"kind-{self.kind_cluster_name}")
@@ -47,16 +50,26 @@ class K8sDeployer(Deployer):
 
     def up(self, detach, services):
         # Create the kind cluster
-        # HACK: pass in the config file path here
-        create_cluster(self.kind_cluster_name, "./test-deployment-dir/kind-config.yml")
+        create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath("kind-config.yml"))
         self.connect_api()
         # Ensure the referenced containers are copied into kind
         load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
+
+        # Create the host-path-mounted PVs for this deployment
+        pvs = self.cluster_info.get_pvs()
+        for pv in pvs:
+            if opts.o.debug:
+                print(f"Sending this pv: {pv}")
+            pv_resp = self.core_api.create_persistent_volume(body=pv)
+            if opts.o.debug:
+                print("PVs created:")
+                print(f"{pv_resp}")
+
         # Figure out the PVCs for this deployment
         pvcs = self.cluster_info.get_pvcs()
         for pvc in pvcs:
             if opts.o.debug:
-                print(f"Sending this: {pvc}")
+                print(f"Sending this pvc: {pvc}")
             pvc_resp = self.core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace=self.k8s_namespace)
             if opts.o.debug:
                 print("PVCs created:")
@@ -65,7 +78,7 @@ class K8sDeployer(Deployer):
         deployment = self.cluster_info.get_deployment()
         # Create the k8s objects
         if opts.o.debug:
-            print(f"Sending this: {deployment}")
+            print(f"Sending this deployment: {deployment}")
         deployment_resp = self.apps_api.create_namespaced_deployment(
             body=deployment, namespace=self.k8s_namespace
         )
@@ -122,6 +135,8 @@ class K8sDeployerConfigGenerator(DeployerConfigGenerator):
         # Check the file isn't already there
         # Get the config file contents
         content = generate_kind_config(deployment_dir)
+        if opts.o.debug:
+            print(f"kind config is: {content}")
         config_file = deployment_dir.joinpath(self.config_file_name)
         # Write the file
         with open(config_file, "w") as output_file:

stack_orchestrator/deploy/k8s/helpers.py

@@ -14,9 +14,11 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
 from kubernetes import client
+from dotenv import dotenv_values
+import os
 from pathlib import Path
 import subprocess
-from typing import Any, Set
+from typing import Any, Set, Mapping, List
 
 from stack_orchestrator.opts import opts
 from stack_orchestrator.util import get_yaml
@@ -73,6 +75,10 @@ def named_volumes_from_pod_files(parsed_pod_files):
     return named_volumes
 
 
+def get_node_pv_mount_path(volume_name: str):
+    return f"/mnt/{volume_name}"
+
+
 def volume_mounts_for_service(parsed_pod_files, service):
     result = []
     # Find the service
@@ -119,6 +125,14 @@ def _get_host_paths_for_volumes(parsed_pod_files):
     return result
 
 
+def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Path:
+    if os.path.isabs(data_mount_path):
+        return data_mount_path
+    else:
+        # Python Path voodo that looks pretty odd:
+        return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve()
+
+
 def parsed_pod_files_map_from_file_names(pod_files):
     parsed_pod_yaml_map : Any = {}
     for pod_file in pod_files:
@@ -130,9 +144,12 @@ def parsed_pod_files_map_from_file_names(pod_files):
     return parsed_pod_yaml_map
 
 
-def _generate_kind_mounts(parsed_pod_files):
+def _generate_kind_mounts(parsed_pod_files, deployment_dir):
     volume_definitions = []
     volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files)
+    # Note these paths are relative to the location of the pod files (at present)
+    # So we need to fix up to make them correct and absolute because kind assumes
+    # relative to the cwd.
     for pod in parsed_pod_files:
         parsed_pod_file = parsed_pod_files[pod]
         if "services" in parsed_pod_file:
@@ -145,7 +162,8 @@ def _generate_kind_mounts(parsed_pod_files):
                         # Looks like: test-data:/data
                         (volume_name, mount_path) = mount_string.split(":")
                         volume_definitions.append(
-                            f"  - hostPath: {volume_host_path_map[volume_name]}\n    containerPath: /var/local-path-provisioner"
+                            f"  - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
+                            f"    containerPath: {get_node_pv_mount_path(volume_name)}"
                             )
     return (
         "" if len(volume_definitions) == 0 else (
@@ -177,6 +195,13 @@ def _generate_kind_port_mappings(parsed_pod_files):
     )
 
 
+def envs_from_environment_variables_map(map: Mapping[str, str]) -> List[client.V1EnvVar]:
+    result = []
+    for env_var, env_val in map.items():
+        result.append(client.V1EnvVar(env_var, env_val))
+    return result
+
+
 # This needs to know:
 # The service ports for the cluster
 # The bind mounted volumes for the cluster
@@ -201,7 +226,7 @@ def generate_kind_config(deployment_dir: Path):
     pod_files = [p for p in compose_file_dir.iterdir() if p.is_file()]
     parsed_pod_files_map = parsed_pod_files_map_from_file_names(pod_files)
     port_mappings_yml = _generate_kind_port_mappings(parsed_pod_files_map)
-    mounts_yml = _generate_kind_mounts(parsed_pod_files_map)
+    mounts_yml = _generate_kind_mounts(parsed_pod_files_map, deployment_dir)
     return (
         "kind: Cluster\n"
         "apiVersion: kind.x-k8s.io/v1alpha4\n"
@@ -210,3 +235,7 @@ def generate_kind_config(deployment_dir: Path):
         f"{port_mappings_yml}\n"
         f"{mounts_yml}\n"
     )
+
+
+def env_var_map_from_file(file: Path) -> Mapping[str, str]:
+    return dotenv_values(file)

stack_orchestrator/deploy/stack.py

@@ -20,10 +20,11 @@ from stack_orchestrator.util import get_yaml
 
 class Stack:
 
+    name: str
     obj: typing.Any
 
-    def __init__(self) -> None:
-        pass
+    def __init__(self, name: str) -> None:
+        self.name = name
 
     def init_from_file(self, file_path: Path):
         with file_path:

tests/k8s-deploy/run-deploy-test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+# Note: eventually this test should be folded into ../deploy/
+# but keeping it separate for now for convenience
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+# Dump environment variables for debugging
+echo "Environment variables:"
+env
+# Set a non-default repo dir
+export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
+echo "Testing this package: $TEST_TARGET_SO"
+echo "Test version command"
+reported_version_string=$( $TEST_TARGET_SO version )
+echo "Version reported is: ${reported_version_string}"
+echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
+rm -rf $CERC_REPO_BASE_DIR
+mkdir -p $CERC_REPO_BASE_DIR
+# Test basic stack-orchestrator deploy
+test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
+test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
+$TEST_TARGET_SO --stack test deploy --deploy-to k8s init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
+# Check the file now exists
+if [ ! -f "$test_deployment_spec" ]; then
+    echo "deploy init test: spec file not present"
+    echo "deploy init test: FAILED"
+    exit 1
+fi
+echo "deploy init test: passed"
+$TEST_TARGET_SO --stack test deploy create --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+# Check the deployment dir exists
+if [ ! -d "$test_deployment_dir" ]; then
+    echo "deploy create test: deployment directory not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create test: passed"
+# Check the file writted by the create command in the stack now exists
+if [ ! -f "$test_deployment_dir/create-file" ]; then
+    echo "deploy create test: create output file not present"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+# And has the right content
+create_file_content=$(<$test_deployment_dir/create-file)
+if [ ! "$create_file_content" == "create-command-output-data"  ]; then
+    echo "deploy create test: create output file contents not correct"
+    echo "deploy create test: FAILED"
+    exit 1
+fi
+echo "deploy create output file test: passed"
+# Try to start the deployment
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+# TODO: add a check to see if the container is up
+# Sleep because k8s not up yet
+sleep 30
+# Check logs command works
+log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
+    echo "deployment logs test: passed"
+else
+    echo "deployment logs test: FAILED"
+    exit 1
+fi
+# Check the config variable CERC_TEST_PARAM_1 was passed correctly
+if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
+    echo "deployment config test: passed"
+else
+    echo "deployment config test: FAILED"
+    exit 1
+fi
+# Stop and clean up
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+echo "Test passed"

tests/webapp-test/run-webapp-test.sh

@@ -51,7 +51,7 @@ else
   echo "BEFORE: PASSED"
 fi
 
-grep "`uuidgen`" test.after > /dev/null
+grep "$UUID" test.after > /dev/null
 if [ $? -ne 0 ]; then
   echo "AFTER: FAILED"
   exit 1
@@ -59,4 +59,4 @@ else
   echo "AFTER: PASSED"
 fi
 
-exit 0
+exit 0