Merge pull request #371 from cerc-io/zramsay-patch-1

optimism: on error, wait, then re-run 'deploy up'
Former-commit-id: b74e89fd3f

app/data/compose/docker-compose-fixturenet-eth.yml

@@ -49,6 +49,8 @@ services:
       timeout: 10s
       retries: 10
       start_period: 3s
+    environment:
+      CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true"
     env_file:
       - ../config/fixturenet-eth/fixturenet-eth.env
     image: cerc/fixturenet-eth-geth:local

app/data/compose/docker-compose-fixturenet-optimism.yml

@@ -122,6 +122,8 @@ services:
     # Waits for L1 endpoint to be up before running the batcher
     command: |
       "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-batcher.sh"
+    ports:
+      - "127.0.0.1:8548:8548"
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -145,6 +147,8 @@ services:
     # Waits for L1 endpoint to be up before running the proposer
     command: |
       "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-proposer.sh"
+    ports:
+      - "127.0.0.1:8560:8560"
     extra_hosts:
       - "host.docker.internal:host-gateway"
 

app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -127,3 +127,9 @@ else
 fi
 
 wait $geth_pid
+
+if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then
+  while [ 1 -eq 1 ]; do
+    sleep 60
+  done
+fi

app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Exports the complete fixturenet-eth ethdb data to a tarball (default, ./ethdb.tgz), waiting for a minimum
+# block height (default 1000) to be reached before exporting.
+
+# Usage: export-ethdb.sh [min_block_number=1000] [output_file=./ethdb.tgz]
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+GETH_EXPORT_MIN_BLOCK=${1:-${GETH_EXPORT_MIN_BLOCK:-1000}}
+
+# Wait for block.
+${SCRIPT_DIR}/status.sh $GETH_EXPORT_MIN_BLOCK
+if [[ $? -ne 0 ]]; then
+  echo "Unable to export ethdb." 1>&2
+  exit 1
+fi
+
+# Stop geth.
+echo -n "Exporting ethdb.... "
+GETH_CONTAINER=`docker ps -q -f "name=${CERC_SO_COMPOSE_PROJECT}-fixturenet-eth-geth-2-1"`
+if [[ -z "$GETH_CONTAINER" ]]; then
+  echo "not found"
+  exit 1
+fi
+
+docker exec $GETH_CONTAINER sh -c 'rm -rf /root/tmp && mkdir -p /root/tmp/export'
+docker exec $GETH_CONTAINER sh -c 'ln -s /opt/testnet/build/el/geth.json /root/tmp/export/genesis.json && ln -s /root/ethdata /root/tmp/export/'
+docker exec $GETH_CONTAINER sh -c 'cat /root/tmp/export/genesis.json | jq ".config" > /root/tmp/export/genesis.config.json'
+
+# Stop geth and zip up ethdb.
+docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_getBlockByNumber\", \"params\": [\"0x0\", false]}" > /root/tmp/export/eth_getBlockByNumber_0x0.json'
+docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_blockNumber\", \"params\": []}" > /root/tmp/export/eth_blockNumber.json'
+docker exec $GETH_CONTAINER sh -c "killall geth && sleep 2 && tar chzf /root/tmp/ethdb.tgz -C /root/tmp/export ."
+
+# Copy ethdb to host.
+GETH_EXPORT_FILE=${2:-${GETH_EXPORT_FILE:-./ethdb.tgz}}
+docker cp $GETH_CONTAINER:/root/tmp/ethdb.tgz $GETH_EXPORT_FILE
+echo "$GETH_EXPORT_FILE"
+docker exec $GETH_CONTAINER sh -c "rm -rf /root/tmp"
+
+# Restart the container to get geth back up and running.
+docker restart $GETH_CONTAINER >/dev/null

app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh

@@ -2,9 +2,10 @@
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
-STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge")
-STATUS=0
 
+MIN_BLOCK_NUM=${1:-${MIN_BLOCK_NUM:-3}}
+STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge" "block number $MIN_BLOCK_NUM")
+STATUS=0
 
 LIGHTHOUSE_BASE_URL=${LIGHTHOUSE_BASE_URL}
 GETH_BASE_URL=${GETH_BASE_URL}
@@ -13,18 +14,29 @@ GETH_BASE_URL=${GETH_BASE_URL}
 # or some execution environment-neutral mechanism.
 if [ -z "$LIGHTHOUSE_BASE_URL" ]; then
   LIGHTHOUSE_CONTAINER=`docker ps -q -f "name=fixturenet-eth-lighthouse-1-1"`
+  if [ -z "$LIGHTHOUSE_CONTAINER" ]; then
+    echo "Lighthouse container not found." 1>&2
+    exit 1
+  fi
   LIGHTHOUSE_PORT=`docker port $LIGHTHOUSE_CONTAINER 8001 | cut -d':' -f2`
   LIGHTHOUSE_BASE_URL="http://localhost:${LIGHTHOUSE_PORT}"
 fi
 
 if [ -z "$GETH_BASE_URL" ]; then
   GETH_CONTAINER=`docker ps -q -f "name=fixturenet-eth-geth-1-1"`
+  if [ -z "$GETH_CONTAINER" ]; then
+    echo "Lighthouse container not found." 1>&2
+    exit 1
+  fi
   GETH_PORT=`docker port $GETH_CONTAINER 8545 | cut -d':' -f2`
   GETH_BASE_URL="http://localhost:${GETH_PORT}"
 fi
 
+MARKER="."
+
 function inc_status() {
   echo " done"
+  MARKEr="."
   STATUS=$((STATUS + 1))
   if [ $STATUS -lt ${#STATUSES[@]} ]; then
     echo -n "Waiting for ${STATUSES[$STATUS]}..."
@@ -34,7 +46,7 @@ function inc_status() {
 echo -n "Waiting for ${STATUSES[$STATUS]}..."
 while [ $STATUS -lt ${#STATUSES[@]} ]; do
   sleep 1
-  echo -n "."
+  echo -n "$MARKER"
   case $STATUS in
     0)
       result=`wget --no-check-certificate --quiet -O - --method POST --header 'Content-Type: application/json' \
@@ -67,5 +79,13 @@ while [ $STATUS -lt ${#STATUSES[@]} ]; do
         inc_status
       fi
       ;;
+    5)
+      result=`wget --no-check-certificate --quiet -O - "$LIGHTHOUSE_BASE_URL/eth/v2/beacon/blocks/head" | jq -r '.data.message.body.execution_payload.block_number'`
+      if [ ! -z "$result" ] && [ $result -gt $MIN_BLOCK_NUM ]; then
+        inc_status
+      else
+        MARKER="$result "
+      fi
+      ;;
   esac
 done

app/data/stacks/fixturenet-eth-tx/README.md

@@ -0,0 +1,37 @@
+# fixturenet-eth-tx
+
+A variation of `fixturenet-eth` that automatically generates transactions using `tx-spammer`.
+
+See `stacks/fixturenet-eth/README.md` for more information.
+
+## Containers
+
+* cerc/go-ethereum
+* cerc/lighthouse
+* cerc/fixturenet-eth-geth
+* cerc/fixturenet-eth-lighthouse
+* cerc/tx-spammer
+
+## Deploy the stack
+```
+$ laconic-so --stack fixturenet-eth-tx setup-repositories
+$ laconic-so --stack fixturenet-eth-tx build-containers
+$ laconic-so --stack fixturenet-eth-tx deploy up
+```
+
+## Export the ethdb (optional)
+
+It is easy to export data from the fixturenet for offline processing of the raw ethdb files (eg, by eth-statediff-service) using the `export-ethdb.sh` script.
+
+For example:
+
+```
+$ app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh 500
+Waiting for geth to generate DAG.... done
+Waiting for beacon phase0.... done
+Waiting for beacon altair.... done
+Waiting for beacon bellatrix pre-merge.... done
+Waiting for beacon bellatrix merge.... done
+Waiting for block number 500.... done
+Exporting ethdb.... ./ethdb.tgz
+```

app/data/stacks/fixturenet-eth-tx/stack.yml

@@ -0,0 +1,15 @@
+version: "1.1"
+name: fixturenet-eth-tx
+decription: "Ethereum Fixturenet w/ tx-spammer"
+repos:
+  - cerc-io/go-ethereum
+  - cerc-io/tx-spammer
+containers:
+  - cerc/go-ethereum
+  - cerc/lighthouse
+  - cerc/fixturenet-eth-geth
+  - cerc/fixturenet-eth-lighthouse
+  - cerc/tx-spammer
+pods:
+  - fixturenet-eth
+  - tx-spammer

app/data/stacks/fixturenet-optimism/README.md

@@ -2,7 +2,7 @@
 
 Instructions to setup and deploy an end-to-end L1+L2 stack with [fixturenet-eth](../fixturenet-eth/) (L1) and [Optimism](https://stack.optimism.io) (L2)
 
-We support running just the L2 part of stack, given an external L1 endpoint. Follow [l2-only](./l2-only.md) for the same.
+We support running just the L2 part of stack, given an external L1 endpoint. Follow the [L2 only doc](./l2-only.md) for the same.
 
 ## Setup
 
@@ -28,6 +28,8 @@ Build the container images:
 laconic-so --stack fixturenet-optimism build-containers
 ```
 
+Note: this will take >10 mins depending on the specs of your machine, and **requires** 16GB of memory or greater.
+
 This should create the required docker images in the local image registry:
 * `cerc/go-ethereum`
 * `cerc/lighthouse`
@@ -48,12 +50,14 @@ Deploy the stack:
 laconic-so --stack fixturenet-optimism deploy up
 ```
 
-The `fixturenet-optimism-contracts` service may take a while (`~15 mins`) to complete running as it:
+If you get the error `service "fixturenet-optimism-contracts" didn't complete successfully: exit 1` with ~25 lines of Traceback, wait 15-20 mins then re-run the command.
+
+The `fixturenet-optimism-contracts` service takes a while to complete running as it:
 1. waits for the 'Merge' to happen on L1
 2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups)
 3. deploys the L1 contracts
 
-To list down and monitor the running containers:
+To list and monitor the running containers:
 
 ```bash
 laconic-so --stack fixturenet-optimism deploy ps