From d8522211f4875ab5a2338a7d4ce2916f5851365f Mon Sep 17 00:00:00 2001
From: Thomas E Lackey <telackey@bozemanpass.com>
Date: Wed, 26 Apr 2023 00:13:35 -0500
Subject: [PATCH] Add script for exporting ethdb from fixturenet. (#370)

* Add script for exporting ethdb from fixturenet.

* Update README

* Script

Former-commit-id: 7a607c299441126d3c9126dc4efa70b18b63ea48
---
 .../compose/docker-compose-fixturenet-eth.yml |  2 +
 .../cerc-fixturenet-eth-geth/run-el.sh        |  6 +++
 .../scripts/export-ethdb.sh                   | 38 +++++++++++++++++++
 .../scripts/status.sh                         | 26 +++++++++++--
 app/data/stacks/fixturenet-eth-tx/README.md   | 37 ++++++++++++++++++
 app/data/stacks/fixturenet-eth-tx/stack.yml   | 15 ++++++++
 6 files changed, 121 insertions(+), 3 deletions(-)
 create mode 100755 app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh
 create mode 100644 app/data/stacks/fixturenet-eth-tx/README.md
 create mode 100644 app/data/stacks/fixturenet-eth-tx/stack.yml

diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml
index c687b326..59f729b9 100644
--- a/app/data/compose/docker-compose-fixturenet-eth.yml
+++ b/app/data/compose/docker-compose-fixturenet-eth.yml
@@ -49,6 +49,8 @@ services:
       timeout: 10s
       retries: 10
       start_period: 3s
+    environment:
+      CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true"
     env_file:
       - ../config/fixturenet-eth/fixturenet-eth.env
     image: cerc/fixturenet-eth-geth:local
diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh
index 526c76d7..edde1ba2 100755
--- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh
+++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh
@@ -127,3 +127,9 @@ else
 fi
 
 wait $geth_pid
+
+if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then
+  while [ 1 -eq 1 ]; do
+    sleep 60
+  done
+fi
diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh
new file mode 100755
index 00000000..77198c61
--- /dev/null
+++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+MY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+GETH_EXPORT_MIN_BLOCK=${1:-${GETH_EXPORT_MIN_BLOCK:-1000}}
+
+# Wait for block.
+${MY_DIR}/status.sh $GETH_EXPORT_MIN_BLOCK
+if [ $? -ne 0 ]; then
+  echo "Unable to export ethdb." 1>&2
+  exit 1
+fi
+
+# Stop geth.
+echo -n "Exporting ethdb.... "
+GETH_CONTAINER=`docker ps -q -f "name=${CERC_SO_COMPOSE_PROJECT}-fixturenet-eth-geth-2-1"`
+if [ -z "$GETH_CONTAINER" ]; then 
+  echo "not found"
+  exit 1
+fi
+
+docker exec $GETH_CONTAINER sh -c 'rm -rf /root/tmp && mkdir -p /root/tmp/export'
+docker exec $GETH_CONTAINER sh -c 'ln -s /opt/testnet/build/el/geth.json /root/tmp/export/genesis.json && ln -s /root/ethdata /root/tmp/export/'
+docker exec $GETH_CONTAINER sh -c 'cat /root/tmp/export/genesis.json | jq ".config" > /root/tmp/export/genesis.config.json'
+
+# Stop geth and zip up ethdb.
+docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_getBlockByNumber\", \"params\": [\"0x0\", false]}" > /root/tmp/export/eth_getBlockByNumber_0x0.json'
+docker exec $GETH_CONTAINER sh -c 'curl -s --location "localhost:8545" --header "Content-Type: application/json" --data "{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"eth_blockNumber\", \"params\": []}" > /root/tmp/export/eth_blockNumber.json'
+docker exec $GETH_CONTAINER sh -c "killall geth && sleep 2 && tar chzf /root/tmp/ethdb.tgz -C /root/tmp/export ."
+
+# Copy ethdb to host.
+GETH_EXPORT_FILE=${2:-${GETH_EXPORT_FILE:-./ethdb.tgz}}
+docker cp $GETH_CONTAINER:/root/tmp/ethdb.tgz $GETH_EXPORT_FILE
+echo "$GETH_EXPORT_FILE"
+docker exec $GETH_CONTAINER sh -c "rm -rf /root/tmp"
+
+# Restart the container to get geth back up and running.
+docker restart $GETH_CONTAINER >/dev/null
diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
index c6e65a68..f6d9d27b 100755
--- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
+++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh
@@ -2,9 +2,10 @@
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
-STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge")
-STATUS=0
 
+MIN_BLOCK_NUM=${1:-${MIN_BLOCK_NUM:-3}}
+STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge" "block number $MIN_BLOCK_NUM")
+STATUS=0
 
 LIGHTHOUSE_BASE_URL=${LIGHTHOUSE_BASE_URL}
 GETH_BASE_URL=${GETH_BASE_URL}
@@ -13,18 +14,29 @@ GETH_BASE_URL=${GETH_BASE_URL}
 # or some execution environment-neutral mechanism.
 if [ -z "$LIGHTHOUSE_BASE_URL" ]; then
   LIGHTHOUSE_CONTAINER=`docker ps -q -f "name=fixturenet-eth-lighthouse-1-1"`
+  if [ -z "$LIGHTHOUSE_CONTAINER" ]; then
+    echo "Lighthouse container not found." 1>&2
+    exit 1
+  fi
   LIGHTHOUSE_PORT=`docker port $LIGHTHOUSE_CONTAINER 8001 | cut -d':' -f2`
   LIGHTHOUSE_BASE_URL="http://localhost:${LIGHTHOUSE_PORT}"
 fi
 
 if [ -z "$GETH_BASE_URL" ]; then
   GETH_CONTAINER=`docker ps -q -f "name=fixturenet-eth-geth-1-1"`
+  if [ -z "$GETH_CONTAINER" ]; then
+    echo "Lighthouse container not found." 1>&2
+    exit 1
+  fi
   GETH_PORT=`docker port $GETH_CONTAINER 8545 | cut -d':' -f2`
   GETH_BASE_URL="http://localhost:${GETH_PORT}"
 fi
 
+MARKER="."
+
 function inc_status() {
   echo " done"
+  MARKEr="."
   STATUS=$((STATUS + 1))
   if [ $STATUS -lt ${#STATUSES[@]} ]; then
     echo -n "Waiting for ${STATUSES[$STATUS]}..."
@@ -34,7 +46,7 @@ function inc_status() {
 echo -n "Waiting for ${STATUSES[$STATUS]}..."
 while [ $STATUS -lt ${#STATUSES[@]} ]; do
   sleep 1
-  echo -n "."
+  echo -n "$MARKER"
   case $STATUS in
     0)
       result=`wget --no-check-certificate --quiet -O - --method POST --header 'Content-Type: application/json' \
@@ -67,5 +79,13 @@ while [ $STATUS -lt ${#STATUSES[@]} ]; do
         inc_status
       fi
       ;;
+    5)
+      result=`wget --no-check-certificate --quiet -O - "$LIGHTHOUSE_BASE_URL/eth/v2/beacon/blocks/head" | jq -r '.data.message.body.execution_payload.block_number'`
+      if [ ! -z "$result" ] && [ $result -gt $MIN_BLOCK_NUM ]; then
+        inc_status
+      else
+        MARKER="$result "
+      fi
+      ;;
   esac
 done
diff --git a/app/data/stacks/fixturenet-eth-tx/README.md b/app/data/stacks/fixturenet-eth-tx/README.md
new file mode 100644
index 00000000..9af9ee3d
--- /dev/null
+++ b/app/data/stacks/fixturenet-eth-tx/README.md
@@ -0,0 +1,37 @@
+# fixturenet-eth-tx
+
+A variation of `fixturenet-eth` that automatically generates transactions using `tx-spammer`.
+
+See `stacks/fixturenet-eth/README.md` for more information.
+
+## Containers
+
+* cerc/go-ethereum
+* cerc/lighthouse
+* cerc/fixturenet-eth-geth
+* cerc/fixturenet-eth-lighthouse
+* cerc/tx-spammer
+
+## Deploy the stack
+```
+$ laconic-so --stack fixturenet-eth-tx setup-repositories
+$ laconic-so --stack fixturenet-eth-tx build-containers
+$ laconic-so --stack fixturenet-eth-tx deploy up
+```
+
+## Export the ethdb (optional)
+
+It is easy to export data from the fixturenet for offline processing of the raw ethdb files (eg, by eth-statediff-service) using the `export-ethdb.sh` script.
+
+For example:
+
+```
+❯ app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/export-ethdb.sh 500
+Waiting for geth to generate DAG.... done
+Waiting for beacon phase0.... done
+Waiting for beacon altair.... done
+Waiting for beacon bellatrix pre-merge.... done
+Waiting for beacon bellatrix merge.... done
+Waiting for block number 500.... done
+Exporting ethdb.... ./ethdb.tgz
+```
diff --git a/app/data/stacks/fixturenet-eth-tx/stack.yml b/app/data/stacks/fixturenet-eth-tx/stack.yml
new file mode 100644
index 00000000..0c3b45a3
--- /dev/null
+++ b/app/data/stacks/fixturenet-eth-tx/stack.yml
@@ -0,0 +1,15 @@
+version: "1.1"
+name: fixturenet-eth-tx
+decription: "Ethereum Fixturenet w/ tx-spammer"
+repos:
+  - cerc-io/go-ethereum
+  - cerc-io/tx-spammer
+containers:
+  - cerc/go-ethereum
+  - cerc/lighthouse
+  - cerc/fixturenet-eth-geth
+  - cerc/fixturenet-eth-lighthouse
+  - cerc/tx-spammer
+pods:
+  - fixturenet-eth
+  - tx-spammer