Add a pod for generating geth data as quickly as possible.

Former-commit-id: e8bdb205c7faa89c583b0652fac1801d0177a3c1

app/data/compose/docker-compose-datanet-eth.yml

@@ -0,0 +1,68 @@
+version: '3.7'
+
+services:
+  datanet-eth-bootnode-geth:
+    hostname: datanet-eth-bootnode-geth
+    env_file:
+      - ../config/datanet-eth/datanet-eth.env
+    environment:
+      RUN_BOOTNODE: "true"
+    image: cerc/datanet-eth-geth:local
+    ports:
+      - "9898"
+      - "30303"
+
+  datanet-eth-geth-1:
+    hostname: datanet-eth-geth-1
+    cap_add:
+      - SYS_PTRACE
+    environment:
+      CERC_REMOTE_DEBUG: "true"
+      CERC_RUN_STATEDIFF: "detect"
+      CERC_STATEDIFF_DB_NODE_ID: 1
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    env_file:
+      - ../config/datanet-eth/datanet-eth.env
+    image: cerc/datanet-eth-geth:local
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8545"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    depends_on:
+      - datanet-eth-bootnode-geth
+    ports:
+      - "8545"
+      - "40000"
+      - "6060"
+
+  datanet-eth-geth-2:
+    hostname: datanet-eth-geth-2
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "8545"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    env_file:
+      - ../config/datanet-eth/datanet-eth.env
+    image: cerc/datanet-eth-geth:local
+    depends_on:
+      - datanet-eth-bootnode-geth
+
+  datanet-tx-spammer:
+    restart: always
+    image: cerc/tx-spammer:local
+    environment:
+      ACCOUNTS_CSV_URL: http://datanet-eth-bootnode-geth:9898/accounts.csv
+      ETH_HTTP_PATH: http://datanet-eth-geth-1:8545
+      LOG_LEVEL: info
+      SPAMMER_COMMAND: autoSend
+      ETH_CALL_FREQ: 0
+      ETH_SEND_FREQ: 0
+    depends_on:
+      datanet-eth-bootnode-geth:
+        condition: service_started
+      datanet-eth-geth-1:
+        condition: service_healthy

app/data/config/datanet-eth/datanet-eth.env

@@ -0,0 +1,23 @@
+# The password used to access test accounts (eg, via personal_unlockAccount).  The password is the same for all accounts.
+ACCOUNT_PASSWORD=secret1212
+
+# ENODE of the geth bootnode.
+BOOTNODE_KEY="b0ac22adcad37213c7c565810a50f1772291e7b0ce53fb73e7ec2a3c75bc13b5"
+ENODE="enode://af22c29c316ad069cf48a09a4ad5cf04a251b411e45098888d114c6dd7f489a13786620d5953738762afa13711d4ffb3b19aa5de772d8af72f851f7e9c5b164a@datanet-eth-bootnode-geth:30303"
+
+# JWT shared by geth and lighthouse for authentication.
+JWT="0x6cdcac3501046a08e186730dd8bd136cfaf0fdc1fc955f6e15ad3068c0ff2af0"
+
+# URL to download the ENR of the lighthouse bootnode (generated at first start).
+ENR_URL="http://datanet-eth-bootnode-lighthouse:3000/bootnode/enr.dat"
+
+# DB connection settings for statediffing (see docker-compose-db.yml)
+CERC_STATEDIFF_DB_HOST="datanet-ipld-eth-db"
+CERC_STATEDIFF_DB_PORT=5432
+CERC_STATEDIFF_DB_NAME="cerc_testing"
+CERC_STATEDIFF_DB_USER="vdbm"
+CERC_STATEDIFF_DB_PASSWORD="password"
+CERC_STATEDIFF_DB_GOOSE_MIN_VER=23
+CERC_STATEDIFF_DB_LOG_STATEMENTS="false"
+
+CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"

app/data/container-build/cerc-datanet-eth-geth/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+#Build cerc/datanet-eth-geth
+
+set +e
+set+x
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+rm -rf $SCRIPT_DIR/build
+cp -rp $SCRIPT_DIR/../cerc-fixturenet-eth-geth $SCRIPT_DIR/build
+
+cp -f $SCRIPT_DIR/el-config.yaml $SCRIPT_DIR/build/genesis/el/el-config.yaml
+
+docker build -t cerc/datanet-eth-geth:local ${SCRIPT_DIR}/build

app/data/container-build/cerc-datanet-eth-geth/build/Dockerfile

@@ -0,0 +1,27 @@
+FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+
+# Using the same golang image as used to build geth: https://github.com/cerc-io/go-ethereum/blob/HEAD/Dockerfile
+FROM golang:1.18-alpine as delve
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+FROM cerc/go-ethereum:local as geth
+
+FROM alpine:latest
+RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client
+
+COPY --from=delve  /go/bin/dlv /usr/local/bin/
+COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
+COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
+COPY --from=ethgen /apps /apps
+
+RUN cd /apps/el-gen && pip3 install -r requirements.txt
+
+COPY genesis /opt/testnet
+COPY run-el.sh /opt/testnet/run.sh
+
+RUN cd /opt/testnet && make genesis-el
+
+COPY --from=geth  /usr/local/bin/geth /usr/local/bin/
+RUN geth init /opt/testnet/build/el/geth.json && rm -f ~/.ethereum/geth/nodekey
+
+ENTRYPOINT ["/opt/testnet/run.sh"]

app/data/container-build/cerc-datanet-eth-geth/build/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Build cerc/fixturenet-eth-geth
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/fixturenet-eth-geth:local -f ${SCRIPT_DIR}/Dockerfile $SCRIPT_DIR

app/data/container-build/cerc-datanet-eth-geth/build/genesis/Makefile

@@ -0,0 +1,13 @@
+.PHONY: build
+build: genesis
+
+.PHONY: genesis
+genesis: genesis-el
+
+.PHONY: genesis-el
+genesis-el:
+	cd el; ./build_el.sh
+
+.PHONY: clean
+clean:
+	rm -rf build

app/data/container-build/cerc-datanet-eth-geth/build/genesis/accounts/import_keys.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+ACCOUNT_PASSWORD=${ACCOUNT_PASSWORD:-secret1212}
+
+for line in `cat ../build/el/accounts.csv`; do
+  BIP44_PATH="`echo "$line" | cut -d',' -f1`"
+  ADDRESS="`echo "$line" | cut -d',' -f2`"
+  PRIVATE_KEY="`echo "$line" | cut -d',' -f3`"
+
+  echo "$ACCOUNT_PASSWORD" > .pw.$$
+  echo "$PRIVATE_KEY" | sed 's/0x//' > .key.$$
+
+  echo ""
+  echo "$ADDRESS"
+  geth account import --password .pw.$$ .key.$$
+  rm -f .pw.$$ .key.$$
+done

app/data/container-build/cerc-datanet-eth-geth/build/genesis/accounts/mnemonic_to_csv.py

@@ -0,0 +1,17 @@
+from web3.auto import w3
+import json
+import ruamel.yaml as yaml
+import sys
+
+w3.eth.account.enable_unaudited_hdwallet_features()
+
+testnet_config_path = "genesis-config.yaml"
+if len(sys.argv) > 1:
+    testnet_config_path = sys.argv[1]
+
+with open(testnet_config_path) as stream:
+    data = yaml.safe_load(stream)
+
+for key, value in data['el_premine'].items():
+    acct = w3.eth.account.from_mnemonic(data['mnemonic'], account_path=key, passphrase='')
+    print("%s,%s,%s" % (key, acct.address, acct.key.hex()))

app/data/container-build/cerc-datanet-eth-geth/build/genesis/el/build_el.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+# See: https://github.com/skylenet/ethereum-genesis-generator/blob/master/entrypoint.sh
+
+rm -rf ../build/el
+mkdir -p ../build/el
+
+tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
+envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml
+
+ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
+homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
+eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
+eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
+eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
+byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
+constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
+petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
+istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
+berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
+london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
+merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
+capped_maximum_difficulty=`cat $tmp_dir/genesis-config.yaml | grep capped_maximum_difficulty | awk '{ print $2 }'`
+
+python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv
+
+python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
+  jq ".config.homesteadBlock=$homestead_block" | \
+  jq ".config.eip150Block=$eip150_block" | \
+  jq ".config.eip155Block=$eip155_block" | \
+  jq ".config.eip158Block=$eip158_block" | \
+  jq ".config.byzantiumBlock=$byzantium_block" | \
+  jq ".config.constantinopleBlock=$constantinople_block" | \
+  jq ".config.petersburgBlock=$petersburg_block" | \
+  jq ".config.istanbulBlock=$istanbul_block" | \
+  jq ".config.berlinBlock=$berlin_block" | \
+  jq ".config.londonBlock=$london_block" | \
+  jq ".config.mergeForkBlock=$merge_fork_block" > ../build/el/geth.json
+
+if [ -n "$ttd" ]; then
+  cat ../build/el/geth.json | jq ".config.terminalTotalDifficulty=$ttd" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+else
+  cat ../build/el/geth.json | jq "del(.config.terminalTotalDifficulty)" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+fi
+
+if [ -n "$capped_maximum_difficulty" ]; then
+  cat ../build/el/geth.json | jq ".config.cappedMaximumDifficulty=$capped_maximum_difficulty" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+else
+  cat ../build/el/geth.json | jq "del(.config.cappedMaximumDifficulty)" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+fi

app/data/container-build/cerc-datanet-eth-geth/build/genesis/el/el-config.yaml

@@ -0,0 +1,31 @@
+mnemonic: "viable ketchup woman library opinion copy rhythm attend rose knock penalty practice photo bundle budget dentist enter round bind holiday useful arch danger lobster"
+el_premine:
+  "m/44'/60'/0'/0/0": 10000000ETH
+  "m/44'/60'/0'/0/1": 10000000ETH
+  "m/44'/60'/0'/0/2": 10000000ETH
+  "m/44'/60'/0'/0/3": 10000000ETH
+  "m/44'/60'/0'/0/4": 10000000ETH
+  "m/44'/60'/0'/0/5": 10000000ETH
+el_premine_addrs: {}
+chain_id: 1212
+deposit_contract_address: "0x1212121212121212121212121212121212121212"
+genesis_timestamp: 0
+capped_maximum_difficulty: 1
+homestead_block: 1
+eip150_block: 1
+eip155_block: 1
+eip158_block: 1
+byzantium_block: 1
+constantinople_block: 1
+petersburg_block: 1
+istanbul_block: 1
+berlin_block: 1
+london_block: 1
+merge_fork_block: 1
+
+clique:
+  enabled: false
+  signers:
+    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
+    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
+    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd

app/data/container-build/cerc-datanet-eth-geth/build/run-el.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
+NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
+NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+
+HOME_DIR=`pwd`
+cd /opt/testnet/build/el
+python3 -m http.server 9898 &
+cd $HOME_DIR
+
+START_CMD="geth"
+if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
+    START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
+fi
+
+if [ "true" == "$RUN_BOOTNODE" ]; then
+    $START_CMD \
+      --nodekeyhex="${BOOTNODE_KEY}" \
+      --nodiscover \
+      --ipcdisable \
+      --networkid=${NETWORK_ID} \
+      --netrestrict="${NETRESTRICT}"
+else
+    cd /opt/testnet/accounts
+    ./import_keys.sh
+    
+    echo -n "$JWT" > /opt/testnet/build/el/jwtsecret
+
+    if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then
+      dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short)
+      dig_status_code=$?
+      if [[ $dig_status_code = 0 && -n $dig_result ]]; then
+        echo "Statediff DB at $CERC_STATEDIFF_DB_HOST"
+        CERC_RUN_STATEDIFF="true"
+      else
+        echo "No statediff DB available."
+        CERC_RUN_STATEDIFF="false"
+      fi
+    fi
+
+    STATEDIFF_OPTS=""
+    if [ "$CERC_RUN_STATEDIFF" == "true" ]; then
+      ready=0
+      while [ $ready -eq 0 ]; do
+        echo "Waiting for statediff DB..."
+        sleep 1
+        export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD"
+        result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \
+          -p "$CERC_STATEDIFF_DB_PORT" \
+          -U "$CERC_STATEDIFF_DB_USER" \
+          -d "$CERC_STATEDIFF_DB_NAME" \
+          -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }')
+        if [ -n "$result" ] && [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then
+          echo "DB ready..."
+          ready=1
+        fi
+      done
+      STATEDIFF_OPTS="--statediff=true \
+      --statediff.db.host=$CERC_STATEDIFF_DB_HOST \
+      --statediff.db.name=$CERC_STATEDIFF_DB_NAME \
+      --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \
+      --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \
+      --statediff.db.port=$CERC_STATEDIFF_DB_PORT \
+      --statediff.db.user=$CERC_STATEDIFF_DB_USER \
+      --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \
+      --statediff.waitforsync=true \
+      --statediff.writing=true"
+    fi
+
+    $START_CMD \
+      --bootnodes="${ENODE}" \
+      --allow-insecure-unlock \
+      --http \
+      --http.addr="0.0.0.0" \
+      --http.vhosts="*" \
+      --http.api="eth,web3,net,admin,personal,debug,statediff" \
+      --http.corsdomain="*" \
+      --authrpc.addr="0.0.0.0" \
+      --authrpc.vhosts="*" \
+      --authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \
+      --networkid="${NETWORK_ID}" \
+      --netrestrict="${NETRESTRICT}" \
+      --gcmode archive \
+      --txlookuplimit=0 \
+      --cache.preimages \
+      --syncmode=full \
+      --mine \
+      --miner.threads=1 \
+      --metrics \
+      --metrics.addr="0.0.0.0" \
+      --verbosity=${CERC_GETH_VERBOSITY:-3} \
+      --vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
+      --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS}
+fi

app/data/container-build/cerc-datanet-eth-geth/el-config.yaml

@@ -0,0 +1,31 @@
+mnemonic: "viable ketchup woman library opinion copy rhythm attend rose knock penalty practice photo bundle budget dentist enter round bind holiday useful arch danger lobster"
+el_premine:
+  "m/44'/60'/0'/0/0": 10000000ETH
+  "m/44'/60'/0'/0/1": 10000000ETH
+  "m/44'/60'/0'/0/2": 10000000ETH
+  "m/44'/60'/0'/0/3": 10000000ETH
+  "m/44'/60'/0'/0/4": 10000000ETH
+  "m/44'/60'/0'/0/5": 10000000ETH
+el_premine_addrs: {}
+chain_id: 1212
+deposit_contract_address: "0x1212121212121212121212121212121212121212"
+genesis_timestamp: 0
+capped_maximum_difficulty: 1
+homestead_block: 1
+eip150_block: 1
+eip155_block: 1
+eip158_block: 1
+byzantium_block: 1
+constantinople_block: 1
+petersburg_block: 1
+istanbul_block: 1
+berlin_block: 1
+london_block: 1
+merge_fork_block: 1
+
+clique:
+  enabled: false
+  signers:
+    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
+    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
+    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd

app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/build_el.sh

@@ -21,9 +21,11 @@ istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{
 berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
 london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
 merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
+capped_maximum_difficulty=`cat $tmp_dir/genesis-config.yaml | grep capped_maximum_difficulty | awk '{ print $2 }'`
+
+python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv
 
 python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
-  jq ".config.terminalTotalDifficulty=$ttd" | \
   jq ".config.homesteadBlock=$homestead_block" | \
   jq ".config.eip150Block=$eip150_block" | \
   jq ".config.eip155Block=$eip155_block" | \
@@ -35,4 +37,19 @@ python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
   jq ".config.berlinBlock=$berlin_block" | \
   jq ".config.londonBlock=$london_block" | \
   jq ".config.mergeForkBlock=$merge_fork_block" > ../build/el/geth.json
-python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv
+
+if [ -n "$ttd" ]; then
+  cat ../build/el/geth.json | jq ".config.terminalTotalDifficulty=$ttd" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+else
+  cat ../build/el/geth.json | jq "del(.config.terminalTotalDifficulty)" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+fi
+
+if [ -n "$capped_maximum_difficulty" ]; then
+  cat ../build/el/geth.json | jq ".config.cappedMaximumDifficulty=$capped_maximum_difficulty" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+else
+  cat ../build/el/geth.json | jq "del(.config.cappedMaximumDifficulty)" > ../build/el/geth.json.jq
+  mv ../build/el/geth.json.jq ../build/el/geth.json
+fi

app/data/container-image-list.txt

@@ -11,6 +11,7 @@ cerc/ipld-eth-beacon-indexer
 cerc/ipld-eth-server
 cerc/laconicd
 cerc/laconic-registry-cli
+cerc/datanet-eth-geth
 cerc/fixturenet-eth-geth
 cerc/fixturenet-eth-lighthouse
 cerc/watcher-mobymask

app/data/pod-list.txt

@@ -8,6 +8,7 @@ ipld-eth-server
 lighthouse
 laconicd
 fixturenet-laconicd
+datanet-eth
 fixturenet-eth
 fixturenet-eth-metrics
 watcher-mobymask