From 98ab0951f9b23c73e2a189e77015ea66edc2d2f9 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 28 Jun 2023 17:59:57 -0500 Subject: [PATCH 1/4] Use latest plugeth-statediff. --- .../docker-compose-fixturenet-plugeth.yml | 2 -- .../cerc-fixturenet-eth-geth/run-el.sh | 31 ++++++++++++++----- .../cerc-fixturenet-eth-plugeth/Dockerfile | 27 ---------------- .../cerc-fixturenet-eth-plugeth/build.sh | 17 ---------- .../Dockerfile | 18 +++++------ .../stacks/fixturenet-plugeth-tx/stack.yml | 8 +++++ 6 files changed, 41 insertions(+), 62 deletions(-) delete mode 100644 app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile delete mode 100755 app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh diff --git a/app/data/compose/docker-compose-fixturenet-plugeth.yml b/app/data/compose/docker-compose-fixturenet-plugeth.yml index 58b00dc6..4780574c 100644 --- a/app/data/compose/docker-compose-fixturenet-plugeth.yml +++ b/app/data/compose/docker-compose-fixturenet-plugeth.yml @@ -29,7 +29,6 @@ services: image: cerc/fixturenet-plugeth-plugeth:local volumes: - fixturenet_plugeth_geth_1_data:/root/ethdata - - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins healthcheck: test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8545/"] interval: 30s @@ -61,7 +60,6 @@ services: - fixturenet-eth-bootnode-geth volumes: - fixturenet_plugeth_geth_2_data:/root/ethdata - - ../config/fixturenet-plugeth/plugins:/root/ethdata/plugins fixturenet-eth-bootnode-lighthouse: restart: always diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index 4c1a0ce1..338ca18f 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -7,11 +7,12 @@ fi ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2` NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'` NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'` +ETH_DATADIR="${ETH_DATADIR:-$HOME/ethdata}" +PLUGINS_DIR="${PLUGINS_DIR:-$ETH_DATADIR/plugins}" -HOME_DIR=`pwd` cd /opt/testnet/build/el python3 -m http.server 9898 & -cd $HOME_DIR +cd $HOME START_CMD="geth" if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then @@ -34,7 +35,7 @@ trap 'cleanup' SIGINT SIGTERM if [ "true" == "$RUN_BOOTNODE" ]; then $START_CMD \ - --datadir=~/ethdata \ + --datadir="$ETH_DATADIR" \ --nodekeyhex="${BOOTNODE_KEY}" \ --nodiscover \ --ipcdisable \ @@ -82,7 +83,7 @@ else fi fi done - STATEDIFF_OPTS="--statediff=true \ + STATEDIFF_OPTS="--statediff \ --statediff.db.host=$CERC_STATEDIFF_DB_HOST \ --statediff.db.name=$CERC_STATEDIFF_DB_NAME \ --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \ @@ -94,10 +95,25 @@ else --statediff.waitforsync=true \ --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ --statediff.writing=true" + + if [ -f "/usr/local/lib/statediff.so" ]; then + # With plugeth, we separate the statediff options by prefixing with ' -- ' + STATEDIFF_OPTS=" -- ${STATEDIFF_OPTS}" + + # Check if the plugins directory exists + if [ ! -d "${PLUGINS_DIR}" ]; then + mkdir -p "${PLUGINS_DIR}" + fi + + # And copy our plugin into place (if needed). + if [ ! -f "${PLUGINS_DIR}/statediff.so" ]; then + cp -f "/usr/local/lib/statediff.so" "${PLUGINS_DIR}/statediff.so" + fi + fi fi $START_CMD \ - --datadir=~/ethdata \ + --datadir="${ETH_DATADIR}" \ --bootnodes="${ENODE}" \ --allow-insecure-unlock \ --http \ @@ -124,8 +140,9 @@ else --metrics \ --metrics.addr="0.0.0.0" \ --verbosity=${CERC_GETH_VERBOSITY:-3} \ - --vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \ - --miner.etherbase="${ETHERBASE}" ${STATEDIFF_OPTS} \ + --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \ + --miner.etherbase="${ETHERBASE}" \ + ${STATEDIFF_OPTS} \ & geth_pid=$! diff --git a/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile b/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile deleted file mode 100644 index 90cd38d3..00000000 --- a/app/data/container-build/cerc-fixturenet-eth-plugeth/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen - -FROM golang:1.19.4-bullseye AS delve -RUN go install github.com/go-delve/delve/cmd/dlv@latest - -FROM ubuntu:22.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=delve /go/bin/dlv /usr/local/bin/ -COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/ -COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/ -COPY --from=ethgen /apps /apps - -RUN wget -O /usr/local/bin/geth https://github.com/openrelayxyz/plugeth/releases/download/v1.11.6.1.0/geth-linux-amd64-v1.1.0-v1.11.6.1.0 && chmod a+x /usr/local/bin/geth -RUN cd /apps/el-gen && pip3 install -r requirements.txt - -COPY genesis /opt/testnet -COPY run-el.sh /opt/testnet/run.sh - -RUN cd /opt/testnet && make genesis-el - -RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey - -ENTRYPOINT ["/opt/testnet/run.sh"] diff --git a/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh b/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh deleted file mode 100755 index e81bc932..00000000 --- a/app/data/container-build/cerc-fixturenet-eth-plugeth/build.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Build cerc/fixturenet-eth-plugeth -set -x - -source ${CERC_CONTAINER_BASE_DIR}/build-base.sh - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -if [ ! -d "${SCRIPT_DIR}/genesis" ]; then - cp -frp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/genesis ${SCRIPT_DIR}/genesis -fi - -if [ ! -d "${SCRIPT_DIR}/run-el.sh" ]; then - cp -fp ${SCRIPT_DIR}/../cerc-fixturenet-eth-geth/run-el.sh ${SCRIPT_DIR}/ -fi - -docker build -t cerc/fixturenet-eth-plugeth:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR diff --git a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile index 9936c3dc..7c362321 100644 --- a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile +++ b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile @@ -1,20 +1,19 @@ FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen -FROM golang:1.19.4-bullseye AS delve +FROM golang:1.19-alpine as delve RUN go install github.com/go-delve/delve/cmd/dlv@latest -FROM ubuntu:22.04 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python3 python3-dev python3-pip curl wget jq gettext gettext-base openssl bash dnsutils postgresql-client make iproute2 netcat psmisc && \ - rm -rf /var/lib/apt/lists/* +FROM cerc/plugeth:local as geth +FROM cerc/plugeth-statediff:local as statediff + +FROM alpine:3.17 +RUN apk add --no-cache python3 python3-dev py3-pip curl wget jq build-base gettext libintl openssl bash bind-tools postgresql-client COPY --from=delve /go/bin/dlv /usr/local/bin/ COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/ COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/ COPY --from=ethgen /apps /apps -RUN wget -O /usr/local/bin/geth https://github.com/openrelayxyz/plugeth/releases/download/v1.11.6.1.0/geth-linux-amd64-v1.1.0-v1.11.6.1.0 && chmod a+x /usr/local/bin/geth RUN cd /apps/el-gen && pip3 install -r requirements.txt COPY genesis /opt/testnet @@ -22,7 +21,8 @@ COPY run-el.sh /opt/testnet/run.sh RUN cd /opt/testnet && make genesis-el -RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey +COPY --from=geth /usr/local/bin/geth /usr/local/bin/ +COPY --from=statediff /usr/local/lib/statediff.so /usr/local/lib/ # Snag the genesis block info. RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey @@ -37,4 +37,4 @@ RUN cp -rp ~/ethdata ~/tmpeth && \ killall -9 geth && \ rm -rf ~/tmpeth -ENTRYPOINT ["/opt/testnet/run.sh"] +ENTRYPOINT ["/opt/testnet/run.sh"] \ No newline at end of file diff --git a/app/data/stacks/fixturenet-plugeth-tx/stack.yml b/app/data/stacks/fixturenet-plugeth-tx/stack.yml index e6d8c615..09d8edc5 100644 --- a/app/data/stacks/fixturenet-plugeth-tx/stack.yml +++ b/app/data/stacks/fixturenet-plugeth-tx/stack.yml @@ -5,14 +5,22 @@ repos: - github.com/cerc-io/tx-spammer - github.com/dboreham/foundry - github.com/cerc-io/lighthouse + - github.com/cerc-io/ipld-eth-db@v5 + - github.com/cerc-io/ipld-eth-server@v5 + - git.vdb.to/cerc-io/plugeth@statediff-wip + - git.vdb.to/cerc-io/plugeth-statediff@dev-local-build containers: - cerc/lighthouse - cerc/lighthouse-cli + - cerc/plugeth-statediff + - cerc/plugeth - cerc/fixturenet-plugeth-plugeth - cerc/fixturenet-plugeth-lighthouse - cerc/tx-spammer - cerc/foundry pods: + - ipld-eth-db + - ipld-eth-server - fixturenet-plugeth - foundry - tx-spammer -- 2.45.2 From 204083cca76f26febb738e16c9915966b8daa36c Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 28 Jun 2023 18:52:18 -0500 Subject: [PATCH 2/4] Change plugeth path --- app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh | 4 ++-- .../cerc-fixturenet-plugeth-plugeth/Dockerfile | 4 +++- app/data/stacks/fixturenet-plugeth-tx/stack.yml | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index 338ca18f..8bd883f0 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -96,7 +96,7 @@ else --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ --statediff.writing=true" - if [ -f "/usr/local/lib/statediff.so" ]; then + if [ -f "/usr/local/lib/plugeth/statediff.so" ]; then # With plugeth, we separate the statediff options by prefixing with ' -- ' STATEDIFF_OPTS=" -- ${STATEDIFF_OPTS}" @@ -107,7 +107,7 @@ else # And copy our plugin into place (if needed). if [ ! -f "${PLUGINS_DIR}/statediff.so" ]; then - cp -f "/usr/local/lib/statediff.so" "${PLUGINS_DIR}/statediff.so" + cp -f "/usr/local/lib/plugeth/statediff.so" "${PLUGINS_DIR}/statediff.so" fi fi fi diff --git a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile index 7c362321..b1a7aaaf 100644 --- a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile +++ b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile @@ -22,7 +22,9 @@ COPY run-el.sh /opt/testnet/run.sh RUN cd /opt/testnet && make genesis-el COPY --from=geth /usr/local/bin/geth /usr/local/bin/ -COPY --from=statediff /usr/local/lib/statediff.so /usr/local/lib/ + +RUN mkdir -p /usr/local/lib/plugeth/ +COPY --from=statediff /usr/local/lib/statediff.so /usr/local/lib/plugeth/ # Snag the genesis block info. RUN geth --datadir ~/ethdata init /opt/testnet/build/el/geth.json && rm -f ~/ethdata/geth/nodekey diff --git a/app/data/stacks/fixturenet-plugeth-tx/stack.yml b/app/data/stacks/fixturenet-plugeth-tx/stack.yml index 09d8edc5..50700e9b 100644 --- a/app/data/stacks/fixturenet-plugeth-tx/stack.yml +++ b/app/data/stacks/fixturenet-plugeth-tx/stack.yml @@ -18,6 +18,8 @@ containers: - cerc/fixturenet-plugeth-lighthouse - cerc/tx-spammer - cerc/foundry + - cerc/ipld-eth-db + - cerc/ipld-eth-server pods: - ipld-eth-db - ipld-eth-server -- 2.45.2 From a74c9b3814b6ba05d154b952a24316db4267ad8b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 28 Jun 2023 18:56:21 -0500 Subject: [PATCH 3/4] EOL --- .../container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile index b1a7aaaf..9469943f 100644 --- a/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile +++ b/app/data/container-build/cerc-fixturenet-plugeth-plugeth/Dockerfile @@ -39,4 +39,4 @@ RUN cp -rp ~/ethdata ~/tmpeth && \ killall -9 geth && \ rm -rf ~/tmpeth -ENTRYPOINT ["/opt/testnet/run.sh"] \ No newline at end of file +ENTRYPOINT ["/opt/testnet/run.sh"] -- 2.45.2 From b3f166266e7a804c4fb5e02db8c4873a066d47ce Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 28 Jun 2023 23:28:47 -0500 Subject: [PATCH 4/4] CERC_PLUGINS_DIR --- .../cerc-fixturenet-eth-geth/run-el.sh | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index 8bd883f0..d7cc3a83 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -7,8 +7,8 @@ fi ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2` NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'` NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'` -ETH_DATADIR="${ETH_DATADIR:-$HOME/ethdata}" -PLUGINS_DIR="${PLUGINS_DIR:-$ETH_DATADIR/plugins}" +CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}" +CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}" cd /opt/testnet/build/el python3 -m http.server 9898 & @@ -35,7 +35,7 @@ trap 'cleanup' SIGINT SIGTERM if [ "true" == "$RUN_BOOTNODE" ]; then $START_CMD \ - --datadir="$ETH_DATADIR" \ + --datadir="${CERC_ETH_DATADIR}" \ --nodekeyhex="${BOOTNODE_KEY}" \ --nodiscover \ --ipcdisable \ @@ -96,24 +96,14 @@ else --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ --statediff.writing=true" - if [ -f "/usr/local/lib/plugeth/statediff.so" ]; then + if [ -d "${CERC_PLUGINS_DIR}" ]; then # With plugeth, we separate the statediff options by prefixing with ' -- ' - STATEDIFF_OPTS=" -- ${STATEDIFF_OPTS}" - - # Check if the plugins directory exists - if [ ! -d "${PLUGINS_DIR}" ]; then - mkdir -p "${PLUGINS_DIR}" - fi - - # And copy our plugin into place (if needed). - if [ ! -f "${PLUGINS_DIR}/statediff.so" ]; then - cp -f "/usr/local/lib/plugeth/statediff.so" "${PLUGINS_DIR}/statediff.so" - fi + STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}" fi fi $START_CMD \ - --datadir="${ETH_DATADIR}" \ + --datadir="${CERC_ETH_DATADIR}" \ --bootnodes="${ENODE}" \ --allow-insecure-unlock \ --http \ -- 2.45.2