From ff616db4ad16e299be79acf465b99029ac615b94 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Wed, 31 May 2023 16:10:58 +0800 Subject: [PATCH] Updates for running IPLD-ETH CI tests (#414) * random nits * geth - visibility of migration status * forward CERC_RUN_STATEDIFF to geth container * fix ipld-eth-server vars * fix fixturenet-eth-loaded stack * fixturenet geth genesis - include mergeNetsplitBlock * forward CERC_STATEDIFF_DB_GOOSE_MIN_VER to env file * add TAG_SUFFIX arg to lighthouse build intended to avoid sporadic failures when running lcli on github CI runners, likely related to non-portable builds --- app/data/compose/docker-compose-fixturenet-eth.yml | 2 +- app/data/compose/docker-compose-ipld-eth-server.yml | 8 +++----- app/data/config/fixturenet-eth/fixturenet-eth.env | 2 +- .../cerc-fixturenet-eth-geth/genesis/el/build_el.sh | 4 +++- .../cerc-fixturenet-eth-geth/run-el.sh | 10 +++++++--- .../genesis/cl/vars.env | 3 --- .../cerc-fixturenet-eth-lighthouse/scripts/status.sh | 12 +++++++++--- .../cerc-go-ethereum-foundry/deploy-local-network.sh | 2 +- app/data/container-build/cerc-lighthouse/Dockerfile | 3 ++- app/data/stacks/fixturenet-eth-loaded/stack.yml | 2 +- app/data/stacks/fixturenet-eth/README.md | 2 +- tests/fixturenet-eth/run-test.sh | 4 ++-- 12 files changed, 31 insertions(+), 23 deletions(-) diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml index 2a39966..530cd01 100644 --- a/app/data/compose/docker-compose-fixturenet-eth.yml +++ b/app/data/compose/docker-compose-fixturenet-eth.yml @@ -22,7 +22,7 @@ services: - SYS_PTRACE environment: CERC_REMOTE_DEBUG: "true" - CERC_RUN_STATEDIFF: "detect" + CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect} CERC_STATEDIFF_DB_NODE_ID: 1 CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} env_file: diff --git a/app/data/compose/docker-compose-ipld-eth-server.yml b/app/data/compose/docker-compose-ipld-eth-server.yml index e02f73c..ee787d5 100644 --- a/app/data/compose/docker-compose-ipld-eth-server.yml +++ b/app/data/compose/docker-compose-ipld-eth-server.yml @@ -7,11 +7,9 @@ services: condition: service_healthy image: cerc/ipld-eth-server:local environment: - IPLD_SERVER_GRAPHQL: "true" - IPLD_POSTGRAPHILEPATH: http://graphql:5000 - ETH_SERVER_HTTPPATH: 0.0.0.0:8081 - ETH_SERVER_GRAPHQL: "true" - ETH_SERVER_GRAPHQLPATH: 0.0.0.0:8082 + SERVER_HTTP_PATH: 0.0.0.0:8081 + SERVER_GRAPHQL: "true" + SERVER_GRAPHQLPATH: 0.0.0.0:8082 VDB_COMMAND: "serve" ETH_CHAIN_CONFIG: "/tmp/chain.json" DATABASE_NAME: cerc_testing diff --git a/app/data/config/fixturenet-eth/fixturenet-eth.env b/app/data/config/fixturenet-eth/fixturenet-eth.env index 1c214b5..06e90b6 100644 --- a/app/data/config/fixturenet-eth/fixturenet-eth.env +++ b/app/data/config/fixturenet-eth/fixturenet-eth.env @@ -17,7 +17,7 @@ CERC_STATEDIFF_DB_PORT=5432 CERC_STATEDIFF_DB_NAME="cerc_testing" CERC_STATEDIFF_DB_USER="vdbm" CERC_STATEDIFF_DB_PASSWORD="password" -CERC_STATEDIFF_DB_GOOSE_MIN_VER=23 +CERC_STATEDIFF_DB_GOOSE_MIN_VER=${CERC_STATEDIFF_DB_GOOSE_MIN_VER:-23} CERC_STATEDIFF_DB_LOG_STATEMENTS="false" CERC_GETH_VMODULE="statediff/*=5,rpc/*=5" diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/build_el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/build_el.sh index 76b43da..9ff77fa 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/build_el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/genesis/el/build_el.sh @@ -34,5 +34,7 @@ python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \ jq ".config.istanbulBlock=$istanbul_block" | \ jq ".config.berlinBlock=$berlin_block" | \ jq ".config.londonBlock=$london_block" | \ - jq ".config.mergeForkBlock=$merge_fork_block" > ../build/el/geth.json + jq ".config.mergeForkBlock=$merge_fork_block" | \ + jq ".config.mergeNetsplitBlock=$merge_fork_block" \ + > ../build/el/geth.json python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index edde1ba..9f6bf1f 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -64,8 +64,8 @@ else STATEDIFF_OPTS="" if [ "$CERC_RUN_STATEDIFF" == "true" ]; then ready=0 + echo "Waiting for statediff DB..." while [ $ready -eq 0 ]; do - echo "Waiting for statediff DB..." sleep 1 export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD" result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \ @@ -73,9 +73,13 @@ else -U "$CERC_STATEDIFF_DB_USER" \ -d "$CERC_STATEDIFF_DB_NAME" \ -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }') - if [ -n "$result" ] && [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then + if [ -n "$result" ]; then echo "DB ready..." - ready=1 + if [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then + ready=1 + else + echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)" + fi fi done STATEDIFF_OPTS="--statediff=true \ diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env b/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env index d68cbdc..b8c2734 100644 --- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env +++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env @@ -15,9 +15,6 @@ GENESIS_VALIDATOR_COUNT=${GENESIS_VALIDATOR_COUNT:-80} # Number of beacon_node instances that you intend to run BN_COUNT=${BN_COUNT:-2} -# Number of validator clients -VC_COUNT=${VC_COUNT:-$BN_COUNT} - # Number of seconds to delay to start genesis block. # If started by a script this can be 0, if starting by hand # use something like 180. diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh index f6d9d27..d9f33d1 100755 --- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh +++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh @@ -4,7 +4,14 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi MIN_BLOCK_NUM=${1:-${MIN_BLOCK_NUM:-3}} -STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge" "block number $MIN_BLOCK_NUM") +STATUSES=( + "geth to generate DAG" + "beacon phase0" + "beacon altair" + "beacon bellatrix pre-merge" + "beacon bellatrix merge" + "block number $MIN_BLOCK_NUM" +) STATUS=0 LIGHTHOUSE_BASE_URL=${LIGHTHOUSE_BASE_URL} @@ -36,7 +43,6 @@ MARKER="." function inc_status() { echo " done" - MARKEr="." STATUS=$((STATUS + 1)) if [ $STATUS -lt ${#STATUSES[@]} ]; then echo -n "Waiting for ${STATUSES[$STATUS]}..." @@ -55,7 +61,7 @@ while [ $STATUS -lt ${#STATUSES[@]} ]; do inc_status fi ;; - 1) + 1) result=`wget --no-check-certificate --quiet -O - "$LIGHTHOUSE_BASE_URL/eth/v2/beacon/blocks/head" | jq -r '.data.message.slot'` if [ ! -z "$result" ] && [ $result -gt 0 ]; then inc_status diff --git a/app/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh b/app/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh index e286bf3..ebc6ef2 100755 --- a/app/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh +++ b/app/data/container-build/cerc-go-ethereum-foundry/deploy-local-network.sh @@ -16,7 +16,7 @@ db-waitforsync=bool Should the statediff service start once geth has synced to rpc-port=port change RPC port (default: 8545) rpc-addr=address change RPC address (default: 127.0.0.1) chain-id=number change chain ID (default: 99) -extra-args=name extra args to pass to geth on startup +extra-args=name extra args to pass to geth on startup period=seconds use a block time instead of instamine accounts=number create multiple accounts (default: 1) address=address eth address to add to genesis diff --git a/app/data/container-build/cerc-lighthouse/Dockerfile b/app/data/container-build/cerc-lighthouse/Dockerfile index cab3af0..9391e93 100644 --- a/app/data/container-build/cerc-lighthouse/Dockerfile +++ b/app/data/container-build/cerc-lighthouse/Dockerfile @@ -1,4 +1,5 @@ -FROM sigp/lighthouse:v4.1.0-modern +ARG TAG_SUFFIX="-modern" +FROM sigp/lighthouse:v4.1.0${TAG_SUFFIX} RUN apt-get update; apt-get install bash netcat curl less jq -y; diff --git a/app/data/stacks/fixturenet-eth-loaded/stack.yml b/app/data/stacks/fixturenet-eth-loaded/stack.yml index 55a0524..676e95e 100644 --- a/app/data/stacks/fixturenet-eth-loaded/stack.yml +++ b/app/data/stacks/fixturenet-eth-loaded/stack.yml @@ -6,8 +6,8 @@ repos: - github.com/cerc-io/tx-spammer - github.com/cerc-io/ipld-eth-server - github.com/cerc-io/ipld-eth-db - - github.com/cerc-io/go-ethereum containers: + - cerc/go-ethereum - cerc/lighthouse - cerc/fixturenet-eth-geth - cerc/fixturenet-eth-lighthouse diff --git a/app/data/stacks/fixturenet-eth/README.md b/app/data/stacks/fixturenet-eth/README.md index 9f01417..ad7b6fa 100644 --- a/app/data/stacks/fixturenet-eth/README.md +++ b/app/data/stacks/fixturenet-eth/README.md @@ -1,6 +1,6 @@ # fixturenet-eth -Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator#user-mode)): +Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator (the installation of which is covered [here](https://github.com/cerc-io/stack-orchestrator)): ## Clone required repositories diff --git a/tests/fixturenet-eth/run-test.sh b/tests/fixturenet-eth/run-test.sh index 3fd4cd5..a7e0403 100755 --- a/tests/fixturenet-eth/run-test.sh +++ b/tests/fixturenet-eth/run-test.sh @@ -3,7 +3,7 @@ set -e if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -set -e + echo "Running stack-orchestrator Ethereum fixturenet test" # Bit of a hack, test the most recent package TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) @@ -15,7 +15,7 @@ reported_version_string=$( $TEST_TARGET_SO version ) echo "Version reported is: ${reported_version_string}" echo "Cloning repositories into: $CERC_REPO_BASE_DIR" $TEST_TARGET_SO --stack fixturenet-eth setup-repositories -$TEST_TARGET_SO --stack fixturenet-eth build-containers +$TEST_TARGET_SO --stack fixturenet-eth build-containers $TEST_TARGET_SO --stack fixturenet-eth deploy up # Verify that the fixturenet is up and running $TEST_TARGET_SO --stack fixturenet-eth deploy ps