update ipld-eth-server

.github/workflows/test.yml

@@ -12,7 +12,8 @@ on:
       - ci-test
 
 env:
-  SO_VERSION: v1.1.0-87fffca-202404110321
+  SO_VERSION: v1.1.0-36d4969-202407091537
+  FIXTURENET_ETH_STACKS_REF: roysc/set-validator-creds # TODO: merge
 
 jobs:
   unit-tests:
@@ -40,11 +41,15 @@ jobs:
           go-version-file: 'go.mod'
           check-latest: true
 
-      - name: Install jq
-        env:
-          DEBIAN_FRONTEND: noninteractive
-        run: apt-get update && apt-get install -y jq
-      - name: Install Python
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: 3.11
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
@@ -55,24 +60,31 @@ jobs:
           ref: ${{ env.SO_VERSION }}
           path: ./stack-orchestrator
       - run: pip install ./stack-orchestrator
+      - name: Clone fixturenet stack repo
+        uses: actions/checkout@v4
+        with:
+          repository: cerc-io/fixturenet-eth-stacks
+          ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
+          path: ./fixturenet-eth-stacks
+          progress: false
 
       - name: Run testnet stack
         env:
           CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
-        run: ./scripts/integration-setup.sh
+        run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
       - name: Run contract deployer
         run: |
           docker compose -f test/compose-deployer.yml up --wait --quiet-pull
       - name: Wait for testnet
         run: |
           # Start validator at current head, but not before Merge (block 1 on test chain)
-          echo "Waiting for chain head to progress..."
           while
+              echo "Waiting for chain head to progress..."
               height=$(./scripts/get-block-number.sh $ETH_HTTP_PATH)
-              [[ "$height" < 2 ]];
+              [[ "$height" -lt 2 ]];
           do sleep 5; done
           echo "Chain has reached block $height"
           echo VALIDATE_FROM_BLOCK=$height >> "$GITHUB_ENV"
       - name: Run tests
         run: |
-          go test ./integration/... -v -timeout=20m
+          go test -v -p 1 ./integration/... -timeout=20m

go.mod

@@ -5,8 +5,8 @@ go 1.21
 require (
 	github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha
 	github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha
-	github.com/cerc-io/ipld-eth-statedb v0.1.0
-	github.com/cerc-io/plugeth-statediff v0.2.1
+	github.com/cerc-io/ipld-eth-statedb v0.1.1
+	github.com/cerc-io/plugeth-statediff v0.3.2
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/holiman/uint256 v1.2.4
 	github.com/jmoiron/sqlx v1.3.5
@@ -33,7 +33,7 @@ require (
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha // indirect
-	github.com/cerc-io/eth-iterator-utils v0.2.0 // indirect
+	github.com/cerc-io/eth-iterator-utils v0.3.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.10.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
@@ -272,3 +272,5 @@ require (
 	lukechampine.com/blake3 v1.2.2 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
+
+replace github.com/cerc-io/ipld-eth-server/v5 => github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha.0.20240719143742-685f1aa4dabc

go.sum

@@ -81,18 +81,18 @@ github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2
 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha h1:gpEk3BQQhnkilXxP12q77LSiIAvXfTdQ+4xxIv/JWiI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha/go.mod h1:xmazdaZ/CUaXt52Mvd0KU7dcbJM3glvJzA2mEpVMgbM=
-github.com/cerc-io/eth-iterator-utils v0.2.0 h1:wikAfWZ0fAqLqUy/Ud/a1n9p/arVeKG8P8tRjZE7oBg=
-github.com/cerc-io/eth-iterator-utils v0.2.0/go.mod h1:wDUJvwKDSOdqTIyeG+yXJ2ckzc9f2Fem614fV61DBcg=
-github.com/cerc-io/eth-testing v0.4.0 h1:ivGbXnEqlXMt/3m3jbsPJaVT7ZDTenFQWCryt1Rd/Jk=
-github.com/cerc-io/eth-testing v0.4.0/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
+github.com/cerc-io/eth-iterator-utils v0.3.1 h1:h4Bp0+fUiwkyug1uCEO2LZr2qxoW1yKszV2EO/2CDB0=
+github.com/cerc-io/eth-iterator-utils v0.3.1/go.mod h1:UNrjsP5bApZkqqqfU7nmnPN/dIIo9GOUUD79tmoX/s4=
+github.com/cerc-io/eth-testing v0.5.1 h1:xxcQf9ymJS0911yWIrUiGvCvqfvEjYmHvhBJkCD/whs=
+github.com/cerc-io/eth-testing v0.5.1/go.mod h1:p86je2PjSM7u8Qd7rMIG/Zw+tQlBoS5Emkh1ECnC5t0=
 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha h1:+XhYHvzC3zFIvcWEb466SNDfeLrvcW3xe/d0cbVVVRA=
 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha/go.mod h1:w5g07b6Uz+c6r/ySml58TEOJdUYAibYYF05H5ULVv2I=
-github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha h1:ZDG5U456ttTbuY8VBJqzGueyppBCiNOAGg/UXlEomTw=
-github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha/go.mod h1:XlVPHVsF6u8kg1Ci8g8/yIzP2nbrRFsOXTST71g8GlE=
-github.com/cerc-io/ipld-eth-statedb v0.1.0 h1:K2Xy8hgmkSRmHaqmw0CZf9hWX0r8/r0563O7osmyVBE=
-github.com/cerc-io/ipld-eth-statedb v0.1.0/go.mod h1:Lo4TKVs7bTGr2b7L9ccmo/EMV96/nGE7T/GYliuOHlM=
-github.com/cerc-io/plugeth-statediff v0.2.1 h1:zUbkazJW0omFGg7z/9MJnttox4VRurLW1pFytDlRQjM=
-github.com/cerc-io/plugeth-statediff v0.2.1/go.mod h1:n85L2n8Q3bQVlAVFnyk2soCLJW+YFKGe3DVRauEaS2s=
+github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha.0.20240719143742-685f1aa4dabc h1:seFt4s1kGqHwEA+sfz0KNExD3TT8B32xZ3Vh4arxGqI=
+github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha.0.20240719143742-685f1aa4dabc/go.mod h1:0NXLwk9KegKqDFalOItzW79whBE5GPhg5gdu6FIBWR4=
+github.com/cerc-io/ipld-eth-statedb v0.1.1 h1:QvhdO9jZqQu+NmiilW4Ef4N3qbdN4+FRHwVVbHg7Q/w=
+github.com/cerc-io/ipld-eth-statedb v0.1.1/go.mod h1:+BF5xGXodtUFlNZ9W3JYsUHRmPyso+td8IgB+x/bYjM=
+github.com/cerc-io/plugeth-statediff v0.3.2 h1:GuOUqDT6nJRCikyaNxDI2pA7TRnOTWOcyJGlJtwSzHY=
+github.com/cerc-io/plugeth-statediff v0.3.2/go.mod h1:r6Mzc6k4V9KD+iN9AXa/LmmRISDsQnnIwKzZMFjJ+eE=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

pkg/validator/ref_integrity_test.go

@@ -6,6 +6,7 @@ import (
 	"math/big"
 	"testing"
 
+	indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
 	helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -67,7 +68,7 @@ var _ = Describe("referential integrity", Ordered, func() {
 
 		db = SetupDB()
 	})
-	AfterAll(func() { helpers.ClearDB(db) })
+	AfterAll(func() { Expect(indexer_helpers.ClearSqlxDB(db)).ToNot(HaveOccurred()) })
 
 	BeforeEach(func() { tx = db.MustBegin() })
 	AfterEach(func() {

pkg/validator/validator.go

@@ -334,7 +334,7 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate
 		evm.Reset(core.NewEVMTxContext(msg), statedb)
 		// Apply the transaction to the current state (included in the env).
 		if _, err := core.ApplyMessage(evm, msg, &gp); err != nil {
-			return nil, fmt.Errorf("transaction %#x failed: %w", tx.Hash(), err)
+			return nil, fmt.Errorf("error applying tx %#x: %w", tx.Hash(), err)
 		}
 
 		if config.IsByzantium(block.Number()) {
@@ -344,6 +344,14 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate
 		}
 	}
 
+	// Withdrawals processing.
+	for _, w := range block.Withdrawals() {
+		// Convert amount from gwei to wei.
+		amount := new(uint256.Int).SetUint64(w.Amount)
+		amount = amount.Mul(amount, uint256.NewInt(params.GWei))
+		statedb.AddBalance(w.Address, amount)
+	}
+
 	if config.Ethash != nil {
 		accumulateRewards(config, statedb, block.Header(), block.Uncles())
 	}

pkg/validator/validator_test.go

@@ -6,12 +6,14 @@ import (
 	"testing"
 
 	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
+	indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
 	helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
 	sdtypes "github.com/cerc-io/plugeth-statediff/types"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/jmoiron/sqlx"
+
 	// import server helpers for non-canonical chain data
 	server_mocks "github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
 
@@ -98,7 +100,9 @@ func setupStateValidator(t *testing.T) *sqlx.DB {
 
 	db := SetupDB()
 	t.Cleanup(func() {
-		helpers.ClearDB(db)
+		if err := indexer_helpers.ClearSqlxDB(db); err != nil {
+			t.Fatal(err)
+		}
 	})
 	return db
 }

scripts/get-block-number.sh

@@ -6,6 +6,7 @@ geth_endpoint="${1:-$ETH_HTTP_PATH}"
 
 latest_block_hex=$(curl -s $geth_endpoint -X POST -H "Content-Type: application/json" \
     --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":42}' | \
-    jq -r .result)
+    python3 -c 'import json, sys; print(int(json.load(sys.stdin)["result"], 16))' \
+)
 
 printf "%d" $latest_block_hex

scripts/run-test-stack.sh

@@ -1,39 +1,41 @@
 #!/bin/bash
-# Builds and deploys a stack with only what we need.
-# This script assumes we are running in the project root.
 
-set -e
+set -ex
 
-laconic_so="${LACONIC_SO:-laconic-so} --stack $(readlink -f test) --verbose"
+stack_dir=$(readlink -f "$1")
+[[ -d "$stack_dir" ]]
 
 CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
+# By default assume we are running in the project root.
+export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
 
+laconic_so="laconic-so --verbose --stack $stack_dir"
+
+# Don't run geth/plugeth in the debugger, it will swallow error backtraces
+echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
+# Passing this lets us run eth_call forwarding tests without running ipld-eth-db
+echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env
+
+
+if [[ -z $SKIP_BUILD ]]; then
   # Prevent conflicting tty output
   export BUILDKIT_PROGRESS=plain
 
-# Point stack-orchestrator to the multi-project root
-export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}"
-# v5 migrations only go up to version 20
-echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=20 >> $CONFIG_DIR/stack.env
-# don't run plugeth in the debugger
-echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
-
-set -x
-
-if [[ -z $SKIP_BUILD ]]; then
-    $laconic_so setup-repositories \
-        --exclude git.vdb.to/cerc-io/ipld-eth-server
-    # Assume the tested image has been built separately
-    $laconic_so build-containers \
-        --exclude cerc/ipld-eth-server
+  $laconic_so setup-repositories
+  $laconic_so build-containers
 fi
 
-$laconic_so deploy \
+if ! $laconic_so deploy \
   --env-file $CONFIG_DIR/stack.env \
   --cluster test up
+then
+  $laconic_so deploy --cluster test logs
+  exit 1
+fi
 
 set +x
 
+# Get IPv4 endpoint of geth and bootnode file server
 bootnode_endpoint=localhost:$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1 | cut -d':' -f2)
 geth_endpoint=localhost:$(docker port test-fixturenet-eth-geth-1-1 8545 | head -1 | cut -d':' -f2)
 

test/compose-db.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"

test/stack.yml

@@ -1,23 +0,0 @@
-version: "1.2"
-name: fixturenet-plugeth-tx
-description: "Plugeth Ethereum Fixturenet for testing ipld-eth-db-validator"
-repos:
-  - git.vdb.to/cerc-io/plugeth@statediff
-  - git.vdb.to/cerc-io/plugeth-statediff
-  - git.vdb.to/cerc-io/lighthouse
-  - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha
-  - git.vdb.to/cerc-io/tx-spammer
-containers:
-  - cerc/plugeth-statediff
-  - cerc/plugeth
-  - cerc/fixturenet-eth-genesis
-  - cerc/fixturenet-plugeth-plugeth
-  - cerc/lighthouse
-  - cerc/lighthouse-cli
-  - cerc/fixturenet-eth-lighthouse
-  - cerc/ipld-eth-db
-  - cerc/tx-spammer
-pods:
-  - fixturenet-plugeth
-  - ipld-eth-db
-  - tx-spammer