.github/workflows/test.yml

@@ -12,8 +12,7 @@ on:
       - ci-test
 
 env:
-  SO_VERSION: v1.1.0-36d4969-202407091537
-  FIXTURENET_ETH_STACKS_REF: roysc/set-validator-creds # TODO: merge
+  SO_VERSION: v1.1.0-87fffca-202404110321
 
 jobs:
   unit-tests:
@@ -41,15 +40,11 @@ jobs:
           go-version-file: 'go.mod'
           check-latest: true
 
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: 3.11
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+      - name: Install jq
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: apt-get update && apt-get install -y jq
+      - name: Install Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.11'
@@ -60,31 +55,24 @@ jobs:
           ref: ${{ env.SO_VERSION }}
           path: ./stack-orchestrator
       - run: pip install ./stack-orchestrator
-      - name: Clone fixturenet stack repo
-        uses: actions/checkout@v4
-        with:
-          repository: cerc-io/fixturenet-eth-stacks
-          ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
-          path: ./fixturenet-eth-stacks
-          progress: false
 
       - name: Run testnet stack
         env:
           CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
-        run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
+        run: ./scripts/integration-setup.sh
       - name: Run contract deployer
         run: |
           docker compose -f test/compose-deployer.yml up --wait --quiet-pull
       - name: Wait for testnet
         run: |
           # Start validator at current head, but not before Merge (block 1 on test chain)
+          echo "Waiting for chain head to progress..."
           while
-              echo "Waiting for chain head to progress..."
               height=$(./scripts/get-block-number.sh $ETH_HTTP_PATH)
-              [[ "$height" -lt 2 ]];
+              [[ "$height" < 2 ]];
           do sleep 5; done
           echo "Chain has reached block $height"
           echo VALIDATE_FROM_BLOCK=$height >> "$GITHUB_ENV"
       - name: Run tests
         run: |
-          go test -v -p 1 ./integration/... -timeout=20m
+          go test ./integration/... -v -timeout=20m

go.mod

@@ -5,8 +5,8 @@ go 1.21
 require (
 	github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha
 	github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha
-	github.com/cerc-io/ipld-eth-statedb v0.1.1
-	github.com/cerc-io/plugeth-statediff v0.3.2
+	github.com/cerc-io/ipld-eth-statedb v0.1.0
+	github.com/cerc-io/plugeth-statediff v0.2.1
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/holiman/uint256 v1.2.4
 	github.com/jmoiron/sqlx v1.3.5
@@ -33,7 +33,7 @@ require (
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha // indirect
-	github.com/cerc-io/eth-iterator-utils v0.3.1 // indirect
+	github.com/cerc-io/eth-iterator-utils v0.2.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.10.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
@@ -272,5 +272,3 @@ require (
 	lukechampine.com/blake3 v1.2.2 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
-
-replace github.com/cerc-io/ipld-eth-server/v5 => github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha.0.20240719143742-685f1aa4dabc

go.sum

@@ -81,18 +81,18 @@ github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2
 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha h1:gpEk3BQQhnkilXxP12q77LSiIAvXfTdQ+4xxIv/JWiI=
 github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha/go.mod h1:xmazdaZ/CUaXt52Mvd0KU7dcbJM3glvJzA2mEpVMgbM=
-github.com/cerc-io/eth-iterator-utils v0.3.1 h1:h4Bp0+fUiwkyug1uCEO2LZr2qxoW1yKszV2EO/2CDB0=
-github.com/cerc-io/eth-iterator-utils v0.3.1/go.mod h1:UNrjsP5bApZkqqqfU7nmnPN/dIIo9GOUUD79tmoX/s4=
-github.com/cerc-io/eth-testing v0.5.1 h1:xxcQf9ymJS0911yWIrUiGvCvqfvEjYmHvhBJkCD/whs=
-github.com/cerc-io/eth-testing v0.5.1/go.mod h1:p86je2PjSM7u8Qd7rMIG/Zw+tQlBoS5Emkh1ECnC5t0=
+github.com/cerc-io/eth-iterator-utils v0.2.0 h1:wikAfWZ0fAqLqUy/Ud/a1n9p/arVeKG8P8tRjZE7oBg=
+github.com/cerc-io/eth-iterator-utils v0.2.0/go.mod h1:wDUJvwKDSOdqTIyeG+yXJ2ckzc9f2Fem614fV61DBcg=
+github.com/cerc-io/eth-testing v0.4.0 h1:ivGbXnEqlXMt/3m3jbsPJaVT7ZDTenFQWCryt1Rd/Jk=
+github.com/cerc-io/eth-testing v0.4.0/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha h1:+XhYHvzC3zFIvcWEb466SNDfeLrvcW3xe/d0cbVVVRA=
 github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha/go.mod h1:w5g07b6Uz+c6r/ySml58TEOJdUYAibYYF05H5ULVv2I=
-github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha.0.20240719143742-685f1aa4dabc h1:seFt4s1kGqHwEA+sfz0KNExD3TT8B32xZ3Vh4arxGqI=
-github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha.0.20240719143742-685f1aa4dabc/go.mod h1:0NXLwk9KegKqDFalOItzW79whBE5GPhg5gdu6FIBWR4=
-github.com/cerc-io/ipld-eth-statedb v0.1.1 h1:QvhdO9jZqQu+NmiilW4Ef4N3qbdN4+FRHwVVbHg7Q/w=
-github.com/cerc-io/ipld-eth-statedb v0.1.1/go.mod h1:+BF5xGXodtUFlNZ9W3JYsUHRmPyso+td8IgB+x/bYjM=
-github.com/cerc-io/plugeth-statediff v0.3.2 h1:GuOUqDT6nJRCikyaNxDI2pA7TRnOTWOcyJGlJtwSzHY=
-github.com/cerc-io/plugeth-statediff v0.3.2/go.mod h1:r6Mzc6k4V9KD+iN9AXa/LmmRISDsQnnIwKzZMFjJ+eE=
+github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha h1:ZDG5U456ttTbuY8VBJqzGueyppBCiNOAGg/UXlEomTw=
+github.com/cerc-io/ipld-eth-server/v5 v5.2.0-alpha/go.mod h1:XlVPHVsF6u8kg1Ci8g8/yIzP2nbrRFsOXTST71g8GlE=
+github.com/cerc-io/ipld-eth-statedb v0.1.0 h1:K2Xy8hgmkSRmHaqmw0CZf9hWX0r8/r0563O7osmyVBE=
+github.com/cerc-io/ipld-eth-statedb v0.1.0/go.mod h1:Lo4TKVs7bTGr2b7L9ccmo/EMV96/nGE7T/GYliuOHlM=
+github.com/cerc-io/plugeth-statediff v0.2.1 h1:zUbkazJW0omFGg7z/9MJnttox4VRurLW1pFytDlRQjM=
+github.com/cerc-io/plugeth-statediff v0.2.1/go.mod h1:n85L2n8Q3bQVlAVFnyk2soCLJW+YFKGe3DVRauEaS2s=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

pkg/validator/ref_integrity_test.go

@@ -6,7 +6,6 @@ import (
 	"math/big"
 	"testing"
 
-	indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
 	helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -68,7 +67,7 @@ var _ = Describe("referential integrity", Ordered, func() {
 
 		db = SetupDB()
 	})
-	AfterAll(func() { Expect(indexer_helpers.ClearSqlxDB(db)).ToNot(HaveOccurred()) })
+	AfterAll(func() { helpers.ClearDB(db) })
 
 	BeforeEach(func() { tx = db.MustBegin() })
 	AfterEach(func() {

pkg/validator/validator.go

@@ -334,7 +334,7 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate
 		evm.Reset(core.NewEVMTxContext(msg), statedb)
 		// Apply the transaction to the current state (included in the env).
 		if _, err := core.ApplyMessage(evm, msg, &gp); err != nil {
-			return nil, fmt.Errorf("error applying tx %#x: %w", tx.Hash(), err)
+			return nil, fmt.Errorf("transaction %#x failed: %w", tx.Hash(), err)
 		}
 
 		if config.IsByzantium(block.Number()) {
@@ -344,14 +344,6 @@ func applyTransactions(block *types.Block, backend *ipldeth.Backend) (*ipldstate
 		}
 	}
 
-	// Withdrawals processing.
-	for _, w := range block.Withdrawals() {
-		// Convert amount from gwei to wei.
-		amount := new(uint256.Int).SetUint64(w.Amount)
-		amount = amount.Mul(amount, uint256.NewInt(params.GWei))
-		statedb.AddBalance(w.Address, amount)
-	}
-
 	if config.Ethash != nil {
 		accumulateRewards(config, statedb, block.Header(), block.Uncles())
 	}

pkg/validator/validator_test.go

@@ -6,14 +6,12 @@ import (
 	"testing"
 
 	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
-	indexer_helpers "github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
 	helpers "github.com/cerc-io/plugeth-statediff/test_helpers"
 	sdtypes "github.com/cerc-io/plugeth-statediff/types"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/jmoiron/sqlx"
-
 	// import server helpers for non-canonical chain data
 	server_mocks "github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
 
@@ -100,9 +98,7 @@ func setupStateValidator(t *testing.T) *sqlx.DB {
 
 	db := SetupDB()
 	t.Cleanup(func() {
-		if err := indexer_helpers.ClearSqlxDB(db); err != nil {
-			t.Fatal(err)
-		}
+		helpers.ClearDB(db)
 	})
 	return db
 }

scripts/get-block-number.sh

@@ -6,7 +6,6 @@ geth_endpoint="${1:-$ETH_HTTP_PATH}"
 
 latest_block_hex=$(curl -s $geth_endpoint -X POST -H "Content-Type: application/json" \
     --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":42}' | \
-    python3 -c 'import json, sys; print(int(json.load(sys.stdin)["result"], 16))' \
-)
+    jq -r .result)
 
 printf "%d" $latest_block_hex

scripts/integration-setup.sh

@@ -1,41 +1,39 @@
 #!/bin/bash
+# Builds and deploys a stack with only what we need.
+# This script assumes we are running in the project root.
 
-set -ex
+set -e
 
-stack_dir=$(readlink -f "$1")
-[[ -d "$stack_dir" ]]
+laconic_so="${LACONIC_SO:-laconic-so} --stack $(readlink -f test) --verbose"
 
 CONFIG_DIR=$(readlink -f "${CONFIG_DIR:-$(mktemp -d)}")
-# By default assume we are running in the project root.
-export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-$(git rev-parse --show-toplevel)/..}"
 
-laconic_so="laconic-so --verbose --stack $stack_dir"
+# Prevent conflicting tty output
+export BUILDKIT_PROGRESS=plain
 
-# Don't run geth/plugeth in the debugger, it will swallow error backtraces
+# Point stack-orchestrator to the multi-project root
+export CERC_REPO_BASE_DIR="${CERC_REPO_BASE_DIR:-..}"
+# v5 migrations only go up to version 20
+echo CERC_STATEDIFF_DB_GOOSE_MIN_VER=20 >> $CONFIG_DIR/stack.env
+# don't run plugeth in the debugger
 echo CERC_REMOTE_DEBUG=false >> $CONFIG_DIR/stack.env
-# Passing this lets us run eth_call forwarding tests without running ipld-eth-db
-echo CERC_RUN_STATEDIFF=${CERC_RUN_STATEDIFF:-true} >> $CONFIG_DIR/stack.env
 
+set -x
 
 if [[ -z $SKIP_BUILD ]]; then
-  # Prevent conflicting tty output
-  export BUILDKIT_PROGRESS=plain
-
-  $laconic_so setup-repositories
-  $laconic_so build-containers
+    $laconic_so setup-repositories \
+        --exclude git.vdb.to/cerc-io/ipld-eth-server
+    # Assume the tested image has been built separately
+    $laconic_so build-containers \
+        --exclude cerc/ipld-eth-server
 fi
 
-if ! $laconic_so deploy \
-  --env-file $CONFIG_DIR/stack.env \
-  --cluster test up
-then
-  $laconic_so deploy --cluster test logs
-  exit 1
-fi
+$laconic_so deploy \
+    --env-file $CONFIG_DIR/stack.env \
+    --cluster test up
 
 set +x
 
-# Get IPv4 endpoint of geth and bootnode file server
 bootnode_endpoint=localhost:$(docker port test-fixturenet-eth-bootnode-geth-1 9898 | head -1 | cut -d':' -f2)
 geth_endpoint=localhost:$(docker port test-fixturenet-eth-geth-1-1 8545 | head -1 | cut -d':' -f2)
 

test/compose-db.yml

@@ -5,7 +5,7 @@ services:
     restart: on-failure
     depends_on:
       - ipld-eth-db
-    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.2.1-alpha
     environment:
       DATABASE_USER: "vdbm"
       DATABASE_NAME: "cerc_testing"

test/stack.yml

@@ -0,0 +1,23 @@
+version: "1.2"
+name: fixturenet-plugeth-tx
+description: "Plugeth Ethereum Fixturenet for testing ipld-eth-db-validator"
+repos:
+  - git.vdb.to/cerc-io/plugeth@statediff
+  - git.vdb.to/cerc-io/plugeth-statediff
+  - git.vdb.to/cerc-io/lighthouse
+  - git.vdb.to/cerc-io/ipld-eth-db@v5.2.1-alpha
+  - git.vdb.to/cerc-io/tx-spammer
+containers:
+  - cerc/plugeth-statediff
+  - cerc/plugeth
+  - cerc/fixturenet-eth-genesis
+  - cerc/fixturenet-plugeth-plugeth
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/fixturenet-eth-lighthouse
+  - cerc/ipld-eth-db
+  - cerc/tx-spammer
+pods:
+  - fixturenet-plugeth
+  - ipld-eth-db
+  - tx-spammer