update test workflow, fixtures

.gitea/workflows/test.yml

@@ -9,13 +9,22 @@ on:
 
 env:
   CANONICAL_VERSION: v5.0.4-alpha
+  ETH_TESTING_REF: v0.5.1
 
 jobs:
-  tests:
+  build:
-    name: Run unit and integration tests
+    name: Build Docker image
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Build docker image
+        run: docker build .
+
+  unit-test:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
       - uses: actions/setup-go@v3
         with:
           go-version-file: go.mod
@@ -25,15 +34,27 @@ jobs:
         with:
           repository: cerc-io/eth-testing
           path: ./fixtures
-          ref: v0.4.0
+          ref: ${{ env.ETH_TESTING_REF }}
-      - name: Build package
-        run: go build .
       - name: Run unit tests
         run: make test
 
-        # TODO test image
+  integration-test:
-      - name: Build docker image
+    name: Run integration tests
-        run: docker build . -t cerc/ipld-eths-state-snapshot:local
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+      - name: Install test fixtures
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/eth-testing
+          path: ./fixtures
+          ref: ${{ env.ETH_TESTING_REF }}
+      - name: Build package
+        run: go build .
       - name: Run DB container
         run: docker compose -f test/compose.yml up --wait
 
@@ -42,17 +63,19 @@ jobs:
       - name: Run basic integration test
         env:
           SNAPSHOT_MODE: postgres
-          SNAPSHOT_BLOCK_HEIGHT: 32
+          ETHDB_PATH: ./fixtures/chains/data/postmerge1/geth/chaindata
-          ETHDB_PATH: ./fixtures/chaindata/_data/small
+          ETH_GENESIS_BLOCK: 0x66ef6002e201cfdb23bd3f615fcf41e59d8382055e5a836f8d4c2af0d484647c
-          ETHDB_ANCIENT: ./fixtures/chaindata/_data/small/ancient
+          SNAPSHOT_BLOCK_HEIGHT: 170
-          ETH_GENESIS_BLOCK: "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
         run: |
           until
               ready_query='select max(version_id) from goose_db_version;'
               version=$(docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
                   psql -tA cerc_testing -U vdbm -c "$ready_query")
-              [[ "$version" -ge 20 ]]
+              [[ "$version" -ge 21 ]]
-          do sleep 1; done
+          do
+            echo "Waiting for ipld-eth-db..."
+            sleep 3
+          done
 
           ./ipld-eth-state-snapshot --config test/ci-config.toml stateSnapshot
 
@@ -63,11 +86,14 @@ jobs:
           }
           set -x
           [[ "$(count_results eth.header_cids)" = 1 ]]
-          [[ "$(count_results eth.state_cids)" = 5 ]]
+          [[ "$(count_results eth.state_cids)" = 264 ]]
-          [[ "$(count_results eth.storage_cids)" = 13 ]]
+          [[ "$(count_results eth.storage_cids)" = 371 ]]
 
   compliance-test:
-    name: Run compliance tests
+    name: Run compliance tests (disabled)
+    # Schema has been updated, so compliance tests are disabled until we have a meaningful way to
+    # compare to previous results.
+    if: false
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -82,7 +108,7 @@ jobs:
         with:
           repository: cerc-io/eth-testing
           path: ./fixtures
-          ref: v0.4.0
+          ref: ${{ env.ETH_TESTING_REF }}
       - name: Build current version
         working-directory: ./ipld-eth-state-snapshot
         run: go build -o ../snapshot-current .
@@ -102,15 +128,15 @@ jobs:
       - name: Compare snapshot output
         env:
           SNAPSHOT_BLOCK_HEIGHT: 200
-          ETHDB_PATH: ./fixtures/chaindata/_data/small2
+          ETHDB_PATH: ./fixtures/chains/data/premerge2/geth/chaindata
-          ETHDB_ANCIENT: ./fixtures/chaindata/_data/small2/ancient
+          ETHDB_ANCIENT: ./fixtures/chains/data/premerge2/geth/chaindata/ancient
           ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
         run: |
           until
               ready_query='select max(version_id) from goose_db_version;'
               version=$(docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
                   psql -tA cerc_testing -U vdbm -c "$ready_query")
-              [[ "$version" -ge 18 ]]
+              [[ "$version" -ge 21 ]]
           do sleep 1; done
 
           ./ipld-eth-state-snapshot/scripts/compare-snapshots.sh \

pkg/snapshot/service_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cerc-io/eth-testing/chaindata"
+	"github.com/cerc-io/eth-testing/chains"
 	"github.com/cerc-io/plugeth-statediff/indexer/models"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -205,7 +205,7 @@ func (expected selectiveData) verify(t *testing.T, data mocks.IndexerData) {
 	}
 }
 
-func doSnapshot(t *testing.T, chain *chaindata.Paths, params SnapshotParams) mocks.IndexerData {
+func doSnapshot(t *testing.T, chain *chains.Paths, params SnapshotParams) mocks.IndexerData {
 	chainDataPath, ancientDataPath := chain.ChainData, chain.Ancient
 	config := testConfig(chainDataPath, ancientDataPath)
 	edb, err := NewEthDB(config.Eth)
@@ -224,7 +224,7 @@ func doSnapshot(t *testing.T, chain *chaindata.Paths, params SnapshotParams) moc
 
 func doSnapshotWithRecovery(
 	t *testing.T,
-	chain *chaindata.Paths,
+	chain *chains.Paths,
 	params SnapshotParams,
 	failAfter uint,
 ) mocks.IndexerData {

test/ci-config.toml

@@ -20,4 +20,4 @@
     nodeID       = "test-node"
     networkID    = "test-network"
     chainID      = 1
-    genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
+    genesisBlock = ""

test/fixture_chain_A.go

@@ -3,12 +3,12 @@ package test
 import (
 	"sort"
 
-	"github.com/cerc-io/eth-testing/chaindata/small2"
+	"github.com/cerc-io/eth-testing/chains/premerge2"
 	"github.com/ethereum/go-ethereum/common"
 )
 
 func init() {
-	for _, path := range small2.Block1_StateNodeLeafKeys {
+	for _, path := range premerge2.Block1_StateNodeLeafKeys {
 		hex := common.BytesToHash(path).String()
 		ChainA_Block1_StateNodeLeafKeys = append(ChainA_Block1_StateNodeLeafKeys, hex)
 	}
@@ -19,7 +19,7 @@ func init() {
 }
 
 var (
-	ChainA = small2.ChainData
+	ChainA = premerge2.ChainData
 
 	ChainA_Block1_StateNodeLeafKeys []string
 	// ChainA_Block1_StateNodeLeafKeys = small2.Block1_StateNodeLeafKeys

test/fixture_chain_B.go

@@ -3,7 +3,7 @@ package test
 import (
 	"math/big"
 
-	"github.com/cerc-io/eth-testing/chaindata/small"
+	"github.com/cerc-io/eth-testing/chains/premerge1"
 	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
 	"github.com/cerc-io/plugeth-statediff/indexer/models"
 	"github.com/ethereum/go-ethereum/common"
@@ -12,7 +12,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-var ChainB = small.ChainData
+var ChainB = premerge1.ChainData
 
 var ChainB_block1_Header = types.Header{
 	ParentHash:  common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177"),