From c4b397de2cfe6abdf99099c410bebb0c9d704aa7 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 31 Jan 2022 12:55:55 -0600 Subject: [PATCH] fix tests in CI --- .github/workflows/on-pr.yml | 9 +- docker-compose.yml | 18 +++- .../database/file/indexer_legacy_test.go | 5 +- .../indexer/database/file/indexer_test.go | 34 ++++--- .../file/mainnet_tests/indexer_test.go | 5 +- statediff/indexer/database/sql/indexer.go | 3 + .../sql/mainnet_tests/indexer_test.go | 17 +++- .../database/sql/pgx_indexer_legacy_test.go | 9 +- .../indexer/database/sql/pgx_indexer_test.go | 82 +++++----------- .../indexer/database/sql/postgres/config.go | 20 +++- .../indexer/database/sql/postgres/errors.go | 5 - .../indexer/database/sql/postgres/pgx_test.go | 17 +--- .../database/sql/postgres/sqlx_test.go | 18 +--- .../database/sql/postgres/test_helpers.go | 8 +- .../database/sql/sqlx_indexer_legacy_test.go | 10 +- .../indexer/database/sql/sqlx_indexer_test.go | 93 +++++++------------ statediff/indexer/mocks/test_data.go | 6 +- 17 files changed, 161 insertions(+), 198 deletions(-) diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index af20fb410..8c1a1e974 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -58,9 +58,12 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Start database - run: docker-compose -f docker-compose.yml up -d ipld-eth-db + - name: Start database v2 + run: docker-compose -f docker-compose.yml up -d ipld-eth-db-v2 + + - name: Start database v3 + run: docker-compose -f docker-compose.yml up -d ipld-eth-db-v3 - name: Run unit tests run: - make statedifftest \ No newline at end of file + make statedifftest diff --git a/docker-compose.yml b/docker-compose.yml index 32dc46147..001c2650a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,21 @@ version: '3.2' services: - ipld-eth-db: + ipld-eth-db-v2: restart: always - image: vulcanize/ipld-eth-db:v0.3.1 + image: vulcanize/ipld-eth-db:v2.0.0 + environment: + POSTGRES_USER: "vdbm" + POSTGRES_DB: "vulcanize_testing_v2" + POSTGRES_PASSWORD: "password" + volumes: + - geth_node:/var/lib/postgresql/data + ports: + - "127.0.0.1:5432:5432" + + ipld-eth-db-v3: + restart: always + image: vulcanize/ipld-eth-db:v3.0.6 environment: POSTGRES_USER: "vdbm" POSTGRES_DB: "vulcanize_testing_v3" @@ -11,7 +23,7 @@ services: volumes: - geth_node:/var/lib/postgresql/data ports: - - "127.0.0.1:5432:5432" + - "127.0.0.1:5433:5432" volumes: geth_node: diff --git a/statediff/indexer/database/file/indexer_legacy_test.go b/statediff/indexer/database/file/indexer_legacy_test.go index 56bca2683..95d968a22 100644 --- a/statediff/indexer/database/file/indexer_legacy_test.go +++ b/statediff/indexer/database/file/indexer_legacy_test.go @@ -52,7 +52,8 @@ func setupLegacy(t *testing.T) { ind, err := file.NewStateDiffIndexer(context.Background(), legacyData.Config, file.TestConfig) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + var headerID int64 + tx, headerID, err = ind.PushBlock( mockLegacyBlock, legacyData.MockReceipts, legacyData.MockBlock.Difficulty()) @@ -67,7 +68,7 @@ func setupLegacy(t *testing.T) { } }() for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String()) + err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String(), headerID) require.NoError(t, err) } diff --git a/statediff/indexer/database/file/indexer_test.go b/statediff/indexer/database/file/indexer_test.go index 52bcfea68..d842f53d3 100644 --- a/statediff/indexer/database/file/indexer_test.go +++ b/statediff/indexer/database/file/indexer_test.go @@ -24,6 +24,9 @@ import ( "os" "testing" + sharedModels "github.com/ethereum/go-ethereum/statediff/indexer/models/shared" + v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/shared" @@ -170,7 +173,8 @@ func setup(t *testing.T) { ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.TestConfig) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + var headerID int64 + tx, headerID, err = ind.PushBlock( mockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty()) @@ -186,7 +190,7 @@ func setup(t *testing.T) { } }() for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, mockBlock.Hash().String()) + err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), headerID) require.NoError(t, err) } @@ -330,7 +334,7 @@ func TestFileIndexer(t *testing.T) { if txRes.Value != transactions[3].Value().String() { t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value) } - accessListElementModels := make([]v2.AccessListElementModel, 0) + accessListElementModels := make([]v3Models.AccessListElementModel, 0) pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC` err = sqlxdb.Select(&accessListElementModels, pgStr, c) if err != nil { @@ -339,11 +343,11 @@ func TestFileIndexer(t *testing.T) { if len(accessListElementModels) != 2 { t.Fatalf("expected two access list entries, got %d", len(accessListElementModels)) } - model1 := v2.AccessListElementModel{ + model1 := v3Models.AccessListElementModel{ Index: accessListElementModels[0].Index, Address: accessListElementModels[0].Address, } - model2 := v2.AccessListElementModel{ + model2 := v3Models.AccessListElementModel{ Index: accessListElementModels[1].Index, Address: accessListElementModels[1].Address, StorageKeys: accessListElementModels[1].StorageKeys, @@ -446,7 +450,7 @@ func TestFileIndexer(t *testing.T) { expectTrue(t, test_helpers.ListContainsString(rcts, rct5CID.String())) for idx, c := range rcts { - result := make([]v3.IPLDModel, 0) + result := make([]sharedModels.IPLDModel, 0) pgStr = `SELECT data FROM eth.receipt_cids INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key) @@ -530,7 +534,7 @@ func TestFileIndexer(t *testing.T) { defer tearDown(t) // check that state nodes were properly indexed and published - stateNodes := make([]v2.StateNodeModel, 0) + stateNodes := make([]v3Models.StateNodeModel, 0) pgStr := `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) WHERE header_cids.block_number = $1 AND node_type != 3` @@ -552,7 +556,7 @@ func TestFileIndexer(t *testing.T) { t.Fatal(err) } pgStr = `SELECT * from eth.state_accounts WHERE header_id = $1 AND state_path = $2` - var account v2.StateAccountModel + var account v3Models.StateAccountModel err = sqlxdb.Get(&account, pgStr, stateNode.HeaderID, stateNode.Path) if err != nil { t.Fatal(err) @@ -562,7 +566,7 @@ func TestFileIndexer(t *testing.T) { test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex()) test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'}) test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode) - test_helpers.ExpectEqual(t, account, v2.StateAccountModel{ + test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{ HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "0", @@ -576,7 +580,7 @@ func TestFileIndexer(t *testing.T) { test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex()) test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'}) test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode) - test_helpers.ExpectEqual(t, account, v2.StateAccountModel{ + test_helpers.ExpectEqual(t, account, v3Models.StateAccountModel{ HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "1000", @@ -588,7 +592,7 @@ func TestFileIndexer(t *testing.T) { } // check that Removed state nodes were properly indexed and published - stateNodes = make([]v2.StateNodeModel, 0) + stateNodes = make([]v3Models.StateNodeModel, 0) pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) WHERE header_cids.block_number = $1 AND node_type = 3` @@ -621,7 +625,7 @@ func TestFileIndexer(t *testing.T) { defer tearDown(t) // check that storage nodes were properly indexed - storageNodes := make([]v2.StorageNodeWithStateKeyModel, 0) + storageNodes := make([]v3Models.StorageNodeWithStateKeyModel, 0) pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path FROM eth.storage_cids, eth.state_cids, eth.header_cids WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) @@ -633,7 +637,7 @@ func TestFileIndexer(t *testing.T) { t.Fatal(err) } test_helpers.ExpectEqual(t, len(storageNodes), 1) - test_helpers.ExpectEqual(t, storageNodes[0], v2.StorageNodeWithStateKeyModel{ + test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{ CID: storageCID.String(), NodeType: 2, StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), @@ -654,7 +658,7 @@ func TestFileIndexer(t *testing.T) { test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode) // check that Removed storage nodes were properly indexed - storageNodes = make([]v2.StorageNodeWithStateKeyModel, 0) + storageNodes = make([]v3Models.StorageNodeWithStateKeyModel, 0) pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path FROM eth.storage_cids, eth.state_cids, eth.header_cids WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) @@ -666,7 +670,7 @@ func TestFileIndexer(t *testing.T) { t.Fatal(err) } test_helpers.ExpectEqual(t, len(storageNodes), 1) - test_helpers.ExpectEqual(t, storageNodes[0], v2.StorageNodeWithStateKeyModel{ + test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{ CID: shared.RemovedNodeStorageCID, NodeType: 3, StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), diff --git a/statediff/indexer/database/file/mainnet_tests/indexer_test.go b/statediff/indexer/database/file/mainnet_tests/indexer_test.go index b297b82d1..59104973c 100644 --- a/statediff/indexer/database/file/mainnet_tests/indexer_test.go +++ b/statediff/indexer/database/file/mainnet_tests/indexer_test.go @@ -88,7 +88,8 @@ func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) { ind, err := file.NewStateDiffIndexer(context.Background(), chainConf, file.TestConfig) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + var headerID int64 + tx, headerID, err = ind.PushBlock( testBlock, testReceipts, testBlock.Difficulty()) @@ -103,7 +104,7 @@ func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) { } }() for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, testBlock.Hash().String()) + err = ind.PushStateNode(tx, node, testBlock.Hash().String(), headerID) require.NoError(t, err) } diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go index 010a675bb..07f4e1ff4 100644 --- a/statediff/indexer/database/sql/indexer.go +++ b/statediff/indexer/database/sql/indexer.go @@ -629,6 +629,9 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt MhKey: stateMhKey, NodeType: stateNode.NodeType.Int(), }) + if err != nil { + return err + } if err := sdi.newDBWriter.InsertStateCID(tx.newDBTx, &v3Models.StateNodeModel{ HeaderID: headerHash, Path: stateNode.Path, diff --git a/statediff/indexer/database/sql/mainnet_tests/indexer_test.go b/statediff/indexer/database/sql/mainnet_tests/indexer_test.go index 68c9bc464..d841bc360 100644 --- a/statediff/indexer/database/sql/mainnet_tests/indexer_test.go +++ b/statediff/indexer/database/sql/mainnet_tests/indexer_test.go @@ -23,6 +23,8 @@ import ( "os" "testing" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/core/types" @@ -36,7 +38,7 @@ import ( var ( err error - db sql.Database + db interfaces.Database ind interfaces.StateDiffIndexer chainConf = params.MainnetChainConfig ) @@ -76,14 +78,19 @@ func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Rece } func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) { - db, err = postgres.SetupSQLXDB() + db, err = postgres.SetupV3SQLXDB() if err != nil { t.Fatal(err) } - ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db) + dbV2, err := postgres.SetupV2SQLXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, nodeinfo.Info{}, dbV2, db) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + var headerID int64 + tx, headerID, err = ind.PushBlock( testBlock, testReceipts, testBlock.Difficulty()) @@ -95,7 +102,7 @@ func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) { } }() for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, testBlock.Hash().String()) + err = ind.PushStateNode(tx, node, testBlock.Hash().String(), headerID) require.NoError(t, err) } diff --git a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go index 37ea11881..717aa8591 100644 --- a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go @@ -39,10 +39,13 @@ func setupLegacyPGX(t *testing.T) { db, err = postgres.SetupV3PGXDB() require.NoError(t, err) - ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, nodeinfo.Info{}, db, nil) + v2DB, err := postgres.SetupV2PGXDB() + require.NoError(t, err) + + ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, nodeinfo.Info{}, v2DB, db) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + tx, headerID, err = ind.PushBlock( mockLegacyBlock, legacyData.MockReceipts, legacyData.MockBlock.Difficulty()) @@ -54,7 +57,7 @@ func setupLegacyPGX(t *testing.T) { } }() for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String(), 0) + err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String(), headerID) require.NoError(t, err) } diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index deb4bd6c6..393cd446e 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -41,26 +41,26 @@ import ( func setupPGX(t *testing.T) { db, err = postgres.SetupV3PGXDB() - if err != nil { - t.Fatal(err) - } - ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, nodeInfo.Info{}, db, nil) + require.NoError(t, err) + + v2DB, err := postgres.SetupV2PGXDB() + require.NoError(t, err) + + ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, nodeInfo.Info{}, v2DB, db) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + tx, headerID, err = ind.PushBlock( mockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer func() { if err := tx.Submit(err); err != nil { t.Fatal(err) } }() for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), 0) + err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), headerID) require.NoError(t, err) } @@ -89,24 +89,18 @@ func TestPGXIndexer(t *testing.T) { &header.Reward, &header.BlockHash, &header.Coinbase) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, header.CID, headerCID.String()) test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250") test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String()) dc, err := cid.Decode(header.CID) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) mhKey := dshelp.MultihashToDsKey(dc.Hash()) prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() var data []byte err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp) }) @@ -118,9 +112,7 @@ func TestPGXIndexer(t *testing.T) { pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) WHERE header_cids.block_number = $1` err = db.Select(context.Background(), &trxs, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, len(trxs), 5) expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String())) @@ -251,9 +243,7 @@ func TestPGXIndexer(t *testing.T) { INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key) WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC` err = db.Select(context.Background(), &rcts, rctsPgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if len(rcts) != len(mocks.MockReceipts) { t.Fatalf("expected %d receipts, got %d", len(mocks.MockReceipts), len(rcts)) } @@ -304,9 +294,7 @@ func TestPGXIndexer(t *testing.T) { AND transaction_cids.header_id = header_cids.block_hash AND header_cids.block_number = $1 order by transaction_cids.index` err = db.Select(context.Background(), &rcts, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, len(rcts), 5) expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String())) @@ -403,9 +391,7 @@ func TestPGXIndexer(t *testing.T) { FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) WHERE header_cids.block_number = $1 AND node_type != 3` err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, len(stateNodes), 2) for _, stateNode := range stateNodes { var data []byte @@ -461,23 +447,17 @@ func TestPGXIndexer(t *testing.T) { FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) WHERE header_cids.block_number = $1 AND node_type = 3` err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, len(stateNodes), 1) stateNode := stateNodes[0] var data []byte dc, err := cid.Decode(stateNode.CID) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) mhKey := dshelp.MultihashToDsKey(dc.Hash()) prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) test_helpers.ExpectEqual(t, data, []byte{}) @@ -495,9 +475,7 @@ func TestPGXIndexer(t *testing.T) { AND header_cids.block_number = $1 AND storage_cids.node_type != 3` err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, len(storageNodes), 1) test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{ CID: storageCID.String(), @@ -508,15 +486,11 @@ func TestPGXIndexer(t *testing.T) { }) var data []byte dc, err := cid.Decode(storageNodes[0].CID) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) mhKey := dshelp.MultihashToDsKey(dc.Hash()) prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode) // check that Removed storage nodes were properly indexed @@ -528,9 +502,7 @@ func TestPGXIndexer(t *testing.T) { AND header_cids.block_number = $1 AND storage_cids.node_type = 3` err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, len(storageNodes), 1) test_helpers.ExpectEqual(t, storageNodes[0], v3Models.StorageNodeWithStateKeyModel{ CID: shared.RemovedNodeStorageCID, @@ -540,16 +512,12 @@ func TestPGXIndexer(t *testing.T) { Path: []byte{'\x03'}, }) dc, err = cid.Decode(storageNodes[0].CID) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) mhKey = dshelp.MultihashToDsKey(dc.Hash()) prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, data, []byte{}) }) } diff --git a/statediff/indexer/database/sql/postgres/config.go b/statediff/indexer/database/sql/postgres/config.go index 3ef5c71c9..bcaf4e0bf 100644 --- a/statediff/indexer/database/sql/postgres/config.go +++ b/statediff/indexer/database/sql/postgres/config.go @@ -45,7 +45,7 @@ func ResolveDriverType(str string) (DriverType, error) { } } -// DefaultConfig are default parameters for connecting to a Postgres sql +// DefaultConfig are default parameters for connecting to a Postgres DB var DefaultConfig = Config{ Hostname: "localhost", Port: 5432, @@ -54,6 +54,24 @@ var DefaultConfig = Config{ Password: "password", } +// DefaultV2Config are default parameters for connecting to a v3 Postgres DB +var DefaultV2Config = Config{ + Hostname: "localhost", + Port: 5432, + DatabaseName: "vulcanize_testing_v2", + Username: "vdbm", + Password: "password", +} + +// DefaultV3Config are default parameters for connecting to a v3 Postgres DB +var DefaultV3Config = Config{ + Hostname: "localhost", + Port: 5433, + DatabaseName: "vulcanize_testing_v3", + Username: "vdbm", + Password: "password", +} + // MultiConfig holds multiple configs type MultiConfig struct { V2 Config diff --git a/statediff/indexer/database/sql/postgres/errors.go b/statediff/indexer/database/sql/postgres/errors.go index effa74aa1..a77cdcb8e 100644 --- a/statediff/indexer/database/sql/postgres/errors.go +++ b/statediff/indexer/database/sql/postgres/errors.go @@ -22,17 +22,12 @@ import ( const ( DbConnectionFailedMsg = "db connection failed" - SettingNodeFailedMsg = "unable to set db node" ) func ErrDBConnectionFailed(connectErr error) error { return formatError(DbConnectionFailedMsg, connectErr.Error()) } -func ErrUnableToSetNode(setErr error) error { - return formatError(SettingNodeFailedMsg, setErr.Error()) -} - func formatError(msg, err string) error { return fmt.Errorf("%s: %s", msg, err) } diff --git a/statediff/indexer/database/sql/postgres/pgx_test.go b/statediff/indexer/database/sql/postgres/pgx_test.go index 64616e356..750acfcbc 100644 --- a/statediff/indexer/database/sql/postgres/pgx_test.go +++ b/statediff/indexer/database/sql/postgres/pgx_test.go @@ -18,7 +18,6 @@ package postgres_test import ( "context" - "fmt" "math/big" "strings" "testing" @@ -26,7 +25,6 @@ import ( "github.com/jackc/pgx/v4/pgxpool" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/node" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) @@ -98,24 +96,11 @@ func TestPostgresPGX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"} - _, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo) + _, err := postgres.NewPGXDriver(ctx, postgres.Config{}) if err == nil { t.Fatal("Expected an error") } expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg) }) - - t.Run("throws error when can't create node", func(t *testing.T) { - badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) - badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"} - - _, err := postgres.NewPGXDriver(ctx, postgres.DefaultConfig, badInfo) - if err == nil { - t.Fatal("Expected an error") - } - - expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg) - }) } diff --git a/statediff/indexer/database/sql/postgres/sqlx_test.go b/statediff/indexer/database/sql/postgres/sqlx_test.go index 03f24e9f5..4124a5cdd 100644 --- a/statediff/indexer/database/sql/postgres/sqlx_test.go +++ b/statediff/indexer/database/sql/postgres/sqlx_test.go @@ -17,16 +17,13 @@ package postgres_test import ( - "fmt" "math/big" - "strings" "testing" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/node" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) @@ -96,24 +93,11 @@ func TestPostgresSQLX(t *testing.T) { }) t.Run("throws error when can't connect to the database", func(t *testing.T) { - goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"} - _, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo) + _, err := postgres.NewSQLXDriver(ctx, postgres.Config{}) if err == nil { t.Fatal("Expected an error") } expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg) }) - - t.Run("throws error when can't create node", func(t *testing.T) { - badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) - badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"} - - _, err := postgres.NewSQLXDriver(ctx, postgres.DefaultConfig, badInfo) - if err == nil { - t.Fatal("Expected an error") - } - - expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg) - }) } diff --git a/statediff/indexer/database/sql/postgres/test_helpers.go b/statediff/indexer/database/sql/postgres/test_helpers.go index db29584b4..8a0ea021e 100644 --- a/statediff/indexer/database/sql/postgres/test_helpers.go +++ b/statediff/indexer/database/sql/postgres/test_helpers.go @@ -26,7 +26,7 @@ import ( // SetupV3SQLXDB is used to setup a sqlx db for tests func SetupV3SQLXDB() (interfaces.Database, error) { - driver, err := NewSQLXDriver(context.Background(), DefaultConfig) + driver, err := NewSQLXDriver(context.Background(), DefaultV3Config) if err != nil { return nil, err } @@ -35,7 +35,7 @@ func SetupV3SQLXDB() (interfaces.Database, error) { // SetupV3PGXDB is used to setup a pgx db for tests func SetupV3PGXDB() (interfaces.Database, error) { - driver, err := NewPGXDriver(context.Background(), DefaultConfig) + driver, err := NewPGXDriver(context.Background(), DefaultV3Config) if err != nil { return nil, err } @@ -44,7 +44,7 @@ func SetupV3PGXDB() (interfaces.Database, error) { // SetupV2SQLXDB is used to setup a sqlx db for tests func SetupV2SQLXDB() (interfaces.Database, error) { - driver, err := NewSQLXDriver(context.Background(), DefaultConfig) + driver, err := NewSQLXDriver(context.Background(), DefaultV2Config) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func SetupV2SQLXDB() (interfaces.Database, error) { // SetupV2PGXDB is used to setup a pgx db for tests func SetupV2PGXDB() (interfaces.Database, error) { - driver, err := NewPGXDriver(context.Background(), DefaultConfig) + driver, err := NewPGXDriver(context.Background(), DefaultV2Config) if err != nil { return nil, err } diff --git a/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go b/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go index 95d03cb8e..6c14ab3ba 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go @@ -40,6 +40,7 @@ var ( legacyData = mocks.NewLegacyData() mockLegacyBlock *types.Block legacyHeaderCID cid.Cid + headerID int64 ) func setupLegacySQLX(t *testing.T) { @@ -49,10 +50,13 @@ func setupLegacySQLX(t *testing.T) { db, err = postgres.SetupV3SQLXDB() require.NoError(t, err) - ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, nodeinfo.Info{}, db, nil) + v2DB, err := postgres.SetupV2PGXDB() + require.NoError(t, err) + + ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, nodeinfo.Info{}, v2DB, db) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + tx, headerID, err = ind.PushBlock( mockLegacyBlock, legacyData.MockReceipts, legacyData.MockBlock.Difficulty()) @@ -64,7 +68,7 @@ func setupLegacySQLX(t *testing.T) { } }() for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String(), 0) + err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String(), headerID) require.NoError(t, err) } diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index cd8ed4e33..9d86b83fd 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -42,26 +42,26 @@ import ( func setupSQLX(t *testing.T) { db, err = postgres.SetupV3SQLXDB() - if err != nil { - t.Fatal(err) - } - ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, nodeinfo.Info{}, db, nil) + require.NoError(t, err) + + v2DB, err := postgres.SetupV2PGXDB() + require.NoError(t, err) + + ind, err = sql.NewStateDiffIndexer(context.Background(), mocks.TestConfig, nodeinfo.Info{}, v2DB, db) require.NoError(t, err) var tx interfaces.Batch - tx, err = ind.PushBlock( + tx, headerID, err = ind.PushBlock( mockBlock, mocks.MockReceipts, mocks.MockBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer func() { if err := tx.Submit(err); err != nil { t.Fatal(err) } }() for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), 0) + err = ind.PushStateNode(tx, node, mockBlock.Hash().String(), headerID) require.NoError(t, err) } @@ -92,24 +92,21 @@ func TestSQLXIndexer(t *testing.T) { } header := new(res) err = db.QueryRow(context.Background(), pgStr, mocks.BlockNumber.Uint64()).(*sqlx.Row).StructScan(header) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + test_helpers.ExpectEqual(t, header.CID, headerCID.String()) test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250") test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String()) dc, err := cid.Decode(header.CID) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() var data []byte err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp) }) @@ -121,9 +118,8 @@ func TestSQLXIndexer(t *testing.T) { pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) WHERE header_cids.block_number = $1` err = db.Select(context.Background(), &trxs, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + test_helpers.ExpectEqual(t, len(trxs), 5) expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String())) @@ -138,25 +134,22 @@ func TestSQLXIndexer(t *testing.T) { } for _, c := range trxs { dc, err := cid.Decode(c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() var data []byte err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + txTypeAndValueStr := `SELECT tx_type, value FROM eth.transaction_cids WHERE cid = $1` switch c { case trx1CID.String(): test_helpers.ExpectEqual(t, data, tx1) txRes := new(txResult) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + if txRes.TxType != 0 { t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType) } @@ -167,9 +160,7 @@ func TestSQLXIndexer(t *testing.T) { test_helpers.ExpectEqual(t, data, tx2) txRes := new(txResult) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if txRes.TxType != 0 { t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType) } @@ -180,9 +171,7 @@ func TestSQLXIndexer(t *testing.T) { test_helpers.ExpectEqual(t, data, tx3) txRes := new(txResult) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if txRes.TxType != 0 { t.Fatalf("expected LegacyTxType (0), got %d", txRes.TxType) } @@ -193,9 +182,7 @@ func TestSQLXIndexer(t *testing.T) { test_helpers.ExpectEqual(t, data, tx4) txRes := new(txResult) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if txRes.TxType != types.AccessListTxType { t.Fatalf("expected AccessListTxType (1), got %d", txRes.TxType) } @@ -205,9 +192,7 @@ func TestSQLXIndexer(t *testing.T) { accessListElementModels := make([]v3Models.AccessListElementModel, 0) pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC` err = db.Select(context.Background(), &accessListElementModels, pgStr, c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if len(accessListElementModels) != 2 { t.Fatalf("expected two access list entries, got %d", len(accessListElementModels)) } @@ -226,9 +211,7 @@ func TestSQLXIndexer(t *testing.T) { test_helpers.ExpectEqual(t, data, tx5) txRes := new(txResult) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if txRes.TxType != types.DynamicFeeTxType { t.Fatalf("expected DynamicFeeTxType (2), got %d", txRes.TxType) } @@ -354,41 +337,31 @@ func TestSQLXIndexer(t *testing.T) { var postStatus uint64 pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1` err = db.Get(context.Background(), &postStatus, pgStr, c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus) case rct2CID.String(): test_helpers.ExpectEqual(t, data, rctLeaf2) var postState string err = db.Get(context.Background(), &postState, postStatePgStr, c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState1) case rct3CID.String(): test_helpers.ExpectEqual(t, data, rctLeaf3) var postState string err = db.Get(context.Background(), &postState, postStatePgStr, c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState2) case rct4CID.String(): test_helpers.ExpectEqual(t, data, rctLeaf4) var postState string err = db.Get(context.Background(), &postState, postStatePgStr, c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) case rct5CID.String(): test_helpers.ExpectEqual(t, data, rctLeaf5) var postState string err = db.Get(context.Background(), &postState, postStatePgStr, c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) } } diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index e10c7d977..f302acc70 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -22,6 +22,8 @@ import ( "crypto/rand" "math/big" + v3Models "github.com/ethereum/go-ethereum/statediff/indexer/models/v3" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -104,11 +106,11 @@ var ( Address: AnotherAddress, StorageKeys: []common.Hash{common.BytesToHash(StorageLeafKey), common.BytesToHash(MockStorageLeafKey)}, } - AccessListEntry1Model = v2.AccessListElementModel{ + AccessListEntry1Model = v3Models.AccessListElementModel{ Index: 0, Address: Address.Hex(), } - AccessListEntry2Model = v2.AccessListElementModel{ + AccessListEntry2Model = v3Models.AccessListElementModel{ Index: 1, Address: AnotherAddress.Hex(), StorageKeys: []string{common.BytesToHash(StorageLeafKey).Hex(), common.BytesToHash(MockStorageLeafKey).Hex()},