From 4a38a97ffe6169ec5736df9bc14fc66e725840d9 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Fri, 12 Aug 2022 19:05:54 +0530 Subject: [PATCH] Refactor indexer tests for legacy block --- .../database/file/csv_indexer_legacy_test.go | 50 ++++---------- .../database/file/indexer_shared_test.go | 30 -------- .../database/file/sql_indexer_legacy_test.go | 50 ++++---------- .../database/sql/pgx_indexer_legacy_test.go | 63 ++++------------- .../database/sql/sqlx_indexer_legacy_test.go | 69 ++++--------------- statediff/indexer/mocks/test_data.go | 3 +- statediff/indexer/test/test.go | 64 +++++++++++++++++ 7 files changed, 117 insertions(+), 212 deletions(-) diff --git a/statediff/indexer/database/file/csv_indexer_legacy_test.go b/statediff/indexer/database/file/csv_indexer_legacy_test.go index a5c30d0eb..d477addb3 100644 --- a/statediff/indexer/database/file/csv_indexer_legacy_test.go +++ b/statediff/indexer/database/file/csv_indexer_legacy_test.go @@ -25,63 +25,38 @@ import ( "strings" "testing" - "github.com/jmoiron/sqlx" - "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/file/types" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/test" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) const dbDirectory = "/file_indexer" const pgCopyStatement = `COPY %s FROM '%s' CSV` -func setupCSVLegacy(t *testing.T) { - mockLegacyBlock = legacyData.MockBlock - legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) - file.CSVTestConfig.OutputDir = "./statediffing_legacy_test" - +func setupLegacyCSVIndexer(t *testing.T) { if _, err := os.Stat(file.CSVTestConfig.OutputDir); !errors.Is(err, os.ErrNotExist) { err := os.RemoveAll(file.CSVTestConfig.OutputDir) require.NoError(t, err) } - ind, err := file.NewStateDiffIndexer(context.Background(), legacyData.Config, file.CSVTestConfig) - require.NoError(t, err) - var tx interfaces.Batch - tx, err = ind.PushBlock( - mockLegacyBlock, - legacyData.MockReceipts, - legacyData.MockBlock.Difficulty()) + ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.CSVTestConfig) require.NoError(t, err) - defer func() { - if err := tx.Submit(err); err != nil { - t.Fatal(err) - } - if err := ind.Close(); err != nil { - t.Fatal(err) - } - }() - - for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String()) - require.NoError(t, err) - } - - require.Equal(t, legacyData.BlockNumber.String(), tx.(*file.BatchTx).BlockNumber) - - connStr := postgres.DefaultConfig.DbConnectionString() - sqlxdb, err = sqlx.Connect("postgres", connStr) + db, err = postgres.SetupSQLXDB() if err != nil { - t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err) + t.Fatal(err) } } +func setupLegacyCSV(t *testing.T) { + setupLegacyCSVIndexer(t) + test.SetupLegacyTestData(t, ind) +} + func dumpCSVFileData(t *testing.T) { outputDir := filepath.Join(dbDirectory, file.CSVTestConfig.OutputDir) workingDir, err := os.Getwd() @@ -133,9 +108,10 @@ func tearDownCSV(t *testing.T) { func TestLegacyCSVFileIndexer(t *testing.T) { t.Run("Publish and index header IPLDs", func(t *testing.T) { - setupCSVLegacy(t) + setupLegacyCSV(t) dumpCSVFileData(t) defer tearDownCSV(t) - testLegacyPublishAndIndexHeaderIPLDs(t) + + test.TestLegacyIndexer(t, db) }) } diff --git a/statediff/indexer/database/file/indexer_shared_test.go b/statediff/indexer/database/file/indexer_shared_test.go index d73956ef6..2d24e0d30 100644 --- a/statediff/indexer/database/file/indexer_shared_test.go +++ b/statediff/indexer/database/file/indexer_shared_test.go @@ -19,24 +19,17 @@ package file_test import ( "testing" - "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" - "github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) var ( - legacyData = mocks.NewLegacyData() - mockLegacyBlock *types.Block - legacyHeaderCID cid.Cid - db sql.Database sqlxdb *sqlx.DB err error @@ -59,29 +52,6 @@ func resetDB(t *testing.T) { } } -func testLegacyPublishAndIndexHeaderIPLDs(t *testing.T) { - pgStr := `SELECT cid, td, reward, block_hash, coinbase - FROM eth.header_cids - WHERE block_number = $1` - // check header was properly indexed - type res struct { - CID string - TD string - Reward string - BlockHash string `db:"block_hash"` - Coinbase string `db:"coinbase"` - } - header := new(res) - err = sqlxdb.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header) - require.NoError(t, err) - - require.Equal(t, legacyHeaderCID.String(), header.CID) - require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD) - require.Equal(t, "5000000000000011250", header.Reward) - require.Equal(t, legacyData.MockBlock.Coinbase().String(), header.Coinbase) - require.Nil(t, legacyData.MockHeader.BaseFee) -} - func testLoadEmptyWatchedAddresses(t *testing.T) { expectedData := []common.Address{} diff --git a/statediff/indexer/database/file/sql_indexer_legacy_test.go b/statediff/indexer/database/file/sql_indexer_legacy_test.go index 190ff4269..998992c18 100644 --- a/statediff/indexer/database/file/sql_indexer_legacy_test.go +++ b/statediff/indexer/database/file/sql_indexer_legacy_test.go @@ -22,57 +22,34 @@ import ( "os" "testing" - "github.com/jmoiron/sqlx" - "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/test" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) -func setupLegacy(t *testing.T) { - mockLegacyBlock = legacyData.MockBlock - legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) - +func setupLegacySQLIndexer(t *testing.T) { if _, err := os.Stat(file.SQLTestConfig.FilePath); !errors.Is(err, os.ErrNotExist) { err := os.Remove(file.SQLTestConfig.FilePath) require.NoError(t, err) } - ind, err := file.NewStateDiffIndexer(context.Background(), legacyData.Config, file.SQLTestConfig) - require.NoError(t, err) - var tx interfaces.Batch - tx, err = ind.PushBlock( - mockLegacyBlock, - legacyData.MockReceipts, - legacyData.MockBlock.Difficulty()) + + ind, err = file.NewStateDiffIndexer(context.Background(), test.LegacyConfig, file.SQLTestConfig) require.NoError(t, err) - defer func() { - if err := tx.Submit(err); err != nil { - t.Fatal(err) - } - if err := ind.Close(); err != nil { - t.Fatal(err) - } - }() - - for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String()) - require.NoError(t, err) - } - - require.Equal(t, legacyData.BlockNumber.String(), tx.(*file.BatchTx).BlockNumber) - - connStr := postgres.DefaultConfig.DbConnectionString() - sqlxdb, err = sqlx.Connect("postgres", connStr) + db, err = postgres.SetupSQLXDB() if err != nil { - t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err) + t.Fatal(err) } } +func setupLegacySQL(t *testing.T) { + setupLegacySQLIndexer(t) + test.SetupLegacyTestData(t, ind) +} + func dumpFileData(t *testing.T) { err := test_helpers.DedupFile(file.SQLTestConfig.FilePath) require.NoError(t, err) @@ -108,9 +85,10 @@ func tearDown(t *testing.T) { func TestLegacySQLFileIndexer(t *testing.T) { t.Run("Publish and index header IPLDs", func(t *testing.T) { - setupLegacy(t) + setupLegacySQL(t) dumpFileData(t) defer tearDown(t) - testLegacyPublishAndIndexHeaderIPLDs(t) + + test.TestLegacyIndexer(t, db) }) } diff --git a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go index bb3b36446..319ecca6d 100644 --- a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go @@ -20,70 +20,33 @@ import ( "context" "testing" - "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/test" ) -func setupLegacyPGX(t *testing.T) { - mockLegacyBlock = legacyData.MockBlock - legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) - +func setupLegacyPGXIndexer(t *testing.T) { db, err = postgres.SetupPGXDB() - require.NoError(t, err) - - ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, db) - require.NoError(t, err) - var tx interfaces.Batch - tx, err = ind.PushBlock( - mockLegacyBlock, - legacyData.MockReceipts, - legacyData.MockBlock.Difficulty()) - require.NoError(t, err) - - defer func() { - if err := tx.Submit(err); err != nil { - t.Fatal(err) - } - }() - for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String()) - require.NoError(t, err) + if err != nil { + t.Fatal(err) } + ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db) + require.NoError(t, err) +} - require.Equal(t, legacyData.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) +func setupLegacyPGX(t *testing.T) { + setupLegacyPGXIndexer(t) + test.SetupLegacyTestData(t, ind) } func TestLegacyPGXIndexer(t *testing.T) { t.Run("Publish and index header IPLDs", func(t *testing.T) { - setupLegacyPGX(t) + setupLegacySQLX(t) defer tearDown(t) - defer checkTxClosure(t, 1, 0, 1) - pgStr := `SELECT cid, cast(td AS TEXT), cast(reward AS TEXT), block_hash, coinbase - FROM eth.header_cids - WHERE block_number = $1` - // check header was properly indexed - type res struct { - CID string - TD string - Reward string - BlockHash string `db:"block_hash"` - Coinbase string `db:"coinbase"` - } - header := new(res) + defer checkTxClosure(t, 0, 0, 0) - err = db.QueryRow(context.Background(), pgStr, legacyData.BlockNumber.Uint64()).Scan( - &header.CID, &header.TD, &header.Reward, &header.BlockHash, &header.Coinbase) - require.NoError(t, err) - - require.Equal(t, legacyHeaderCID.String(), header.CID) - require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD) - require.Equal(t, "5000000000000011250", header.Reward) - require.Equal(t, legacyData.MockHeader.Coinbase.String(), header.Coinbase) - require.Nil(t, legacyData.MockHeader.BaseFee) + test.TestLegacyIndexer(t, db) }) } diff --git a/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go b/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go index daaa1550c..4a07b8a0e 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_legacy_test.go @@ -20,52 +20,25 @@ import ( "context" "testing" - "github.com/ipfs/go-cid" - "github.com/jmoiron/sqlx" - "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" - "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/test" ) -var ( - legacyData = mocks.NewLegacyData() - mockLegacyBlock *types.Block - legacyHeaderCID cid.Cid -) +func setupLegacySQLXIndexer(t *testing.T) { + db, err = postgres.SetupSQLXDB() + if err != nil { + t.Fatal(err) + } + ind, err = sql.NewStateDiffIndexer(context.Background(), test.LegacyConfig, db) + require.NoError(t, err) +} func setupLegacySQLX(t *testing.T) { - mockLegacyBlock = legacyData.MockBlock - legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) - - db, err = postgres.SetupSQLXDB() - require.NoError(t, err) - - ind, err = sql.NewStateDiffIndexer(context.Background(), legacyData.Config, db) - require.NoError(t, err) - var tx interfaces.Batch - tx, err = ind.PushBlock( - mockLegacyBlock, - legacyData.MockReceipts, - legacyData.MockBlock.Difficulty()) - require.NoError(t, err) - - defer func() { - if err := tx.Submit(err); err != nil { - t.Fatal(err) - } - }() - for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String()) - require.NoError(t, err) - } - - require.Equal(t, legacyData.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) + setupLegacySQLXIndexer(t) + test.SetupLegacyTestData(t, ind) } func TestLegacySQLXIndexer(t *testing.T) { @@ -73,25 +46,7 @@ func TestLegacySQLXIndexer(t *testing.T) { setupLegacySQLX(t) defer tearDown(t) defer checkTxClosure(t, 0, 0, 0) - pgStr := `SELECT cid, td, reward, block_hash, coinbase - FROM eth.header_cids - WHERE block_number = $1` - // check header was properly indexed - type res struct { - CID string - TD string - Reward string - BlockHash string `db:"block_hash"` - Coinbase string `db:"coinbase"` - } - header := new(res) - err = db.QueryRow(context.Background(), pgStr, legacyData.BlockNumber.Uint64()).(*sqlx.Row).StructScan(header) - require.NoError(t, err) - require.Equal(t, legacyHeaderCID.String(), header.CID) - require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD) - require.Equal(t, "5000000000000011250", header.Reward) - require.Equal(t, legacyData.MockHeader.Coinbase.String(), header.Coinbase) - require.Nil(t, legacyData.MockHeader.BaseFee) + test.TestLegacyIndexer(t, db) }) } diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index ceb45233c..efb91a1d8 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -284,8 +284,7 @@ type LegacyData struct { StateDiffs []sdtypes.StateNode } -func NewLegacyData() *LegacyData { - config := params.MainnetChainConfig +func NewLegacyData(config *params.ChainConfig) *LegacyData { // Block number before london fork. blockNumber := config.EIP155Block diff --git a/statediff/indexer/test/test.go b/statediff/indexer/test/test.go index 1127c4f65..99b7047f4 100644 --- a/statediff/indexer/test/test.go +++ b/statediff/indexer/test/test.go @@ -17,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" @@ -1485,3 +1486,66 @@ func TestPublishAndIndexStorageNonCanonical(t *testing.T, db sql.Database) { require.Equal(t, expectedStorageNode, storageNodes[i]) } } + +var ( + LegacyConfig = params.MainnetChainConfig + legacyData = mocks.NewLegacyData(LegacyConfig) + mockLegacyBlock *types.Block + legacyHeaderCID cid.Cid +) + +func SetupLegacyTestData(t *testing.T, ind interfaces.StateDiffIndexer) { + mockLegacyBlock = legacyData.MockBlock + legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) + + var tx interfaces.Batch + tx, err = ind.PushBlock( + mockLegacyBlock, + legacyData.MockReceipts, + legacyData.MockBlock.Difficulty()) + require.NoError(t, err) + + defer func() { + if err := tx.Submit(err); err != nil { + t.Fatal(err) + } + }() + for _, node := range legacyData.StateDiffs { + err = ind.PushStateNode(tx, node, mockLegacyBlock.Hash().String()) + require.NoError(t, err) + } + + if batchTx, ok := tx.(*sql.BatchTx); ok { + require.Equal(t, legacyData.BlockNumber.String(), batchTx.BlockNumber) + } else if batchTx, ok := tx.(*file.BatchTx); ok { + require.Equal(t, legacyData.BlockNumber.String(), batchTx.BlockNumber) + } +} + +func TestLegacyIndexer(t *testing.T, db sql.Database) { + pgStr := `SELECT cid, td, reward, block_hash, coinbase + FROM eth.header_cids + WHERE block_number = $1` + // check header was properly indexed + type res struct { + CID string + TD string + Reward string + BlockHash string `db:"block_hash"` + Coinbase string `db:"coinbase"` + } + header := new(res) + err = db.QueryRow(context.Background(), pgStr, legacyData.BlockNumber.Uint64()).Scan( + &header.CID, + &header.TD, + &header.Reward, + &header.BlockHash, + &header.Coinbase) + require.NoError(t, err) + + require.Equal(t, legacyHeaderCID.String(), header.CID) + require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD) + require.Equal(t, "5000000000000011250", header.Reward) + require.Equal(t, legacyData.MockHeader.Coinbase.String(), header.Coinbase) + require.Nil(t, legacyData.MockHeader.BaseFee) +}