From 6e1679e59bdf9558cf199c266d25c0d52fffdfb2 Mon Sep 17 00:00:00 2001 From: i-norden Date: Fri, 12 Aug 2022 12:44:33 -0500 Subject: [PATCH] fix non-deterministic ordering in tests --- statediff/indexer/database/dump/indexer.go | 4 +++- statediff/indexer/database/file/indexer.go | 3 ++- .../database/file/indexer_shared_test.go | 22 +++++++++++-------- statediff/indexer/database/file/sql_writer.go | 2 +- statediff/indexer/database/sql/indexer.go | 3 ++- statediff/indexer/database/sql/interfaces.go | 1 - .../indexer/database/sql/pgx_indexer_test.go | 20 ++++++++++------- .../indexer/database/sql/postgres/database.go | 7 ------ .../indexer/database/sql/sqlx_indexer_test.go | 20 ++++++++++------- statediff/indexer/models/batch.go | 3 ++- statediff/indexer/models/models.go | 10 +-------- 11 files changed, 48 insertions(+), 47 deletions(-) diff --git a/statediff/indexer/database/dump/indexer.go b/statediff/indexer/database/dump/indexer.go index 723a13350..26a8c6eba 100644 --- a/statediff/indexer/database/dump/indexer.go +++ b/statediff/indexer/database/dump/indexer.go @@ -23,6 +23,8 @@ import ( "math/big" "time" + "github.com/ethereum/go-ethereum/common/hexutil" + blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" @@ -460,7 +462,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt StatePath: stateNode.Path, Balance: account.Balance.String(), Nonce: account.Nonce, - CodeHash: account.CodeHash, + CodeHash: hexutil.Encode(account.CodeHash), StorageRoot: account.Root.String(), } if _, err := fmt.Fprintf(sdi.dump, "%+v\r\n", accountModel); err != nil { diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go index 3159ea8fd..1ddc1bdbe 100644 --- a/statediff/indexer/database/file/indexer.go +++ b/statediff/indexer/database/file/indexer.go @@ -35,6 +35,7 @@ import ( "github.com/multiformats/go-multihash" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -504,7 +505,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt StatePath: stateNode.Path, Balance: account.Balance.String(), Nonce: account.Nonce, - CodeHash: account.CodeHash, + CodeHash: hexutil.Encode(account.CodeHash), StorageRoot: account.Root.String(), } sdi.fileWriter.upsertStateAccount(accountModel) diff --git a/statediff/indexer/database/file/indexer_shared_test.go b/statediff/indexer/database/file/indexer_shared_test.go index f38ff0507..691e3e191 100644 --- a/statediff/indexer/database/file/indexer_shared_test.go +++ b/statediff/indexer/database/file/indexer_shared_test.go @@ -577,7 +577,7 @@ func testPublishAndIndexStateIPLDs(t *testing.T) { HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "0", - CodeHash: mocks.ContractCodeHash.Bytes(), + CodeHash: mocks.ContractCodeHash.Hex(), StorageRoot: mocks.ContractRoot, Nonce: 1, }, account) @@ -592,7 +592,7 @@ func testPublishAndIndexStateIPLDs(t *testing.T) { HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "1000", - CodeHash: mocks.AccountCodeHash.Bytes(), + CodeHash: mocks.AccountCodeHash.Hex(), StorageRoot: mocks.AccountRoot, Nonce: 0, }, account) @@ -687,8 +687,12 @@ func testPublishAndIndexStorageIPLDs(t *testing.T) { t.Fatal(err) } require.Equal(t, 3, len(storageNodes)) - expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ - { + gotStorageNodes := make(map[string]models.StorageNodeWithStateKeyModel, 3) + for _, model := range storageNodes { + gotStorageNodes[model.StorageKey] = model + } + expectedStorageNodes := map[string]models.StorageNodeWithStateKeyModel{ + common.BytesToHash(mocks.RemovedLeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -696,7 +700,7 @@ func testPublishAndIndexStorageIPLDs(t *testing.T) { StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), Path: []byte{'\x03'}, }, - { + common.BytesToHash(mocks.Storage2LeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -704,7 +708,7 @@ func testPublishAndIndexStorageIPLDs(t *testing.T) { StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), Path: []byte{'\x0e'}, }, - { + common.BytesToHash(mocks.Storage3LeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -713,15 +717,15 @@ func testPublishAndIndexStorageIPLDs(t *testing.T) { Path: []byte{'\x0f'}, }, } - for idx, storageNode := range storageNodes { - require.Equal(t, expectedStorageNodes[idx], storageNode) + for storageKey, storageNode := range gotStorageNodes { + require.Equal(t, expectedStorageNodes[storageKey], storageNode) dc, err = cid.Decode(storageNode.CID) if err != nil { t.Fatal(err) } mhKey = dshelp.MultihashToDsKey(dc.Hash()) prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() - require.Equal(t, shared.RemovedNodeMhKey, prefixedKey, mocks.BlockNumber.Uint64()) + require.Equal(t, shared.RemovedNodeMhKey, prefixedKey) err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) if err != nil { t.Fatal(err) diff --git a/statediff/indexer/database/file/sql_writer.go b/statediff/indexer/database/file/sql_writer.go index 500aa35c9..56b97d204 100644 --- a/statediff/indexer/database/file/sql_writer.go +++ b/statediff/indexer/database/file/sql_writer.go @@ -165,7 +165,7 @@ const ( "VALUES ('%s', '%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n" accountInsert = "INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) " + - "VALUES ('%s', '%s', '\\x%x', '%s', %d, '\\x%x', '%s');\n" + "VALUES ('%s', '%s', '\\x%x', '%s', %d, '%s', '%s');\n" storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, " + "node_type, diff, mh_key) VALUES ('%s', '%s', '\\x%x', '%s', '%s', '\\x%x', %d, %t, '%s');\n" diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go index 114804af3..fa90dd396 100644 --- a/statediff/indexer/database/sql/indexer.go +++ b/statediff/indexer/database/sql/indexer.go @@ -34,6 +34,7 @@ import ( "github.com/multiformats/go-multihash" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -518,7 +519,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt StatePath: stateNode.Path, Balance: account.Balance.String(), Nonce: account.Nonce, - CodeHash: account.CodeHash, + CodeHash: hexutil.Encode(account.CodeHash), StorageRoot: account.Root.String(), } if err := sdi.dbWriter.upsertStateAccount(tx.dbtx, accountModel); err != nil { diff --git a/statediff/indexer/database/sql/interfaces.go b/statediff/indexer/database/sql/interfaces.go index 613a09251..445b35d9b 100644 --- a/statediff/indexer/database/sql/interfaces.go +++ b/statediff/indexer/database/sql/interfaces.go @@ -54,7 +54,6 @@ type Statements interface { InsertStorageStm() string InsertIPLDStm() string InsertIPLDsStm() string - InsertKnownGapsStm() string } // Tx interface to accommodate different concrete SQL transaction types diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index 79d3ae252..a21cb5557 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -435,7 +435,7 @@ func TestPGXIndexer(t *testing.T) { HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "0", - CodeHash: mocks.ContractCodeHash.Bytes(), + CodeHash: mocks.ContractCodeHash.Hex(), StorageRoot: mocks.ContractRoot, Nonce: 1, }, account) @@ -450,7 +450,7 @@ func TestPGXIndexer(t *testing.T) { HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "1000", - CodeHash: mocks.AccountCodeHash.Bytes(), + CodeHash: mocks.AccountCodeHash.Hex(), StorageRoot: mocks.AccountRoot, Nonce: 0, }, account) @@ -548,8 +548,12 @@ func TestPGXIndexer(t *testing.T) { t.Fatal(err) } require.Equal(t, 3, len(storageNodes)) - expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ - { + gotStorageNodes := make(map[string]models.StorageNodeWithStateKeyModel, 3) + for _, model := range storageNodes { + gotStorageNodes[model.StorageKey] = model + } + expectedStorageNodes := map[string]models.StorageNodeWithStateKeyModel{ + common.BytesToHash(mocks.RemovedLeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -557,7 +561,7 @@ func TestPGXIndexer(t *testing.T) { StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), Path: []byte{'\x03'}, }, - { + common.BytesToHash(mocks.Storage2LeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -565,7 +569,7 @@ func TestPGXIndexer(t *testing.T) { StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), Path: []byte{'\x0e'}, }, - { + common.BytesToHash(mocks.Storage3LeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -574,8 +578,8 @@ func TestPGXIndexer(t *testing.T) { Path: []byte{'\x0f'}, }, } - for idx, storageNode := range storageNodes { - require.Equal(t, expectedStorageNodes[idx], storageNode) + for storageKey, storageNode := range gotStorageNodes { + require.Equal(t, expectedStorageNodes[storageKey], storageNode) dc, err = cid.Decode(storageNode.CID) if err != nil { t.Fatal(err) diff --git a/statediff/indexer/database/sql/postgres/database.go b/statediff/indexer/database/sql/postgres/database.go index b865aeef7..f52358304 100644 --- a/statediff/indexer/database/sql/postgres/database.go +++ b/statediff/indexer/database/sql/postgres/database.go @@ -100,10 +100,3 @@ func (db *DB) InsertIPLDStm() string { func (db *DB) InsertIPLDsStm() string { return `INSERT INTO public.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING` } - -// InsertKnownGapsStm satisfies the sql.Statements interface -func (db *DB) InsertKnownGapsStm() string { - return `INSERT INTO eth_meta.known_gaps (starting_block_number, ending_block_number, checked_out, processing_key) VALUES ($1, $2, $3, $4) - ON CONFLICT (starting_block_number) DO UPDATE SET (ending_block_number, processing_key) = ($2, $4) - WHERE eth_meta.known_gaps.ending_block_number <= $2` -} diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index a2bf0afd4..74fe2c84b 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -428,7 +428,7 @@ func TestSQLXIndexer(t *testing.T) { HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "0", - CodeHash: mocks.ContractCodeHash.Bytes(), + CodeHash: mocks.ContractCodeHash.Hex(), StorageRoot: mocks.ContractRoot, Nonce: 1, }, account) @@ -443,7 +443,7 @@ func TestSQLXIndexer(t *testing.T) { HeaderID: account.HeaderID, StatePath: stateNode.Path, Balance: "1000", - CodeHash: mocks.AccountCodeHash.Bytes(), + CodeHash: mocks.AccountCodeHash.Hex(), StorageRoot: mocks.AccountRoot, Nonce: 0, }, account) @@ -541,8 +541,12 @@ func TestSQLXIndexer(t *testing.T) { t.Fatal(err) } require.Equal(t, 3, len(storageNodes)) - expectedStorageNodes := []models.StorageNodeWithStateKeyModel{ - { + gotStorageNodes := make(map[string]models.StorageNodeWithStateKeyModel, 3) + for _, model := range storageNodes { + gotStorageNodes[model.StorageKey] = model + } + expectedStorageNodes := map[string]models.StorageNodeWithStateKeyModel{ + common.BytesToHash(mocks.RemovedLeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -550,7 +554,7 @@ func TestSQLXIndexer(t *testing.T) { StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), Path: []byte{'\x03'}, }, - { + common.BytesToHash(mocks.Storage2LeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -558,7 +562,7 @@ func TestSQLXIndexer(t *testing.T) { StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(), Path: []byte{'\x0e'}, }, - { + common.BytesToHash(mocks.Storage3LeafKey).Hex(): { BlockNumber: mocks.BlockNumber.String(), CID: shared.RemovedNodeStorageCID, NodeType: 3, @@ -567,8 +571,8 @@ func TestSQLXIndexer(t *testing.T) { Path: []byte{'\x0f'}, }, } - for idx, storageNode := range storageNodes { - require.Equal(t, expectedStorageNodes[idx], storageNode) + for storageKey, storageNode := range gotStorageNodes { + require.Equal(t, expectedStorageNodes[storageKey], storageNode) dc, err = cid.Decode(storageNode.CID) if err != nil { t.Fatal(err) diff --git a/statediff/indexer/models/batch.go b/statediff/indexer/models/batch.go index 76858c96f..924c56788 100644 --- a/statediff/indexer/models/batch.go +++ b/statediff/indexer/models/batch.go @@ -34,6 +34,7 @@ type UncleBatch struct { CIDs []string MhKeys []string Rewards []string + Indexes []int64 } // TxBatch holds the arguments for a batch insert of tx data @@ -108,7 +109,7 @@ type AccountBatch struct { StatePaths [][]byte Balances []string Nonces []uint64 - CodeHashes [][]byte + CodeHashes []string StorageRoots []string } diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index d552b0de4..d4a6cb57b 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -141,7 +141,7 @@ type StateAccountModel struct { StatePath []byte `db:"state_path"` Balance string `db:"balance"` Nonce uint64 `db:"nonce"` - CodeHash []byte `db:"code_hash"` + CodeHash string `db:"code_hash"` StorageRoot string `db:"storage_root"` } @@ -160,11 +160,3 @@ type LogsModel struct { Topic2 string `db:"topic2"` Topic3 string `db:"topic3"` } - -// KnownGaps is the data structure for eth_meta.known_gaps -type KnownGapsModel struct { - StartingBlockNumber string `db:"starting_block_number"` - EndingBlockNumber string `db:"ending_block_number"` - CheckedOut bool `db:"checked_out"` - ProcessingKey int64 `db:"processing_key"` -}