diff --git a/statediff/indexer/database/file/mainnet_tests/block_12579670.rlp b/statediff/indexer/database/file/mainnet_tests/block_12579670.rlp new file mode 100644 index 000000000..6b4f3f773 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/block_12579670.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/block_12600011.rlp b/statediff/indexer/database/file/mainnet_tests/block_12600011.rlp new file mode 100644 index 000000000..96032b0c2 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/block_12600011.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/block_12619985.rlp b/statediff/indexer/database/file/mainnet_tests/block_12619985.rlp new file mode 100644 index 000000000..0e735313f Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/block_12619985.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/block_12625121.rlp b/statediff/indexer/database/file/mainnet_tests/block_12625121.rlp new file mode 100644 index 000000000..d031e30ea Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/block_12625121.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/block_12655432.rlp b/statediff/indexer/database/file/mainnet_tests/block_12655432.rlp new file mode 100644 index 000000000..fafc6bd88 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/block_12655432.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/indexer_test.go b/statediff/indexer/database/file/mainnet_tests/indexer_test.go index 7b7ba8ff6..32850ca29 100644 --- a/statediff/indexer/database/file/mainnet_tests/indexer_test.go +++ b/statediff/indexer/database/file/mainnet_tests/indexer_test.go @@ -20,32 +20,25 @@ import ( "context" "errors" "fmt" + "math/big" "os" "testing" - "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" - "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers" ) var ( - testBlock *types.Block - testReceipts types.Receipts - testHeaderCID cid.Cid - sqlxdb *sqlx.DB - err error - chainConf = params.MainnetChainConfig + sqlxdb *sqlx.DB + chainConf = params.MainnetChainConfig ) func init() { @@ -55,13 +48,35 @@ func init() { } } -func setup(t *testing.T) { - testBlock, testReceipts, err = TestBlocksAndReceiptsFromEnv() - require.NoError(t, err) - headerRLP, err := rlp.EncodeToBytes(testBlock.Header()) +func TestPushBlockAndState(t *testing.T) { + conf := DefaultTestConfig + rawURL := os.Getenv(TEST_RAW_URL) + if rawURL == "" { + fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+ + "then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL) + } else { + conf.RawURL = rawURL + } + for _, blockNumber := range problemBlocks { + conf.BlockNumber = big.NewInt(blockNumber) + tb, trs, err := TestBlockAndReceipts(conf) + require.NoError(t, err) + testPushBlockAndState(t, tb, trs) + } + testBlock, testReceipts, err := TestBlockAndReceiptsFromEnv(conf) require.NoError(t, err) + testPushBlockAndState(t, testBlock, testReceipts) +} - testHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256) +func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) { + t.Run("Test PushBlock and PushStateNode", func(t *testing.T) { + setup(t, block, receipts) + dumpData(t) + tearDown(t) + }) +} + +func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) { if _, err := os.Stat(file.TestConfig.FilePath); !errors.Is(err, os.ErrNotExist) { err := os.Remove(file.TestConfig.FilePath) require.NoError(t, err) @@ -113,11 +128,3 @@ func tearDown(t *testing.T) { err = sqlxdb.Close() require.NoError(t, err) } - -func TestPushBlockAndState(t *testing.T) { - t.Run("Test PushBlock and PushStateNode", func(t *testing.T) { - setup(t) - dumpData(t) - tearDown(t) - }) -} diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12579670.rlp b/statediff/indexer/database/file/mainnet_tests/receipts_12579670.rlp new file mode 100644 index 000000000..e69de8fd9 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/receipts_12579670.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12600011.rlp b/statediff/indexer/database/file/mainnet_tests/receipts_12600011.rlp new file mode 100644 index 000000000..ae6d4f0c2 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/receipts_12600011.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12619985.rlp b/statediff/indexer/database/file/mainnet_tests/receipts_12619985.rlp new file mode 100644 index 000000000..a9ba84bd2 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/receipts_12619985.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12625121.rlp b/statediff/indexer/database/file/mainnet_tests/receipts_12625121.rlp new file mode 100644 index 000000000..4d3a8532c Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/receipts_12625121.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12655432.rlp b/statediff/indexer/database/file/mainnet_tests/receipts_12655432.rlp new file mode 100644 index 000000000..f209f01d8 Binary files /dev/null and b/statediff/indexer/database/file/mainnet_tests/receipts_12655432.rlp differ diff --git a/statediff/indexer/database/file/mainnet_tests/test_helpers.go b/statediff/indexer/database/file/mainnet_tests/test_helpers.go index ae2b81435..7203649d4 100644 --- a/statediff/indexer/database/file/mainnet_tests/test_helpers.go +++ b/statediff/indexer/database/file/mainnet_tests/test_helpers.go @@ -36,9 +36,17 @@ const ( const ( TEST_RAW_URL = "TEST_RAW_URL" TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER" - TEST_LOCAL_CACHE = "TEST_LOCAL_CACHE" ) +var problemBlocks = []int64{ + 12600011, + 12619985, + 12625121, + 12655432, + 12579670, + 12914664, +} + // TestConfig holds configuration params for mainnet tests type TestConfig struct { RawURL string @@ -53,16 +61,8 @@ var DefaultTestConfig = TestConfig{ LocalCache: true, } -// TestBlocksAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config -func TestBlocksAndReceiptsFromEnv() (*types.Block, types.Receipts, error) { - conf := DefaultTestConfig - rawURL := os.Getenv(TEST_RAW_URL) - if rawURL == "" { - fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+ - "then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL) - } else { - conf.RawURL = rawURL - } +// TestBlockAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config block number +func TestBlockAndReceiptsFromEnv(conf TestConfig) (*types.Block, types.Receipts, error) { blockNumberStr := os.Getenv(TEST_BLOCK_NUMBER) blockNumber, ok := new(big.Int).SetString(blockNumberStr, 10) if !ok { @@ -71,12 +71,12 @@ func TestBlocksAndReceiptsFromEnv() (*types.Block, types.Receipts, error) { } else { conf.BlockNumber = blockNumber } - return TestBlocksAndReceipts(conf) + return TestBlockAndReceipts(conf) } -// TestBlocksAndReceipts retrieves the block and receipts for the provided test config +// TestBlockAndReceipts retrieves the block and receipts for the provided test config // It first tries to load files from the local system before setting up and using an ethclient.Client to pull the data -func TestBlocksAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) { +func TestBlockAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) { var cli *ethclient.Client var err error var block *types.Block diff --git a/statediff/indexer/database/file/writer.go b/statediff/indexer/database/file/writer.go index 62fd5cb8d..e66e71f94 100644 --- a/statediff/indexer/database/file/writer.go +++ b/statediff/indexer/database/file/writer.go @@ -31,9 +31,9 @@ import ( ) var ( - nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") - pipeSize = 65336 // min(linuxPipeSize, macOSPipeSize) - collatedStmtSize = pipeSize * 16 + nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") + pipeSize = 65336 // min(linuxPipeSize, macOSPipeSize) + writeBufferSize = pipeSize * 16 * 48 ) // SQLWriter writes sql statements to a file @@ -54,7 +54,7 @@ func NewSQLWriter(wc io.WriteCloser) *SQLWriter { return &SQLWriter{ wc: wc, stmts: make(chan []byte), - collatedStmt: make([]byte, collatedStmtSize), + collatedStmt: make([]byte, writeBufferSize), flushChan: make(chan struct{}), flushFinished: make(chan struct{}), quitChan: make(chan struct{}), @@ -74,7 +74,7 @@ func (sqw *SQLWriter) Loop() { select { case stmt := <-sqw.stmts: l = len(stmt) - if l+sqw.collationIndex+1 > collatedStmtSize { + if sqw.collationIndex+l > writeBufferSize { if err := sqw.flush(); err != nil { panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err)) } diff --git a/statediff/indexer/ipld/eth_header.go b/statediff/indexer/ipld/eth_header.go index 5905bdd7e..9bc307277 100644 --- a/statediff/indexer/ipld/eth_header.go +++ b/statediff/indexer/ipld/eth_header.go @@ -20,13 +20,13 @@ import ( "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/common" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" "github.com/ipfs/go-cid" node "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" ) // EthHeader (eth-block, codec 0x90), represents an ethereum block header diff --git a/statediff/indexer/ipld/eth_header_test.go b/statediff/indexer/ipld/eth_header_test.go index d1ed13d37..ebbab2129 100644 --- a/statediff/indexer/ipld/eth_header_test.go +++ b/statediff/indexer/ipld/eth_header_test.go @@ -9,10 +9,11 @@ import ( "strconv" "testing" - "github.com/ethereum/go-ethereum/core/types" block "github.com/ipfs/go-block-format" node "github.com/ipfs/go-ipld-format" "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" ) func TestBlockBodyRlpParsing(t *testing.T) { diff --git a/statediff/indexer/ipld/eth_log.go b/statediff/indexer/ipld/eth_log.go index ebc762065..225c44117 100644 --- a/statediff/indexer/ipld/eth_log.go +++ b/statediff/indexer/ipld/eth_log.go @@ -3,11 +3,12 @@ package ipld import ( "fmt" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" "github.com/ipfs/go-cid" node "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" ) // EthLog (eth-log, codec 0x9a), represents an ethereum block header diff --git a/statediff/indexer/ipld/eth_log_trie.go b/statediff/indexer/ipld/eth_log_trie.go index edfd35af9..1df21642d 100644 --- a/statediff/indexer/ipld/eth_log_trie.go +++ b/statediff/indexer/ipld/eth_log_trie.go @@ -3,10 +3,11 @@ package ipld import ( "fmt" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" ) // EthLogTrie (eth-tx-trie codec 0x9p) represents diff --git a/statediff/indexer/ipld/eth_parser.go b/statediff/indexer/ipld/eth_parser.go index 0f294ad30..497facba4 100644 --- a/statediff/indexer/ipld/eth_parser.go +++ b/statediff/indexer/ipld/eth_parser.go @@ -23,11 +23,12 @@ import ( "io" "io/ioutil" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" ) // FromBlockRLP takes an RLP message representing diff --git a/statediff/indexer/ipld/eth_parser_test.go b/statediff/indexer/ipld/eth_parser_test.go index 9cb8d4e46..bcf28efde 100644 --- a/statediff/indexer/ipld/eth_parser_test.go +++ b/statediff/indexer/ipld/eth_parser_test.go @@ -21,10 +21,11 @@ import ( "path/filepath" "testing" + "github.com/stretchr/testify/require" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" - "github.com/stretchr/testify/require" ) type kind string diff --git a/statediff/indexer/ipld/eth_receipt.go b/statediff/indexer/ipld/eth_receipt.go index 0eb6a2f81..ccd785515 100644 --- a/statediff/indexer/ipld/eth_receipt.go +++ b/statediff/indexer/ipld/eth_receipt.go @@ -21,10 +21,11 @@ import ( "fmt" "strconv" - "github.com/ethereum/go-ethereum/core/types" "github.com/ipfs/go-cid" node "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" ) type EthReceipt struct { diff --git a/statediff/indexer/ipld/eth_tx.go b/statediff/indexer/ipld/eth_tx.go index 394e235a8..99b1f9dbe 100644 --- a/statediff/indexer/ipld/eth_tx.go +++ b/statediff/indexer/ipld/eth_tx.go @@ -22,11 +22,12 @@ import ( "strconv" "strings" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" "github.com/ipfs/go-cid" node "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" ) // EthTx (eth-tx codec 0x93) represents an ethereum transaction diff --git a/statediff/indexer/ipld/trie_node.go b/statediff/indexer/ipld/trie_node.go index a344bab4f..816217064 100644 --- a/statediff/indexer/ipld/trie_node.go +++ b/statediff/indexer/ipld/trie_node.go @@ -20,9 +20,10 @@ import ( "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/rlp" "github.com/ipfs/go-cid" node "github.com/ipfs/go-ipld-format" + + "github.com/ethereum/go-ethereum/rlp" ) const (