statediff/indexer/database/file/mainnet_tests/indexer_test.go

@@ -20,32 +20,25 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/big"
 	"os"
 	"testing"
 
-	"github.com/ipfs/go-cid"
 	"github.com/jmoiron/sqlx"
-	"github.com/multiformats/go-multihash"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
-	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
 	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
 )
 
 var (
-	testBlock     *types.Block
-	testReceipts  types.Receipts
-	testHeaderCID cid.Cid
-	sqlxdb        *sqlx.DB
-	err           error
-	chainConf     = params.MainnetChainConfig
+	sqlxdb    *sqlx.DB
+	chainConf = params.MainnetChainConfig
 )
 
 func init() {
@@ -55,13 +48,35 @@ func init() {
 	}
 }
 
-func setup(t *testing.T) {
-	testBlock, testReceipts, err = TestBlocksAndReceiptsFromEnv()
-	require.NoError(t, err)
-	headerRLP, err := rlp.EncodeToBytes(testBlock.Header())
+func TestPushBlockAndState(t *testing.T) {
+	conf := DefaultTestConfig
+	rawURL := os.Getenv(TEST_RAW_URL)
+	if rawURL == "" {
+		fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
+			"then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL)
+	} else {
+		conf.RawURL = rawURL
+	}
+	for _, blockNumber := range problemBlocks {
+		conf.BlockNumber = big.NewInt(blockNumber)
+		tb, trs, err := TestBlockAndReceipts(conf)
+		require.NoError(t, err)
+		testPushBlockAndState(t, tb, trs)
+	}
+	testBlock, testReceipts, err := TestBlockAndReceiptsFromEnv(conf)
 	require.NoError(t, err)
+	testPushBlockAndState(t, testBlock, testReceipts)
+}
 
-	testHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256)
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+	t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+		setup(t, block, receipts)
+		dumpData(t)
+		tearDown(t)
+	})
+}
+
+func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
 	if _, err := os.Stat(file.TestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
 		err := os.Remove(file.TestConfig.FilePath)
 		require.NoError(t, err)
@@ -113,11 +128,3 @@ func tearDown(t *testing.T) {
 	err = sqlxdb.Close()
 	require.NoError(t, err)
 }
-
-func TestPushBlockAndState(t *testing.T) {
-	t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
-		setup(t)
-		dumpData(t)
-		tearDown(t)
-	})
-}

statediff/indexer/database/file/mainnet_tests/test_helpers.go

@@ -36,9 +36,17 @@ const (
 const (
 	TEST_RAW_URL      = "TEST_RAW_URL"
 	TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER"
-	TEST_LOCAL_CACHE  = "TEST_LOCAL_CACHE"
 )
 
+var problemBlocks = []int64{
+	12600011,
+	12619985,
+	12625121,
+	12655432,
+	12579670,
+	12914664,
+}
+
 // TestConfig holds configuration params for mainnet tests
 type TestConfig struct {
 	RawURL      string
@@ -53,16 +61,8 @@ var DefaultTestConfig = TestConfig{
 	LocalCache:  true,
 }
 
-// TestBlocksAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config
-func TestBlocksAndReceiptsFromEnv() (*types.Block, types.Receipts, error) {
-	conf := DefaultTestConfig
-	rawURL := os.Getenv(TEST_RAW_URL)
-	if rawURL == "" {
-		fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
-			"then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL)
-	} else {
-		conf.RawURL = rawURL
-	}
+// TestBlockAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config block number
+func TestBlockAndReceiptsFromEnv(conf TestConfig) (*types.Block, types.Receipts, error) {
 	blockNumberStr := os.Getenv(TEST_BLOCK_NUMBER)
 	blockNumber, ok := new(big.Int).SetString(blockNumberStr, 10)
 	if !ok {
@@ -71,12 +71,12 @@ func TestBlocksAndReceiptsFromEnv() (*types.Block, types.Receipts, error) {
 	} else {
 		conf.BlockNumber = blockNumber
 	}
-	return TestBlocksAndReceipts(conf)
+	return TestBlockAndReceipts(conf)
 }
 
-// TestBlocksAndReceipts retrieves the block and receipts for the provided test config
+// TestBlockAndReceipts retrieves the block and receipts for the provided test config
 // It first tries to load files from the local system before setting up and using an ethclient.Client to pull the data
-func TestBlocksAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) {
+func TestBlockAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) {
 	var cli *ethclient.Client
 	var err error
 	var block *types.Block

statediff/indexer/database/file/writer.go

@@ -31,9 +31,9 @@ import (
 )
 
 var (
-	nullHash         = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
-	pipeSize         = 65336 // min(linuxPipeSize, macOSPipeSize)
-	collatedStmtSize = pipeSize * 16
+	nullHash        = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+	pipeSize        = 65336 // min(linuxPipeSize, macOSPipeSize)
+	writeBufferSize = pipeSize * 16 * 48
 )
 
 // SQLWriter writes sql statements to a file
@@ -54,7 +54,7 @@ func NewSQLWriter(wc io.WriteCloser) *SQLWriter {
 	return &SQLWriter{
 		wc:            wc,
 		stmts:         make(chan []byte),
-		collatedStmt:  make([]byte, collatedStmtSize),
+		collatedStmt:  make([]byte, writeBufferSize),
 		flushChan:     make(chan struct{}),
 		flushFinished: make(chan struct{}),
 		quitChan:      make(chan struct{}),
@@ -74,7 +74,7 @@ func (sqw *SQLWriter) Loop() {
 			select {
 			case stmt := <-sqw.stmts:
 				l = len(stmt)
-				if l+sqw.collationIndex+1 > collatedStmtSize {
+				if sqw.collationIndex+l > writeBufferSize {
 					if err := sqw.flush(); err != nil {
 						panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
 					}

statediff/indexer/ipld/eth_header.go

@@ -20,13 +20,13 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthHeader (eth-block, codec 0x90), represents an ethereum block header

statediff/indexer/ipld/eth_header_test.go

@@ -9,10 +9,11 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/core/types"
 	block "github.com/ipfs/go-block-format"
 	node "github.com/ipfs/go-ipld-format"
 	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 func TestBlockBodyRlpParsing(t *testing.T) {

statediff/indexer/ipld/eth_log.go

@@ -3,11 +3,12 @@ package ipld
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthLog (eth-log, codec 0x9a), represents an ethereum block header

statediff/indexer/ipld/eth_log_trie.go

@@ -3,10 +3,11 @@ package ipld
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthLogTrie (eth-tx-trie codec 0x9p) represents

statediff/indexer/ipld/eth_parser.go

@@ -23,11 +23,12 @@ import (
 	"io"
 	"io/ioutil"
 
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
 )
 
 // FromBlockRLP takes an RLP message representing

statediff/indexer/ipld/eth_parser_test.go

@@ -21,10 +21,11 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
-	"github.com/stretchr/testify/require"
 )
 
 type kind string

statediff/indexer/ipld/eth_receipt.go

@@ -21,10 +21,11 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 type EthReceipt struct {

statediff/indexer/ipld/eth_tx.go

@@ -22,11 +22,12 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 // EthTx (eth-tx codec 0x93) represents an ethereum transaction

statediff/indexer/ipld/trie_node.go

@@ -20,9 +20,10 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
+
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 const (