Add indexer tests for receipts in a non-canonical block and refactor

statediff/indexer/database/sql/indexer_shared_test.go

@@ -2,20 +2,26 @@ package sql_test
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
+	"sort"
 	"testing"
 
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 	"github.com/stretchr/testify/require"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
 	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 )
 
 var (
@@ -25,10 +31,13 @@ var (
 	ipfsPgGet = `SELECT data FROM public.blocks
 					WHERE key = $1 AND block_number = $2`
 	tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5                             []byte
+	nonCanonicalRct1, nonCanonicalRct2                                                []byte
 	mockBlock, mockNonCanonicalBlock                                                  *types.Block
 	headerCID, mockNonCanonicalHeaderCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid
 	rct1CID, rct2CID, rct3CID, rct4CID, rct5CID                                       cid.Cid
+	nonCanonicalRct1CID, nonCanonicalRct2CID                                          cid.Cid
 	rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5                                  []byte
+	nonCanonicalRctLeaf1, nonCanonicalRctLeaf2                                        []byte
 	state1CID, state2CID, storageCID                                                  cid.Cid
 	contract1Address, contract2Address, contract3Address, contract4Address            string
 	contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt    uint64
@@ -45,6 +54,7 @@ func init() {
 	txs, rcts := mocks.MockBlock.Transactions(), mocks.MockReceipts
 
 	mockNonCanonicalBlock = mocks.MockNonCanonicalBlock
+	nonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts
 
 	buf := new(bytes.Buffer)
 	txs.EncodeIndex(0, buf)
@@ -97,6 +107,16 @@ func init() {
 	copy(rct5, buf.Bytes())
 	buf.Reset()
 
+	nonCanonicalBlockRcts.EncodeIndex(0, buf)
+	nonCanonicalRct1 = make([]byte, buf.Len())
+	copy(nonCanonicalRct1, buf.Bytes())
+	buf.Reset()
+
+	nonCanonicalBlockRcts.EncodeIndex(1, buf)
+	nonCanonicalRct2 = make([]byte, buf.Len())
+	copy(nonCanonicalRct2, buf.Bytes())
+	buf.Reset()
+
 	headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256)
 	mockNonCanonicalHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeaderRlp, multihash.KECCAK_256)
 	trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256)
@@ -141,6 +161,30 @@ func init() {
 	rctLeaf4 = orderedRctLeafNodes[3]
 	rctLeaf5 = orderedRctLeafNodes[4]
 
+	nonCanonicalRctTrie := ipld.NewRctTrie()
+
+	nonCanonicalRctTrie.Add(0, nonCanonicalRct1)
+	nonCanonicalRctTrie.Add(1, nonCanonicalRct2)
+
+	nonCanonicalRctLeafNodes, keys, _ := nonCanonicalRctTrie.GetLeafNodes()
+
+	nonCanonicalRctLeafNodeCids := make([]cid.Cid, len(nonCanonicalRctLeafNodes))
+	nonCanonicalRawRctLeafNodes := make([][]byte, len(nonCanonicalRctLeafNodes))
+	for i, rln := range nonCanonicalRctLeafNodes {
+		var idx uint
+
+		r := bytes.NewReader(keys[i].TrieKey)
+		rlp.Decode(r, &idx)
+		nonCanonicalRctLeafNodeCids[idx] = rln.Cid()
+		nonCanonicalRawRctLeafNodes[idx] = rln.RawData()
+	}
+
+	nonCanonicalRct1CID = nonCanonicalRctLeafNodeCids[0]
+	nonCanonicalRct2CID = nonCanonicalRctLeafNodeCids[1]
+
+	nonCanonicalRctLeaf1 = nonCanonicalRawRctLeafNodes[0]
+	nonCanonicalRctLeaf2 = nonCanonicalRawRctLeafNodes[1]
+
 	contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F"
 	contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B"
 	contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc"
@@ -173,3 +217,385 @@ func tearDown(t *testing.T) {
 	err := ind.Close()
 	require.NoError(t, err)
 }
+
+// TODO Refactor rest of the tests to avoid duplication
+// func testPublishAndIndexHeaderIPLDs() (t *testing.T) {
+
+// }
+
+func setupTestData(t *testing.T) {
+	var tx interfaces.Batch
+	tx, err = ind.PushBlock(
+		mockBlock,
+		mocks.MockReceipts,
+		mocks.MockBlock.Difficulty())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := tx.Submit(err); err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for _, node := range mocks.StateDiffs {
+		err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
+		require.NoError(t, err)
+	}
+
+	require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
+}
+
+func setupTestDataNonCanonical(t *testing.T) {
+	var tx1 interfaces.Batch
+
+	tx1, err = ind.PushBlock(
+		mockBlock,
+		mocks.MockReceipts,
+		mocks.MockBlock.Difficulty())
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, node := range mocks.StateDiffs {
+		err = ind.PushStateNode(tx1, node, mockBlock.Hash().String())
+		require.NoError(t, err)
+	}
+
+	require.Equal(t, mocks.BlockNumber.String(), tx1.(*sql.BatchTx).BlockNumber)
+	if err := tx1.Submit(err); err != nil {
+		t.Fatal(err)
+	}
+
+	var tx2 interfaces.Batch
+	tx2, err = ind.PushBlock(
+		mocks.MockNonCanonicalBlock,
+		mocks.MockNonCanonicalBlockReceipts,
+		mocks.MockNonCanonicalBlock.Difficulty())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	require.Equal(t, mocks.BlockNumber.String(), tx2.(*sql.BatchTx).BlockNumber)
+	if err := tx2.Submit(err); err != nil {
+		t.Fatal(err)
+	}
+
+	// TODO index state & storage nodes for second block
+}
+
+func testPublishAndIndexHeaderNonCanonical(t *testing.T) {
+	// check indexed headers
+	pgStr := `SELECT block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT),
+			tx_root, receipt_root, uncle_root, coinbase
+			FROM eth.header_cids
+			WHERE block_number = $1`
+	headerRes := make([]models.HeaderModel, 0)
+	err = db.Select(context.Background(), &headerRes, pgStr, mocks.BlockNumber.Uint64())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedRes := []models.HeaderModel{
+		{
+			BlockHash:       mockBlock.Hash().String(),
+			CID:             headerCID.String(),
+			TotalDifficulty: mockBlock.Difficulty().String(),
+			TxRoot:          mockBlock.TxHash().String(),
+			RctRoot:         mockBlock.ReceiptHash().String(),
+			UncleRoot:       mockBlock.UncleHash().String(),
+			Coinbase:        mocks.MockHeader.Coinbase.String(),
+		},
+		{
+			BlockHash:       mockNonCanonicalBlock.Hash().String(),
+			CID:             mockNonCanonicalHeaderCID.String(),
+			TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(),
+			TxRoot:          mockNonCanonicalBlock.TxHash().String(),
+			RctRoot:         mockNonCanonicalBlock.ReceiptHash().String(),
+			UncleRoot:       mockNonCanonicalBlock.UncleHash().String(),
+			Coinbase:        mocks.MockNonCanonicalHeader.Coinbase.String(),
+		},
+	}
+	expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String()
+	expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String()
+
+	require.Equal(t, len(expectedRes), len(headerRes))
+	require.ElementsMatch(t,
+		[]string{mockBlock.Hash().String(), mocks.MockNonCanonicalBlock.Hash().String()},
+		[]string{headerRes[0].BlockHash, headerRes[1].BlockHash},
+	)
+
+	if headerRes[0].BlockHash == mockBlock.Hash().String() {
+		require.Equal(t, expectedRes[0], headerRes[0])
+		require.Equal(t, expectedRes[1], headerRes[1])
+	} else {
+		require.Equal(t, expectedRes[1], headerRes[0])
+		require.Equal(t, expectedRes[0], headerRes[1])
+	}
+
+	// check indexed IPLD blocks
+	var data []byte
+	var prefixedKey string
+
+	prefixedKey = shared.MultihashKeyFromCID(headerCID)
+	err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
+	if err != nil {
+		t.Fatal(err)
+	}
+	require.Equal(t, mocks.MockHeaderRlp, data)
+
+	prefixedKey = shared.MultihashKeyFromCID(mockNonCanonicalHeaderCID)
+	err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
+	if err != nil {
+		t.Fatal(err)
+	}
+	require.Equal(t, mocks.MockNonCanonicalHeaderRlp, data)
+}
+
+func testPublishAndIndexTransactionsNonCanonical(t *testing.T) {
+	// check indexed transactions
+	pgStr := `SELECT header_id, tx_hash, cid, dst, src, index,
+		tx_data, tx_type, CAST(value as TEXT)
+		FROM eth.transaction_cids
+		WHERE block_number = $1`
+	txRes := make([]models.TxModel, 0)
+	err = db.Select(context.Background(), &txRes, pgStr, mocks.BlockNumber.Uint64())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mockBlockTxs := mocks.MockBlock.Transactions()
+	expectedBlockTxs := []models.TxModel{
+		{
+			HeaderID: mockBlock.Hash().String(),
+			TxHash:   mockBlockTxs[0].Hash().String(),
+			CID:      trx1CID.String(),
+			Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[0].To()),
+			Src:      mocks.SenderAddr.String(),
+			Index:    0,
+			Data:     mockBlockTxs[0].Data(),
+			Type:     mockBlockTxs[0].Type(),
+			Value:    mockBlockTxs[0].Value().String(),
+		},
+		{
+			HeaderID: mockBlock.Hash().String(),
+			TxHash:   mockBlockTxs[1].Hash().String(),
+			CID:      trx2CID.String(),
+			Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[1].To()),
+			Src:      mocks.SenderAddr.String(),
+			Index:    1,
+			Data:     mockBlockTxs[1].Data(),
+			Type:     mockBlockTxs[1].Type(),
+			Value:    mockBlockTxs[1].Value().String(),
+		},
+		{
+			HeaderID: mockBlock.Hash().String(),
+			TxHash:   mockBlockTxs[2].Hash().String(),
+			CID:      trx3CID.String(),
+			Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[2].To()),
+			Src:      mocks.SenderAddr.String(),
+			Index:    2,
+			Data:     mockBlockTxs[2].Data(),
+			Type:     mockBlockTxs[2].Type(),
+			Value:    mockBlockTxs[2].Value().String(),
+		},
+		{
+			HeaderID: mockBlock.Hash().String(),
+			TxHash:   mockBlockTxs[3].Hash().String(),
+			CID:      trx4CID.String(),
+			Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[3].To()),
+			Src:      mocks.SenderAddr.String(),
+			Index:    3,
+			Data:     mockBlockTxs[3].Data(),
+			Type:     mockBlockTxs[3].Type(),
+			Value:    mockBlockTxs[3].Value().String(),
+		},
+		{
+			HeaderID: mockBlock.Hash().String(),
+			TxHash:   mockBlockTxs[4].Hash().String(),
+			CID:      trx5CID.String(),
+			Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[4].To()),
+			Src:      mocks.SenderAddr.String(),
+			Index:    4,
+			Data:     mockBlockTxs[4].Data(),
+			Type:     mockBlockTxs[4].Type(),
+			Value:    mockBlockTxs[4].Value().String(),
+		},
+	}
+
+	mockNonCanonicalBlockTxs := mocks.MockNonCanonicalBlock.Transactions()
+	expectedNonCanonicalBlockTxs := []models.TxModel{
+		{
+			HeaderID: mockNonCanonicalBlock.Hash().String(),
+			TxHash:   mockNonCanonicalBlockTxs[0].Hash().String(),
+			CID:      trx2CID.String(),
+			Dst:      mockNonCanonicalBlockTxs[0].To().String(),
+			Src:      mocks.SenderAddr.String(),
+			Index:    0,
+			Data:     mockNonCanonicalBlockTxs[0].Data(),
+			Type:     mockNonCanonicalBlockTxs[0].Type(),
+			Value:    mockNonCanonicalBlockTxs[0].Value().String(),
+		},
+		{
+			HeaderID: mockNonCanonicalBlock.Hash().String(),
+			TxHash:   mockNonCanonicalBlockTxs[1].Hash().String(),
+			CID:      trx5CID.String(),
+			Dst:      mockNonCanonicalBlockTxs[1].To().String(),
+			Src:      mocks.SenderAddr.String(),
+			Index:    1,
+			Data:     mockNonCanonicalBlockTxs[1].Data(),
+			Type:     mockNonCanonicalBlockTxs[1].Type(),
+			Value:    mockNonCanonicalBlockTxs[1].Value().String(),
+		},
+	}
+
+	require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs), len(txRes))
+
+	// sort results such that non-canonical block reciepts come after canonical block ones
+	sort.Slice(txRes, func(i, j int) bool {
+		if txRes[i].HeaderID == txRes[j].HeaderID {
+			return txRes[i].Index < txRes[j].Index
+		} else if txRes[i].HeaderID == mockBlock.Hash().String() {
+			return true
+		} else {
+			return false
+		}
+	})
+
+	for i, expectedTx := range expectedBlockTxs {
+		require.Equal(t, expectedTx, txRes[i])
+	}
+	for i, expectedTx := range expectedNonCanonicalBlockTxs {
+		require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i])
+	}
+
+	// check indexed IPLD blocks
+	var data []byte
+	var prefixedKey string
+
+	txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID}
+	txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5}
+	for i, txCID := range txCIDs {
+		prefixedKey = shared.MultihashKeyFromCID(txCID)
+		err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
+		if err != nil {
+			t.Fatal(err)
+		}
+		require.Equal(t, txRLPs[i], data)
+	}
+}
+
+func testPublishAndIndexReceiptsNonCanonical(t *testing.T) {
+	// check indexed receipts
+	pgStr := `SELECT header_id, tx_id, leaf_cid, leaf_mh_key, post_status, post_state, contract, contract_hash, log_root
+		FROM eth.receipt_cids
+		WHERE block_number = $1`
+	rctRes := make([]models.ReceiptModel, 0)
+	err = db.Select(context.Background(), &rctRes, pgStr, mocks.BlockNumber.Uint64())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mockBlockRcts := mocks.MockReceipts
+	rctCids := []cid.Cid{rct1CID, rct2CID, rct3CID, rct4CID, rct5CID}
+	expectedBlockRcts := make(map[string]models.ReceiptModel, len(mockBlockRcts))
+	for i, mockBlockRct := range mockBlockRcts {
+		rctModel := models.ReceiptModel{
+			HeaderID:  mockBlock.Hash().String(),
+			TxID:      mockBlockRct.TxHash.String(),
+			LeafCID:   rctCids[i].String(),
+			LeafMhKey: shared.MultihashKeyFromCID(rctCids[i]),
+			LogRoot:   mockBlockRct.LogRoot.String(),
+		}
+
+		contract := shared.HandleZeroAddr(mockBlockRct.ContractAddress)
+		rctModel.Contract = contract
+		if contract != "" {
+			rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
+		}
+
+		if len(mockBlockRct.PostState) == 0 {
+			rctModel.PostStatus = mockBlockRct.Status
+		} else {
+			rctModel.PostState = common.Bytes2Hex(mockBlockRct.PostState)
+		}
+
+		expectedBlockRcts[rctCids[i].String()] = rctModel
+	}
+
+	mockNonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts
+	nonCanonicalRctCids := []cid.Cid{nonCanonicalRct1CID, nonCanonicalRct2CID}
+	expectedNonCanonicalBlockRcts := make(map[string]models.ReceiptModel, len(mockNonCanonicalBlockRcts))
+	for i, mockNonCanonicalBlockRct := range mockNonCanonicalBlockRcts {
+		rctModel := models.ReceiptModel{
+			HeaderID:  mockNonCanonicalBlock.Hash().String(),
+			TxID:      mockNonCanonicalBlockRct.TxHash.String(),
+			LeafCID:   nonCanonicalRctCids[i].String(),
+			LeafMhKey: shared.MultihashKeyFromCID(nonCanonicalRctCids[i]),
+			LogRoot:   mockNonCanonicalBlockRct.LogRoot.String(),
+		}
+
+		contract := shared.HandleZeroAddr(mockNonCanonicalBlockRct.ContractAddress)
+		rctModel.Contract = contract
+		if contract != "" {
+			rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
+		}
+
+		if len(mockNonCanonicalBlockRct.PostState) == 0 {
+			rctModel.PostStatus = mockNonCanonicalBlockRct.Status
+		} else {
+			rctModel.PostState = common.Bytes2Hex(mockNonCanonicalBlockRct.PostState)
+		}
+
+		expectedBlockRcts[nonCanonicalRctCids[i].String()] = rctModel
+	}
+
+	require.Equal(t, len(expectedBlockRcts)+len(expectedNonCanonicalBlockRcts), len(rctRes))
+
+	// sort results such that non-canonical block reciepts come after canonical block ones
+	sort.Slice(rctRes, func(i, j int) bool {
+		if rctRes[i].HeaderID == rctRes[j].HeaderID {
+			return false
+		} else {
+			if rctRes[i].HeaderID == mockBlock.Hash().String() {
+				return true
+			} else {
+				return false
+			}
+		}
+	})
+
+	for i := 0; i < len(expectedBlockRcts); i++ {
+		rct := rctRes[i]
+		require.Contains(t, expectedBlockRcts, rct.LeafCID)
+		require.Equal(t, expectedBlockRcts[rct.LeafCID], rct)
+	}
+
+	for i := 0; i < len(expectedNonCanonicalBlockRcts); i++ {
+		rct := rctRes[len(expectedBlockRcts)+i]
+		require.Contains(t, expectedNonCanonicalBlockRcts, rct.LeafCID)
+		require.Equal(t, expectedNonCanonicalBlockRcts[rct.LeafCID], rct)
+	}
+
+	// check indexed rct IPLD blocks
+	var data []byte
+	var prefixedKey string
+
+	rctRLPs := [][]byte{rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5}
+	for i, rctCid := range rctCids {
+		prefixedKey = shared.MultihashKeyFromCID(rctCid)
+		err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
+		if err != nil {
+			t.Fatal(err)
+		}
+		require.Equal(t, rctRLPs[i], data)
+	}
+
+	nonCanonicalrctRLPs := [][]byte{nonCanonicalRctLeaf1, nonCanonicalRctLeaf2}
+	for i, rctCid := range nonCanonicalRctCids {
+		prefixedKey = shared.MultihashKeyFromCID(rctCid)
+		err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
+		if err != nil {
+			t.Fatal(err)
+		}
+		require.Equal(t, nonCanonicalrctRLPs[i], data)
+	}
+}

statediff/indexer/database/sql/pgx_indexer_test.go

@@ -19,7 +19,6 @@ package sql_test
 import (
 	"context"
 	"math/big"
-	"sort"
 	"testing"
 
 	"github.com/ipfs/go-cid"
@@ -32,7 +31,6 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
 	"github.com/ethereum/go-ethereum/statediff/indexer/models"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
@@ -51,63 +49,12 @@ func setupPGXIndexer(t *testing.T) {
 
 func setupPGX(t *testing.T) {
 	setupPGXIndexer(t)
-	var tx interfaces.Batch
+	setupTestData(t)
-	tx, err = ind.PushBlock(
-		mockBlock,
-		mocks.MockReceipts,
-		mocks.MockBlock.Difficulty())
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		if err := tx.Submit(err); err != nil {
-			t.Fatal(err)
-		}
-	}()
-	for _, node := range mocks.StateDiffs {
-		err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
-		require.NoError(t, err)
-	}
-
-	require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
 }
 
 func setupPGXNonCanonical(t *testing.T) {
 	setupPGXIndexer(t)
-	var tx1 interfaces.Batch
+	setupTestDataNonCanonical(t)
-
-	tx1, err = ind.PushBlock(
-		mockBlock,
-		mocks.MockReceipts,
-		mocks.MockBlock.Difficulty())
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, node := range mocks.StateDiffs {
-		err = ind.PushStateNode(tx1, node, mockBlock.Hash().String())
-		require.NoError(t, err)
-	}
-
-	require.Equal(t, mocks.BlockNumber.String(), tx1.(*sql.BatchTx).BlockNumber)
-	if err := tx1.Submit(err); err != nil {
-		t.Fatal(err)
-	}
-
-	var tx2 interfaces.Batch
-	tx2, err = ind.PushBlock(
-		mocks.MockNonCanonicalBlock,
-		mocks.MockNonCanonicalBlockReceipts,
-		mocks.MockNonCanonicalBlock.Difficulty())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	require.Equal(t, mocks.BlockNumber.String(), tx2.(*sql.BatchTx).BlockNumber)
-	if err := tx2.Submit(err); err != nil {
-		t.Fatal(err)
-	}
-
-	// TODO index state & storage nodes for second block
 }
 
 func TestPGXIndexer(t *testing.T) {
@@ -376,7 +323,7 @@ func TestPGXIndexer(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			// Decode the log leaf node.
+			// Decode the receipt leaf node.
 			var nodeElements []interface{}
 			err = rlp.DecodeBytes(result[0].Data, &nodeElements)
 			require.NoError(t, err)
@@ -644,77 +591,13 @@ func TestPGXIndexer(t *testing.T) {
 	})
 }
 
-func XTestPGXIndexerNonCanonical(t *testing.T) {
+func TestPGXIndexerNonCanonical(t *testing.T) {
 	t.Run("Publish and index header", func(t *testing.T) {
 		setupPGXNonCanonical(t)
 		defer tearDown(t)
 		defer checkTxClosure(t, 1, 0, 1)
 
-		// check indexed headers
+		testPublishAndIndexHeaderNonCanonical(t)
-		pgStr := `SELECT block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT),
-		tx_root, receipt_root, uncle_root, coinbase
-		FROM eth.header_cids
-		WHERE block_number = $1`
-		headerRes := make([]models.HeaderModel, 0)
-		err = db.Select(context.Background(), &headerRes, pgStr, mocks.BlockNumber.Uint64())
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		expectedRes := []models.HeaderModel{
-			{
-				BlockHash:       mockBlock.Hash().String(),
-				CID:             headerCID.String(),
-				TotalDifficulty: mockBlock.Difficulty().String(),
-				TxRoot:          mockBlock.TxHash().String(),
-				RctRoot:         mockBlock.ReceiptHash().String(),
-				UncleRoot:       mockBlock.UncleHash().String(),
-				Coinbase:        mocks.MockHeader.Coinbase.String(),
-			},
-			{
-				BlockHash:       mockNonCanonicalBlock.Hash().String(),
-				CID:             mockNonCanonicalHeaderCID.String(),
-				TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(),
-				TxRoot:          mockNonCanonicalBlock.TxHash().String(),
-				RctRoot:         mockNonCanonicalBlock.ReceiptHash().String(),
-				UncleRoot:       mockNonCanonicalBlock.UncleHash().String(),
-				Coinbase:        mocks.MockNonCanonicalHeader.Coinbase.String(),
-			},
-		}
-		expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String()
-		expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String()
-
-		require.Equal(t, len(expectedRes), len(headerRes))
-		require.ElementsMatch(t,
-			[]string{mockBlock.Hash().String(), mocks.MockNonCanonicalBlock.Hash().String()},
-			[]string{headerRes[0].BlockHash, headerRes[1].BlockHash},
-		)
-
-		if headerRes[0].BlockHash == mockBlock.Hash().String() {
-			require.Equal(t, expectedRes[0], headerRes[0])
-			require.Equal(t, expectedRes[1], headerRes[1])
-		} else {
-			require.Equal(t, expectedRes[1], headerRes[0])
-			require.Equal(t, expectedRes[0], headerRes[1])
-		}
-
-		// check indexed IPLD blocks
-		var data []byte
-		var prefixedKey string
-
-		prefixedKey = shared.MultihashKeyFromCID(headerCID)
-		err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
-		if err != nil {
-			t.Fatal(err)
-		}
-		require.Equal(t, mocks.MockHeaderRlp, data)
-
-		prefixedKey = shared.MultihashKeyFromCID(mockNonCanonicalHeaderCID)
-		err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
-		if err != nil {
-			t.Fatal(err)
-		}
-		require.Equal(t, mocks.MockNonCanonicalHeaderRlp, data)
 	})
 
 	t.Run("Publish and index transactions", func(t *testing.T) {
@@ -722,134 +605,15 @@ func XTestPGXIndexerNonCanonical(t *testing.T) {
 		defer tearDown(t)
 		defer checkTxClosure(t, 1, 0, 1)
 
-		// check indexed transactions
+		testPublishAndIndexTransactionsNonCanonical(t)
-		pgStr := `SELECT header_id, tx_hash, cid, dst, src, index,
+	})
-		tx_data, tx_type, CAST(value as TEXT)
-		FROM eth.transaction_cids
-		WHERE block_number = $1`
-		txRes := make([]models.TxModel, 0)
-		err = db.Select(context.Background(), &txRes, pgStr, mocks.BlockNumber.Uint64())
-		if err != nil {
-			t.Fatal(err)
-		}
 
-		mockBlockTxs := mocks.MockBlock.Transactions()
+	t.Run("Publish and index receipts", func(t *testing.T) {
-		expectedBlockTxs := []models.TxModel{
+		setupPGXNonCanonical(t)
-			{
+		defer tearDown(t)
-				HeaderID: mockBlock.Hash().String(),
+		defer checkTxClosure(t, 1, 0, 1)
-				TxHash:   mockBlockTxs[0].Hash().String(),
-				CID:      trx1CID.String(),
-				Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[0].To()),
-				Src:      mocks.SenderAddr.String(),
-				Index:    0,
-				Data:     mockBlockTxs[0].Data(),
-				Type:     mockBlockTxs[0].Type(),
-				Value:    mockBlockTxs[0].Value().String(),
-			},
-			{
-				HeaderID: mockBlock.Hash().String(),
-				TxHash:   mockBlockTxs[1].Hash().String(),
-				CID:      trx2CID.String(),
-				Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[1].To()),
-				Src:      mocks.SenderAddr.String(),
-				Index:    1,
-				Data:     mockBlockTxs[1].Data(),
-				Type:     mockBlockTxs[1].Type(),
-				Value:    mockBlockTxs[1].Value().String(),
-			},
-			{
-				HeaderID: mockBlock.Hash().String(),
-				TxHash:   mockBlockTxs[2].Hash().String(),
-				CID:      trx3CID.String(),
-				Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[2].To()),
-				Src:      mocks.SenderAddr.String(),
-				Index:    2,
-				Data:     mockBlockTxs[2].Data(),
-				Type:     mockBlockTxs[2].Type(),
-				Value:    mockBlockTxs[2].Value().String(),
-			},
-			{
-				HeaderID: mockBlock.Hash().String(),
-				TxHash:   mockBlockTxs[3].Hash().String(),
-				CID:      trx4CID.String(),
-				Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[3].To()),
-				Src:      mocks.SenderAddr.String(),
-				Index:    3,
-				Data:     mockBlockTxs[3].Data(),
-				Type:     mockBlockTxs[3].Type(),
-				Value:    mockBlockTxs[3].Value().String(),
-			},
-			{
-				HeaderID: mockBlock.Hash().String(),
-				TxHash:   mockBlockTxs[4].Hash().String(),
-				CID:      trx5CID.String(),
-				Dst:      shared.HandleZeroAddrPointer(mockBlockTxs[4].To()),
-				Src:      mocks.SenderAddr.String(),
-				Index:    4,
-				Data:     mockBlockTxs[4].Data(),
-				Type:     mockBlockTxs[4].Type(),
-				Value:    mockBlockTxs[4].Value().String(),
-			},
-		}
 
-		mockNonCanonicalBlockTxs := mocks.MockNonCanonicalBlock.Transactions()
+		testPublishAndIndexReceiptsNonCanonical(t)
-		expectedNonCanonicalBlockTxs := []models.TxModel{
-			{
-				HeaderID: mockNonCanonicalBlock.Hash().String(),
-				TxHash:   mockNonCanonicalBlockTxs[0].Hash().String(),
-				CID:      trx2CID.String(),
-				Dst:      mockNonCanonicalBlockTxs[0].To().String(),
-				Src:      mocks.SenderAddr.String(),
-				Index:    0,
-				Data:     mockNonCanonicalBlockTxs[0].Data(),
-				Type:     mockNonCanonicalBlockTxs[0].Type(),
-				Value:    mockNonCanonicalBlockTxs[0].Value().String(),
-			},
-			{
-				HeaderID: mockNonCanonicalBlock.Hash().String(),
-				TxHash:   mockNonCanonicalBlockTxs[1].Hash().String(),
-				CID:      trx5CID.String(),
-				Dst:      mockNonCanonicalBlockTxs[1].To().String(),
-				Src:      mocks.SenderAddr.String(),
-				Index:    1,
-				Data:     mockNonCanonicalBlockTxs[1].Data(),
-				Type:     mockNonCanonicalBlockTxs[1].Type(),
-				Value:    mockNonCanonicalBlockTxs[1].Value().String(),
-			},
-		}
-
-		require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs), len(txRes))
-		sort.Slice(txRes, func(i, j int) bool {
-			if txRes[i].HeaderID == txRes[j].HeaderID {
-				return txRes[i].Index < txRes[j].Index
-			} else if txRes[i].HeaderID == mockBlock.Hash().String() {
-				return true
-			} else {
-				return false
-			}
-		})
-
-		for i, expectedTx := range expectedBlockTxs {
-			require.Equal(t, expectedTx, txRes[i])
-		}
-		for i, expectedTx := range expectedNonCanonicalBlockTxs {
-			require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i])
-		}
-
-		// check indexed IPLD blocks
-		var data []byte
-		var prefixedKey string
-
-		txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID}
-		txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5}
-		for i, txCID := range txCIDs {
-			prefixedKey = shared.MultihashKeyFromCID(txCID)
-			err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
-			if err != nil {
-				t.Fatal(err)
-			}
-			require.Equal(t, txRLPs[i], data)
-		}
 	})
 }
 

statediff/indexer/database/sql/sqlx_indexer_test.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
 	"github.com/ethereum/go-ethereum/statediff/indexer/models"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
@@ -51,25 +50,12 @@ func setupSQLXIndexer(t *testing.T) {
 
 func setupSQLX(t *testing.T) {
 	setupSQLXIndexer(t)
-	var tx interfaces.Batch
+	setupTestData(t)
-	tx, err = ind.PushBlock(
+}
-		mockBlock,
-		mocks.MockReceipts,
-		mocks.MockBlock.Difficulty())
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		if err := tx.Submit(err); err != nil {
-			t.Fatal(err)
-		}
-	}()
-	for _, node := range mocks.StateDiffs {
-		err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
-		require.NoError(t, err)
-	}
 
-	require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
+func setupSQLXNonCanonical(t *testing.T) {
+	setupPGXIndexer(t)
+	setupTestDataNonCanonical(t)
 }
 
 func TestSQLXIndexer(t *testing.T) {
@@ -331,7 +317,7 @@ func TestSQLXIndexer(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			// Decode the log leaf node.
+			// Decode the receipt leaf node.
 			var nodeElements []interface{}
 			err = rlp.DecodeBytes(result[0].Data, &nodeElements)
 			require.NoError(t, err)
@@ -598,6 +584,32 @@ func TestSQLXIndexer(t *testing.T) {
 	})
 }
 
+func TestSQLXIndexerNonCanonical(t *testing.T) {
+	t.Run("Publish and index header", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		testPublishAndIndexHeaderNonCanonical(t)
+	})
+
+	t.Run("Publish and index transactions", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		testPublishAndIndexTransactionsNonCanonical(t)
+	})
+
+	t.Run("Publish and index receipts", func(t *testing.T) {
+		setupSQLXNonCanonical(t)
+		defer tearDown(t)
+		defer checkTxClosure(t, 1, 0, 1)
+
+		testPublishAndIndexReceiptsNonCanonical(t)
+	})
+}
+
 func TestSQLXWatchAddressMethods(t *testing.T) {
 	setupSQLXIndexer(t)
 	defer tearDown(t)