From f8c715fedd1d924d0e9c5a5b71f40d32e4f2939a Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Mon, 4 Jul 2022 16:13:13 +0530 Subject: [PATCH 1/9] Add indexer tests for header and transactions in a non canonical block --- .../database/sql/indexer_shared_test.go | 21 +- .../indexer/database/sql/pgx_indexer_test.go | 250 +++++++++++++++++- statediff/indexer/mocks/test_data.go | 75 ++++-- 3 files changed, 319 insertions(+), 27 deletions(-) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 197fe02b1..9d34d7fe2 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -24,15 +24,15 @@ var ( ind interfaces.StateDiffIndexer ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` - tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte - mockBlock *types.Block - headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid - rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid - rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte - state1CID, state2CID, storageCID cid.Cid - contract1Address, contract2Address, contract3Address, contract4Address string - contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 - lastFilledAt, watchedAt1, watchedAt2, watchedAt3 uint64 + tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + mockBlock, mockNonCanonicalBlock *types.Block + headerCID, mockNonCanonicalHeaderCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid + rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte + state1CID, state2CID, storageCID cid.Cid + contract1Address, contract2Address, contract3Address, contract4Address string + contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 + lastFilledAt, watchedAt1, watchedAt2, watchedAt3 uint64 ) func init() { @@ -44,6 +44,8 @@ func init() { mockBlock = mocks.MockBlock txs, rcts := mocks.MockBlock.Transactions(), mocks.MockReceipts + mockNonCanonicalBlock = mocks.MockNonCanonicalBlock + buf := new(bytes.Buffer) txs.EncodeIndex(0, buf) tx1 = make([]byte, buf.Len()) @@ -96,6 +98,7 @@ func init() { buf.Reset() headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256) + mockNonCanonicalHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeaderRlp, multihash.KECCAK_256) trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256) trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx2, multihash.KECCAK_256) trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256) diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index c304a4871..22d7123a0 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -19,6 +19,7 @@ package sql_test import ( "context" "math/big" + "sort" "testing" "github.com/ipfs/go-cid" @@ -71,6 +72,44 @@ func setupPGX(t *testing.T) { require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) } +func setupPGXNonCanonical(t *testing.T) { + setupPGXIndexer(t) + var tx1 interfaces.Batch + + tx1, err = ind.PushBlock( + mockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx1, node, mockBlock.Hash().String()) + require.NoError(t, err) + } + + require.Equal(t, mocks.BlockNumber.String(), tx1.(*sql.BatchTx).BlockNumber) + if err := tx1.Submit(err); err != nil { + t.Fatal(err) + } + + var tx2 interfaces.Batch + tx2, err = ind.PushBlock( + mocks.MockNonCanonicalBlock, + mocks.MockNonCanonicalBlockReceipts, + mocks.MockNonCanonicalBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, mocks.BlockNumber.String(), tx2.(*sql.BatchTx).BlockNumber) + if err := tx2.Submit(err); err != nil { + t.Fatal(err) + } + + // TODO index state & storage nodes for second block +} + func TestPGXIndexer(t *testing.T) { t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { setupPGX(t) @@ -258,7 +297,7 @@ func TestPGXIndexer(t *testing.T) { AND header_cids.block_number = $1 ORDER BY transaction_cids.index` logsPgStr := `SELECT log_cids.index, log_cids.address, log_cids.topic0, log_cids.topic1, data FROM eth.log_cids - INNER JOIN eth.receipt_cids ON (log_cids.rct_id = receipt_cids.tx_id) + INNER JOIN eth.receipt_cids ON (log_cids.rct_id = receipt_cids.tx_id) INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key) WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC` err = db.Select(context.Background(), &rcts, rctsPgStr, mocks.BlockNumber.Uint64()) @@ -605,6 +644,215 @@ func TestPGXIndexer(t *testing.T) { }) } +func XTestPGXIndexerNonCanonical(t *testing.T) { + t.Run("Publish and index header", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + // check indexed headers + pgStr := `SELECT block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT), + tx_root, receipt_root, uncle_root, coinbase + FROM eth.header_cids + WHERE block_number = $1` + headerRes := make([]models.HeaderModel, 0) + err = db.Select(context.Background(), &headerRes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + expectedRes := []models.HeaderModel{ + { + BlockHash: mockBlock.Hash().String(), + CID: headerCID.String(), + TotalDifficulty: mockBlock.Difficulty().String(), + TxRoot: mockBlock.TxHash().String(), + RctRoot: mockBlock.ReceiptHash().String(), + UncleRoot: mockBlock.UncleHash().String(), + Coinbase: mocks.MockHeader.Coinbase.String(), + }, + { + BlockHash: mockNonCanonicalBlock.Hash().String(), + CID: mockNonCanonicalHeaderCID.String(), + TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(), + TxRoot: mockNonCanonicalBlock.TxHash().String(), + RctRoot: mockNonCanonicalBlock.ReceiptHash().String(), + UncleRoot: mockNonCanonicalBlock.UncleHash().String(), + Coinbase: mocks.MockNonCanonicalHeader.Coinbase.String(), + }, + } + expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String() + expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String() + + require.Equal(t, len(expectedRes), len(headerRes)) + require.ElementsMatch(t, + []string{mockBlock.Hash().String(), mocks.MockNonCanonicalBlock.Hash().String()}, + []string{headerRes[0].BlockHash, headerRes[1].BlockHash}, + ) + + if headerRes[0].BlockHash == mockBlock.Hash().String() { + require.Equal(t, expectedRes[0], headerRes[0]) + require.Equal(t, expectedRes[1], headerRes[1]) + } else { + require.Equal(t, expectedRes[1], headerRes[0]) + require.Equal(t, expectedRes[0], headerRes[1]) + } + + // check indexed IPLD blocks + var data []byte + var prefixedKey string + + prefixedKey = shared.MultihashKeyFromCID(headerCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.MockHeaderRlp, data) + + prefixedKey = shared.MultihashKeyFromCID(mockNonCanonicalHeaderCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.MockNonCanonicalHeaderRlp, data) + }) + + t.Run("Publish and index transactions", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + // check indexed transactions + pgStr := `SELECT header_id, tx_hash, cid, dst, src, index, + tx_data, tx_type, CAST(value as TEXT) + FROM eth.transaction_cids + WHERE block_number = $1` + txRes := make([]models.TxModel, 0) + err = db.Select(context.Background(), &txRes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + mockBlockTxs := mocks.MockBlock.Transactions() + expectedBlockTxs := []models.TxModel{ + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[0].Hash().String(), + CID: trx1CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockBlockTxs[0].Data(), + Type: mockBlockTxs[0].Type(), + Value: mockBlockTxs[0].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[1].Hash().String(), + CID: trx2CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockBlockTxs[1].Data(), + Type: mockBlockTxs[1].Type(), + Value: mockBlockTxs[1].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[2].Hash().String(), + CID: trx3CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()), + Src: mocks.SenderAddr.String(), + Index: 2, + Data: mockBlockTxs[2].Data(), + Type: mockBlockTxs[2].Type(), + Value: mockBlockTxs[2].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[3].Hash().String(), + CID: trx4CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()), + Src: mocks.SenderAddr.String(), + Index: 3, + Data: mockBlockTxs[3].Data(), + Type: mockBlockTxs[3].Type(), + Value: mockBlockTxs[3].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[4].Hash().String(), + CID: trx5CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()), + Src: mocks.SenderAddr.String(), + Index: 4, + Data: mockBlockTxs[4].Data(), + Type: mockBlockTxs[4].Type(), + Value: mockBlockTxs[4].Value().String(), + }, + } + + mockNonCanonicalBlockTxs := mocks.MockNonCanonicalBlock.Transactions() + expectedNonCanonicalBlockTxs := []models.TxModel{ + { + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[0].Hash().String(), + CID: trx2CID.String(), + Dst: mockNonCanonicalBlockTxs[0].To().String(), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockNonCanonicalBlockTxs[0].Data(), + Type: mockNonCanonicalBlockTxs[0].Type(), + Value: mockNonCanonicalBlockTxs[0].Value().String(), + }, + { + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[1].Hash().String(), + CID: trx5CID.String(), + Dst: mockNonCanonicalBlockTxs[1].To().String(), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockNonCanonicalBlockTxs[1].Data(), + Type: mockNonCanonicalBlockTxs[1].Type(), + Value: mockNonCanonicalBlockTxs[1].Value().String(), + }, + } + + require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs), len(txRes)) + sort.Slice(txRes, func(i, j int) bool { + if txRes[i].HeaderID == txRes[j].HeaderID { + return txRes[i].Index < txRes[j].Index + } else if txRes[i].HeaderID == mockBlock.Hash().String() { + return true + } else { + return false + } + }) + + for i, expectedTx := range expectedBlockTxs { + require.Equal(t, expectedTx, txRes[i]) + } + for i, expectedTx := range expectedNonCanonicalBlockTxs { + require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i]) + } + + // check indexed IPLD blocks + var data []byte + var prefixedKey string + + txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID} + txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5} + for i, txCID := range txCIDs { + prefixedKey = shared.MultihashKeyFromCID(txCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, txRLPs[i], data) + } + }) +} + func TestPGXWatchAddressMethods(t *testing.T) { setupPGXIndexer(t) defer tearDown(t) diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index 3a8226656..b353a61e2 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -51,23 +51,30 @@ var ( BaseFee: big.NewInt(params.InitialBaseFee), Coinbase: common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476777"), } - MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts(TestConfig, BlockNumber) - MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) - MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) - Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") - AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") - ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) - ContractAddress2 = crypto.CreateAddress(SenderAddr, MockTransactions[3].Nonce()) - MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} - mockTopic11 = common.HexToHash("0x04") - mockTopic12 = common.HexToHash("0x06") - mockTopic21 = common.HexToHash("0x05") - mockTopic22 = common.HexToHash("0x07") - ExpectedPostStatus uint64 = 1 - ExpectedPostState1 = common.Bytes2Hex(common.HexToHash("0x1").Bytes()) - ExpectedPostState2 = common.Bytes2Hex(common.HexToHash("0x2").Bytes()) - ExpectedPostState3 = common.Bytes2Hex(common.HexToHash("0x3").Bytes()) - MockLog1 = &types.Log{ + MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts(TestConfig, BlockNumber) + MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) + MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) + + MockNonCanonicalHeader = MockHeader + MockNonCanonicalBlockTransactions = types.Transactions{MockTransactions[1], MockTransactions[4]} + MockNonCanonicalBlockReceipts = createNonCanonicalBlockReceipts(TestConfig, BlockNumber, MockNonCanonicalBlockTransactions) + MockNonCanonicalBlock = types.NewBlock(&MockNonCanonicalHeader, MockNonCanonicalBlockTransactions, nil, MockNonCanonicalBlockReceipts, new(trie.Trie)) + MockNonCanonicalHeaderRlp, _ = rlp.EncodeToBytes(MockNonCanonicalBlock.Header()) + + Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") + ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractAddress2 = crypto.CreateAddress(SenderAddr, MockTransactions[3].Nonce()) + MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} + mockTopic11 = common.HexToHash("0x04") + mockTopic12 = common.HexToHash("0x06") + mockTopic21 = common.HexToHash("0x05") + mockTopic22 = common.HexToHash("0x07") + ExpectedPostStatus uint64 = 1 + ExpectedPostState1 = common.Bytes2Hex(common.HexToHash("0x1").Bytes()) + ExpectedPostState2 = common.Bytes2Hex(common.HexToHash("0x2").Bytes()) + ExpectedPostState3 = common.Bytes2Hex(common.HexToHash("0x3").Bytes()) + MockLog1 = &types.Log{ Address: Address, Topics: []common.Hash{mockTopic11, mockTopic12}, Data: []byte{}, @@ -440,3 +447,37 @@ func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big. return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4, signedTrx5}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4, mockReceipt5}, senderAddr } + +func createNonCanonicalBlockReceipts(config *params.ChainConfig, blockNumber *big.Int, transactions types.Transactions) types.Receipts { + transactionSigner := types.MakeSigner(config, blockNumber) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Crit(err.Error()) + } + + signedTrx0, err := types.SignTx(transactions[0], transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + signedTrx1, err := types.SignTx(transactions[1], transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + mockReceipt0 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) + mockReceipt0.Logs = []*types.Log{MockLog2, ShortLog1} + mockReceipt0.TxHash = signedTrx0.Hash() + + mockReceipt1 := &types.Receipt{ + Type: types.DynamicFeeTxType, + PostState: common.HexToHash("0x3").Bytes(), + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 175, + Logs: []*types.Log{}, + TxHash: signedTrx1.Hash(), + } + + return types.Receipts{mockReceipt0, mockReceipt1} +} -- 2.45.2 From 97ed3c847c79ab16dc39452822f0112398a76493 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 9 Aug 2022 13:12:32 +0530 Subject: [PATCH 2/9] Add indexer tests for receipts in a non-canonical block and refactor --- .../database/sql/indexer_shared_test.go | 426 ++++++++++++++++++ .../indexer/database/sql/pgx_indexer_test.go | 260 +---------- .../indexer/database/sql/sqlx_indexer_test.go | 52 ++- 3 files changed, 470 insertions(+), 268 deletions(-) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 9d34d7fe2..57ee6478f 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -2,20 +2,26 @@ package sql_test import ( "bytes" + "context" "fmt" "os" + "sort" "testing" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" ) var ( @@ -25,10 +31,13 @@ var ( ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + nonCanonicalRct1, nonCanonicalRct2 []byte mockBlock, mockNonCanonicalBlock *types.Block headerCID, mockNonCanonicalHeaderCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + nonCanonicalRct1CID, nonCanonicalRct2CID cid.Cid rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte + nonCanonicalRctLeaf1, nonCanonicalRctLeaf2 []byte state1CID, state2CID, storageCID cid.Cid contract1Address, contract2Address, contract3Address, contract4Address string contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 @@ -45,6 +54,7 @@ func init() { txs, rcts := mocks.MockBlock.Transactions(), mocks.MockReceipts mockNonCanonicalBlock = mocks.MockNonCanonicalBlock + nonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts buf := new(bytes.Buffer) txs.EncodeIndex(0, buf) @@ -97,6 +107,16 @@ func init() { copy(rct5, buf.Bytes()) buf.Reset() + nonCanonicalBlockRcts.EncodeIndex(0, buf) + nonCanonicalRct1 = make([]byte, buf.Len()) + copy(nonCanonicalRct1, buf.Bytes()) + buf.Reset() + + nonCanonicalBlockRcts.EncodeIndex(1, buf) + nonCanonicalRct2 = make([]byte, buf.Len()) + copy(nonCanonicalRct2, buf.Bytes()) + buf.Reset() + headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256) mockNonCanonicalHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeaderRlp, multihash.KECCAK_256) trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256) @@ -141,6 +161,30 @@ func init() { rctLeaf4 = orderedRctLeafNodes[3] rctLeaf5 = orderedRctLeafNodes[4] + nonCanonicalRctTrie := ipld.NewRctTrie() + + nonCanonicalRctTrie.Add(0, nonCanonicalRct1) + nonCanonicalRctTrie.Add(1, nonCanonicalRct2) + + nonCanonicalRctLeafNodes, keys, _ := nonCanonicalRctTrie.GetLeafNodes() + + nonCanonicalRctLeafNodeCids := make([]cid.Cid, len(nonCanonicalRctLeafNodes)) + nonCanonicalRawRctLeafNodes := make([][]byte, len(nonCanonicalRctLeafNodes)) + for i, rln := range nonCanonicalRctLeafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + rlp.Decode(r, &idx) + nonCanonicalRctLeafNodeCids[idx] = rln.Cid() + nonCanonicalRawRctLeafNodes[idx] = rln.RawData() + } + + nonCanonicalRct1CID = nonCanonicalRctLeafNodeCids[0] + nonCanonicalRct2CID = nonCanonicalRctLeafNodeCids[1] + + nonCanonicalRctLeaf1 = nonCanonicalRawRctLeafNodes[0] + nonCanonicalRctLeaf2 = nonCanonicalRawRctLeafNodes[1] + contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" contract3Address = "0xcfeB164C328CA13EFd3C77E1980d94975aDfedfc" @@ -173,3 +217,385 @@ func tearDown(t *testing.T) { err := ind.Close() require.NoError(t, err) } + +// TODO Refactor rest of the tests to avoid duplication +// func testPublishAndIndexHeaderIPLDs() (t *testing.T) { + +// } + +func setupTestData(t *testing.T) { + var tx interfaces.Batch + tx, err = ind.PushBlock( + mockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := tx.Submit(err); err != nil { + t.Fatal(err) + } + }() + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx, node, mockBlock.Hash().String()) + require.NoError(t, err) + } + + require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) +} + +func setupTestDataNonCanonical(t *testing.T) { + var tx1 interfaces.Batch + + tx1, err = ind.PushBlock( + mockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx1, node, mockBlock.Hash().String()) + require.NoError(t, err) + } + + require.Equal(t, mocks.BlockNumber.String(), tx1.(*sql.BatchTx).BlockNumber) + if err := tx1.Submit(err); err != nil { + t.Fatal(err) + } + + var tx2 interfaces.Batch + tx2, err = ind.PushBlock( + mocks.MockNonCanonicalBlock, + mocks.MockNonCanonicalBlockReceipts, + mocks.MockNonCanonicalBlock.Difficulty()) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, mocks.BlockNumber.String(), tx2.(*sql.BatchTx).BlockNumber) + if err := tx2.Submit(err); err != nil { + t.Fatal(err) + } + + // TODO index state & storage nodes for second block +} + +func testPublishAndIndexHeaderNonCanonical(t *testing.T) { + // check indexed headers + pgStr := `SELECT block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT), + tx_root, receipt_root, uncle_root, coinbase + FROM eth.header_cids + WHERE block_number = $1` + headerRes := make([]models.HeaderModel, 0) + err = db.Select(context.Background(), &headerRes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + expectedRes := []models.HeaderModel{ + { + BlockHash: mockBlock.Hash().String(), + CID: headerCID.String(), + TotalDifficulty: mockBlock.Difficulty().String(), + TxRoot: mockBlock.TxHash().String(), + RctRoot: mockBlock.ReceiptHash().String(), + UncleRoot: mockBlock.UncleHash().String(), + Coinbase: mocks.MockHeader.Coinbase.String(), + }, + { + BlockHash: mockNonCanonicalBlock.Hash().String(), + CID: mockNonCanonicalHeaderCID.String(), + TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(), + TxRoot: mockNonCanonicalBlock.TxHash().String(), + RctRoot: mockNonCanonicalBlock.ReceiptHash().String(), + UncleRoot: mockNonCanonicalBlock.UncleHash().String(), + Coinbase: mocks.MockNonCanonicalHeader.Coinbase.String(), + }, + } + expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String() + expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String() + + require.Equal(t, len(expectedRes), len(headerRes)) + require.ElementsMatch(t, + []string{mockBlock.Hash().String(), mocks.MockNonCanonicalBlock.Hash().String()}, + []string{headerRes[0].BlockHash, headerRes[1].BlockHash}, + ) + + if headerRes[0].BlockHash == mockBlock.Hash().String() { + require.Equal(t, expectedRes[0], headerRes[0]) + require.Equal(t, expectedRes[1], headerRes[1]) + } else { + require.Equal(t, expectedRes[1], headerRes[0]) + require.Equal(t, expectedRes[0], headerRes[1]) + } + + // check indexed IPLD blocks + var data []byte + var prefixedKey string + + prefixedKey = shared.MultihashKeyFromCID(headerCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.MockHeaderRlp, data) + + prefixedKey = shared.MultihashKeyFromCID(mockNonCanonicalHeaderCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, mocks.MockNonCanonicalHeaderRlp, data) +} + +func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { + // check indexed transactions + pgStr := `SELECT header_id, tx_hash, cid, dst, src, index, + tx_data, tx_type, CAST(value as TEXT) + FROM eth.transaction_cids + WHERE block_number = $1` + txRes := make([]models.TxModel, 0) + err = db.Select(context.Background(), &txRes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + mockBlockTxs := mocks.MockBlock.Transactions() + expectedBlockTxs := []models.TxModel{ + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[0].Hash().String(), + CID: trx1CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockBlockTxs[0].Data(), + Type: mockBlockTxs[0].Type(), + Value: mockBlockTxs[0].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[1].Hash().String(), + CID: trx2CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockBlockTxs[1].Data(), + Type: mockBlockTxs[1].Type(), + Value: mockBlockTxs[1].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[2].Hash().String(), + CID: trx3CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()), + Src: mocks.SenderAddr.String(), + Index: 2, + Data: mockBlockTxs[2].Data(), + Type: mockBlockTxs[2].Type(), + Value: mockBlockTxs[2].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[3].Hash().String(), + CID: trx4CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()), + Src: mocks.SenderAddr.String(), + Index: 3, + Data: mockBlockTxs[3].Data(), + Type: mockBlockTxs[3].Type(), + Value: mockBlockTxs[3].Value().String(), + }, + { + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[4].Hash().String(), + CID: trx5CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()), + Src: mocks.SenderAddr.String(), + Index: 4, + Data: mockBlockTxs[4].Data(), + Type: mockBlockTxs[4].Type(), + Value: mockBlockTxs[4].Value().String(), + }, + } + + mockNonCanonicalBlockTxs := mocks.MockNonCanonicalBlock.Transactions() + expectedNonCanonicalBlockTxs := []models.TxModel{ + { + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[0].Hash().String(), + CID: trx2CID.String(), + Dst: mockNonCanonicalBlockTxs[0].To().String(), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockNonCanonicalBlockTxs[0].Data(), + Type: mockNonCanonicalBlockTxs[0].Type(), + Value: mockNonCanonicalBlockTxs[0].Value().String(), + }, + { + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[1].Hash().String(), + CID: trx5CID.String(), + Dst: mockNonCanonicalBlockTxs[1].To().String(), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockNonCanonicalBlockTxs[1].Data(), + Type: mockNonCanonicalBlockTxs[1].Type(), + Value: mockNonCanonicalBlockTxs[1].Value().String(), + }, + } + + require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs), len(txRes)) + + // sort results such that non-canonical block reciepts come after canonical block ones + sort.Slice(txRes, func(i, j int) bool { + if txRes[i].HeaderID == txRes[j].HeaderID { + return txRes[i].Index < txRes[j].Index + } else if txRes[i].HeaderID == mockBlock.Hash().String() { + return true + } else { + return false + } + }) + + for i, expectedTx := range expectedBlockTxs { + require.Equal(t, expectedTx, txRes[i]) + } + for i, expectedTx := range expectedNonCanonicalBlockTxs { + require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i]) + } + + // check indexed IPLD blocks + var data []byte + var prefixedKey string + + txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID} + txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5} + for i, txCID := range txCIDs { + prefixedKey = shared.MultihashKeyFromCID(txCID) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, txRLPs[i], data) + } +} + +func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { + // check indexed receipts + pgStr := `SELECT header_id, tx_id, leaf_cid, leaf_mh_key, post_status, post_state, contract, contract_hash, log_root + FROM eth.receipt_cids + WHERE block_number = $1` + rctRes := make([]models.ReceiptModel, 0) + err = db.Select(context.Background(), &rctRes, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + + mockBlockRcts := mocks.MockReceipts + rctCids := []cid.Cid{rct1CID, rct2CID, rct3CID, rct4CID, rct5CID} + expectedBlockRcts := make(map[string]models.ReceiptModel, len(mockBlockRcts)) + for i, mockBlockRct := range mockBlockRcts { + rctModel := models.ReceiptModel{ + HeaderID: mockBlock.Hash().String(), + TxID: mockBlockRct.TxHash.String(), + LeafCID: rctCids[i].String(), + LeafMhKey: shared.MultihashKeyFromCID(rctCids[i]), + LogRoot: mockBlockRct.LogRoot.String(), + } + + contract := shared.HandleZeroAddr(mockBlockRct.ContractAddress) + rctModel.Contract = contract + if contract != "" { + rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + if len(mockBlockRct.PostState) == 0 { + rctModel.PostStatus = mockBlockRct.Status + } else { + rctModel.PostState = common.Bytes2Hex(mockBlockRct.PostState) + } + + expectedBlockRcts[rctCids[i].String()] = rctModel + } + + mockNonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts + nonCanonicalRctCids := []cid.Cid{nonCanonicalRct1CID, nonCanonicalRct2CID} + expectedNonCanonicalBlockRcts := make(map[string]models.ReceiptModel, len(mockNonCanonicalBlockRcts)) + for i, mockNonCanonicalBlockRct := range mockNonCanonicalBlockRcts { + rctModel := models.ReceiptModel{ + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxID: mockNonCanonicalBlockRct.TxHash.String(), + LeafCID: nonCanonicalRctCids[i].String(), + LeafMhKey: shared.MultihashKeyFromCID(nonCanonicalRctCids[i]), + LogRoot: mockNonCanonicalBlockRct.LogRoot.String(), + } + + contract := shared.HandleZeroAddr(mockNonCanonicalBlockRct.ContractAddress) + rctModel.Contract = contract + if contract != "" { + rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + if len(mockNonCanonicalBlockRct.PostState) == 0 { + rctModel.PostStatus = mockNonCanonicalBlockRct.Status + } else { + rctModel.PostState = common.Bytes2Hex(mockNonCanonicalBlockRct.PostState) + } + + expectedBlockRcts[nonCanonicalRctCids[i].String()] = rctModel + } + + require.Equal(t, len(expectedBlockRcts)+len(expectedNonCanonicalBlockRcts), len(rctRes)) + + // sort results such that non-canonical block reciepts come after canonical block ones + sort.Slice(rctRes, func(i, j int) bool { + if rctRes[i].HeaderID == rctRes[j].HeaderID { + return false + } else { + if rctRes[i].HeaderID == mockBlock.Hash().String() { + return true + } else { + return false + } + } + }) + + for i := 0; i < len(expectedBlockRcts); i++ { + rct := rctRes[i] + require.Contains(t, expectedBlockRcts, rct.LeafCID) + require.Equal(t, expectedBlockRcts[rct.LeafCID], rct) + } + + for i := 0; i < len(expectedNonCanonicalBlockRcts); i++ { + rct := rctRes[len(expectedBlockRcts)+i] + require.Contains(t, expectedNonCanonicalBlockRcts, rct.LeafCID) + require.Equal(t, expectedNonCanonicalBlockRcts[rct.LeafCID], rct) + } + + // check indexed rct IPLD blocks + var data []byte + var prefixedKey string + + rctRLPs := [][]byte{rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5} + for i, rctCid := range rctCids { + prefixedKey = shared.MultihashKeyFromCID(rctCid) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, rctRLPs[i], data) + } + + nonCanonicalrctRLPs := [][]byte{nonCanonicalRctLeaf1, nonCanonicalRctLeaf2} + for i, rctCid := range nonCanonicalRctCids { + prefixedKey = shared.MultihashKeyFromCID(rctCid) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, nonCanonicalrctRLPs[i], data) + } +} diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index 22d7123a0..5e9f9dca2 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -19,7 +19,6 @@ package sql_test import ( "context" "math/big" - "sort" "testing" "github.com/ipfs/go-cid" @@ -32,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/shared" @@ -51,63 +49,12 @@ func setupPGXIndexer(t *testing.T) { func setupPGX(t *testing.T) { setupPGXIndexer(t) - var tx interfaces.Batch - tx, err = ind.PushBlock( - mockBlock, - mocks.MockReceipts, - mocks.MockBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := tx.Submit(err); err != nil { - t.Fatal(err) - } - }() - for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, mockBlock.Hash().String()) - require.NoError(t, err) - } - - require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) + setupTestData(t) } func setupPGXNonCanonical(t *testing.T) { setupPGXIndexer(t) - var tx1 interfaces.Batch - - tx1, err = ind.PushBlock( - mockBlock, - mocks.MockReceipts, - mocks.MockBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } - for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx1, node, mockBlock.Hash().String()) - require.NoError(t, err) - } - - require.Equal(t, mocks.BlockNumber.String(), tx1.(*sql.BatchTx).BlockNumber) - if err := tx1.Submit(err); err != nil { - t.Fatal(err) - } - - var tx2 interfaces.Batch - tx2, err = ind.PushBlock( - mocks.MockNonCanonicalBlock, - mocks.MockNonCanonicalBlockReceipts, - mocks.MockNonCanonicalBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } - - require.Equal(t, mocks.BlockNumber.String(), tx2.(*sql.BatchTx).BlockNumber) - if err := tx2.Submit(err); err != nil { - t.Fatal(err) - } - - // TODO index state & storage nodes for second block + setupTestDataNonCanonical(t) } func TestPGXIndexer(t *testing.T) { @@ -376,7 +323,7 @@ func TestPGXIndexer(t *testing.T) { t.Fatal(err) } - // Decode the log leaf node. + // Decode the receipt leaf node. var nodeElements []interface{} err = rlp.DecodeBytes(result[0].Data, &nodeElements) require.NoError(t, err) @@ -644,77 +591,13 @@ func TestPGXIndexer(t *testing.T) { }) } -func XTestPGXIndexerNonCanonical(t *testing.T) { +func TestPGXIndexerNonCanonical(t *testing.T) { t.Run("Publish and index header", func(t *testing.T) { setupPGXNonCanonical(t) defer tearDown(t) defer checkTxClosure(t, 1, 0, 1) - // check indexed headers - pgStr := `SELECT block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT), - tx_root, receipt_root, uncle_root, coinbase - FROM eth.header_cids - WHERE block_number = $1` - headerRes := make([]models.HeaderModel, 0) - err = db.Select(context.Background(), &headerRes, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } - - expectedRes := []models.HeaderModel{ - { - BlockHash: mockBlock.Hash().String(), - CID: headerCID.String(), - TotalDifficulty: mockBlock.Difficulty().String(), - TxRoot: mockBlock.TxHash().String(), - RctRoot: mockBlock.ReceiptHash().String(), - UncleRoot: mockBlock.UncleHash().String(), - Coinbase: mocks.MockHeader.Coinbase.String(), - }, - { - BlockHash: mockNonCanonicalBlock.Hash().String(), - CID: mockNonCanonicalHeaderCID.String(), - TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(), - TxRoot: mockNonCanonicalBlock.TxHash().String(), - RctRoot: mockNonCanonicalBlock.ReceiptHash().String(), - UncleRoot: mockNonCanonicalBlock.UncleHash().String(), - Coinbase: mocks.MockNonCanonicalHeader.Coinbase.String(), - }, - } - expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String() - expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String() - - require.Equal(t, len(expectedRes), len(headerRes)) - require.ElementsMatch(t, - []string{mockBlock.Hash().String(), mocks.MockNonCanonicalBlock.Hash().String()}, - []string{headerRes[0].BlockHash, headerRes[1].BlockHash}, - ) - - if headerRes[0].BlockHash == mockBlock.Hash().String() { - require.Equal(t, expectedRes[0], headerRes[0]) - require.Equal(t, expectedRes[1], headerRes[1]) - } else { - require.Equal(t, expectedRes[1], headerRes[0]) - require.Equal(t, expectedRes[0], headerRes[1]) - } - - // check indexed IPLD blocks - var data []byte - var prefixedKey string - - prefixedKey = shared.MultihashKeyFromCID(headerCID) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } - require.Equal(t, mocks.MockHeaderRlp, data) - - prefixedKey = shared.MultihashKeyFromCID(mockNonCanonicalHeaderCID) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } - require.Equal(t, mocks.MockNonCanonicalHeaderRlp, data) + testPublishAndIndexHeaderNonCanonical(t) }) t.Run("Publish and index transactions", func(t *testing.T) { @@ -722,134 +605,15 @@ func XTestPGXIndexerNonCanonical(t *testing.T) { defer tearDown(t) defer checkTxClosure(t, 1, 0, 1) - // check indexed transactions - pgStr := `SELECT header_id, tx_hash, cid, dst, src, index, - tx_data, tx_type, CAST(value as TEXT) - FROM eth.transaction_cids - WHERE block_number = $1` - txRes := make([]models.TxModel, 0) - err = db.Select(context.Background(), &txRes, pgStr, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } + testPublishAndIndexTransactionsNonCanonical(t) + }) - mockBlockTxs := mocks.MockBlock.Transactions() - expectedBlockTxs := []models.TxModel{ - { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[0].Hash().String(), - CID: trx1CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()), - Src: mocks.SenderAddr.String(), - Index: 0, - Data: mockBlockTxs[0].Data(), - Type: mockBlockTxs[0].Type(), - Value: mockBlockTxs[0].Value().String(), - }, - { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[1].Hash().String(), - CID: trx2CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()), - Src: mocks.SenderAddr.String(), - Index: 1, - Data: mockBlockTxs[1].Data(), - Type: mockBlockTxs[1].Type(), - Value: mockBlockTxs[1].Value().String(), - }, - { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[2].Hash().String(), - CID: trx3CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()), - Src: mocks.SenderAddr.String(), - Index: 2, - Data: mockBlockTxs[2].Data(), - Type: mockBlockTxs[2].Type(), - Value: mockBlockTxs[2].Value().String(), - }, - { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[3].Hash().String(), - CID: trx4CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()), - Src: mocks.SenderAddr.String(), - Index: 3, - Data: mockBlockTxs[3].Data(), - Type: mockBlockTxs[3].Type(), - Value: mockBlockTxs[3].Value().String(), - }, - { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[4].Hash().String(), - CID: trx5CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()), - Src: mocks.SenderAddr.String(), - Index: 4, - Data: mockBlockTxs[4].Data(), - Type: mockBlockTxs[4].Type(), - Value: mockBlockTxs[4].Value().String(), - }, - } + t.Run("Publish and index receipts", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) - mockNonCanonicalBlockTxs := mocks.MockNonCanonicalBlock.Transactions() - expectedNonCanonicalBlockTxs := []models.TxModel{ - { - HeaderID: mockNonCanonicalBlock.Hash().String(), - TxHash: mockNonCanonicalBlockTxs[0].Hash().String(), - CID: trx2CID.String(), - Dst: mockNonCanonicalBlockTxs[0].To().String(), - Src: mocks.SenderAddr.String(), - Index: 0, - Data: mockNonCanonicalBlockTxs[0].Data(), - Type: mockNonCanonicalBlockTxs[0].Type(), - Value: mockNonCanonicalBlockTxs[0].Value().String(), - }, - { - HeaderID: mockNonCanonicalBlock.Hash().String(), - TxHash: mockNonCanonicalBlockTxs[1].Hash().String(), - CID: trx5CID.String(), - Dst: mockNonCanonicalBlockTxs[1].To().String(), - Src: mocks.SenderAddr.String(), - Index: 1, - Data: mockNonCanonicalBlockTxs[1].Data(), - Type: mockNonCanonicalBlockTxs[1].Type(), - Value: mockNonCanonicalBlockTxs[1].Value().String(), - }, - } - - require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs), len(txRes)) - sort.Slice(txRes, func(i, j int) bool { - if txRes[i].HeaderID == txRes[j].HeaderID { - return txRes[i].Index < txRes[j].Index - } else if txRes[i].HeaderID == mockBlock.Hash().String() { - return true - } else { - return false - } - }) - - for i, expectedTx := range expectedBlockTxs { - require.Equal(t, expectedTx, txRes[i]) - } - for i, expectedTx := range expectedNonCanonicalBlockTxs { - require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i]) - } - - // check indexed IPLD blocks - var data []byte - var prefixedKey string - - txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID} - txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5} - for i, txCID := range txCIDs { - prefixedKey = shared.MultihashKeyFromCID(txCID) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } - require.Equal(t, txRLPs[i], data) - } + testPublishAndIndexReceiptsNonCanonical(t) }) } diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index 71814b8c5..a41310f12 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" - "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/shared" @@ -51,25 +50,12 @@ func setupSQLXIndexer(t *testing.T) { func setupSQLX(t *testing.T) { setupSQLXIndexer(t) - var tx interfaces.Batch - tx, err = ind.PushBlock( - mockBlock, - mocks.MockReceipts, - mocks.MockBlock.Difficulty()) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := tx.Submit(err); err != nil { - t.Fatal(err) - } - }() - for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node, mockBlock.Hash().String()) - require.NoError(t, err) - } + setupTestData(t) +} - require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) +func setupSQLXNonCanonical(t *testing.T) { + setupPGXIndexer(t) + setupTestDataNonCanonical(t) } func TestSQLXIndexer(t *testing.T) { @@ -331,7 +317,7 @@ func TestSQLXIndexer(t *testing.T) { t.Fatal(err) } - // Decode the log leaf node. + // Decode the receipt leaf node. var nodeElements []interface{} err = rlp.DecodeBytes(result[0].Data, &nodeElements) require.NoError(t, err) @@ -598,6 +584,32 @@ func TestSQLXIndexer(t *testing.T) { }) } +func TestSQLXIndexerNonCanonical(t *testing.T) { + t.Run("Publish and index header", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexHeaderNonCanonical(t) + }) + + t.Run("Publish and index transactions", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexTransactionsNonCanonical(t) + }) + + t.Run("Publish and index receipts", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexReceiptsNonCanonical(t) + }) +} + func TestSQLXWatchAddressMethods(t *testing.T) { setupSQLXIndexer(t) defer tearDown(t) -- 2.45.2 From 889d1bdc6d892347a89dfe27e92d5d89d1e9231c Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 9 Aug 2022 15:48:47 +0530 Subject: [PATCH 3/9] Add indexer tests for logs in a non-canonical block --- .../database/sql/indexer_shared_test.go | 74 +++++++++++++++++++ .../indexer/database/sql/pgx_indexer_test.go | 8 ++ .../indexer/database/sql/sqlx_indexer_test.go | 8 ++ 3 files changed, 90 insertions(+) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 57ee6478f..06d76296d 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -599,3 +599,77 @@ func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { require.Equal(t, nonCanonicalrctRLPs[i], data) } } + +func testPublishAndIndexLogsNonCanonical(t *testing.T) { + // check indexed logs + pgStr := `SELECT address, log_data, topic0, topic1, topic2, topic3, data + FROM eth.log_cids + INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key) + WHERE log_cids.block_number = $1 AND header_id = $2 AND rct_id = $3 + ORDER BY log_cids.index ASC` + + type rctWithBlockHash struct { + rct *types.Receipt + blockHash string + } + mockRcts := make([]rctWithBlockHash, 0) + + for _, mockBlockRct := range mocks.MockReceipts { + mockRcts = append(mockRcts, rctWithBlockHash{ + mockBlockRct, + mockBlock.Hash().String(), + }) + } + + for _, mockBlockRct := range mocks.MockNonCanonicalBlockReceipts { + mockRcts = append(mockRcts, rctWithBlockHash{ + mockBlockRct, + mockNonCanonicalBlock.Hash().String(), + }) + } + + for _, mockRct := range mockRcts { + type logWithIPLD struct { + models.LogsModel + IPLDData []byte `db:"data"` + } + logRes := make([]logWithIPLD, 0) + err = db.Select(context.Background(), &logRes, pgStr, mocks.BlockNumber.Uint64(), mockRct.blockHash, mockRct.rct.TxHash.String()) + require.NoError(t, err) + require.Equal(t, len(mockRct.rct.Logs), len(logRes)) + + for i, log := range mockRct.rct.Logs { + topicSet := make([]string, 4) + for ti, topic := range log.Topics { + topicSet[ti] = topic.Hex() + } + + expectedLog := models.LogsModel{ + Address: log.Address.String(), + Data: log.Data, + Topic0: topicSet[0], + Topic1: topicSet[1], + Topic2: topicSet[2], + Topic3: topicSet[3], + } + require.Equal(t, expectedLog, logRes[i].LogsModel) + + // check indexed log IPLD block + var nodeElements []interface{} + err = rlp.DecodeBytes(logRes[i].IPLDData, &nodeElements) + require.NoError(t, err) + + if len(nodeElements) == 2 { + logRaw, err := rlp.EncodeToBytes(&log) + require.NoError(t, err) + // 2nd element of the leaf node contains the encoded log data. + require.Equal(t, nodeElements[1].([]byte), logRaw) + } else { + logRaw, err := rlp.EncodeToBytes(&log) + require.NoError(t, err) + // raw log was IPLDized + require.Equal(t, logRes[i].IPLDData, logRaw) + } + } + } +} diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index 5e9f9dca2..6a147ce7c 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -615,6 +615,14 @@ func TestPGXIndexerNonCanonical(t *testing.T) { testPublishAndIndexReceiptsNonCanonical(t) }) + + t.Run("Publish and index logs", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexLogsNonCanonical(t) + }) } func TestPGXWatchAddressMethods(t *testing.T) { diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index a41310f12..2d1076bf8 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -608,6 +608,14 @@ func TestSQLXIndexerNonCanonical(t *testing.T) { testPublishAndIndexReceiptsNonCanonical(t) }) + + t.Run("Publish and index logs", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexLogsNonCanonical(t) + }) } func TestSQLXWatchAddressMethods(t *testing.T) { -- 2.45.2 From 8c4c61ea6f6305a556fc4b1381dd978f7426f360 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 9 Aug 2022 18:41:08 +0530 Subject: [PATCH 4/9] Add indexer tests for state and storage nodes in a non-canonical block --- .../database/sql/indexer_shared_test.go | 126 +++++++++++++++++- .../indexer/database/sql/pgx_indexer_test.go | 16 +++ .../indexer/database/sql/sqlx_indexer_test.go | 16 +++ 3 files changed, 151 insertions(+), 7 deletions(-) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 06d76296d..7988aa4fe 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -218,11 +218,6 @@ func tearDown(t *testing.T) { require.NoError(t, err) } -// TODO Refactor rest of the tests to avoid duplication -// func testPublishAndIndexHeaderIPLDs() (t *testing.T) { - -// } - func setupTestData(t *testing.T) { var tx interfaces.Batch tx, err = ind.PushBlock( @@ -273,13 +268,15 @@ func setupTestDataNonCanonical(t *testing.T) { if err != nil { t.Fatal(err) } + for _, node := range mocks.StateDiffs { + err = ind.PushStateNode(tx2, node, mocks.MockNonCanonicalBlock.Hash().String()) + require.NoError(t, err) + } require.Equal(t, mocks.BlockNumber.String(), tx2.(*sql.BatchTx).BlockNumber) if err := tx2.Submit(err); err != nil { t.Fatal(err) } - - // TODO index state & storage nodes for second block } func testPublishAndIndexHeaderNonCanonical(t *testing.T) { @@ -673,3 +670,118 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { } } } + +func testPublishAndIndexStateNonCanonical(t *testing.T) { + // check indexed state nodes + pgStr := `SELECT state_path, state_leaf_key, node_type, cid, mh_key, diff + FROM eth.state_cids + WHERE block_number = $1 + AND header_id = $2 + ORDER BY state_path` + + removedNodeCID, _ := cid.Decode(shared.RemovedNodeStateCID) + stateNodeCIDs := []cid.Cid{state1CID, state2CID, removedNodeCID, removedNodeCID} + + expectedStateNodes := make([]models.StateNodeModel, 0) + for i, stateDiff := range mocks.StateDiffs { + expectedStateNodes = append(expectedStateNodes, models.StateNodeModel{ + Path: stateDiff.Path, + StateKey: common.BytesToHash(stateDiff.LeafKey).Hex(), + NodeType: stateDiff.NodeType.Int(), + CID: stateNodeCIDs[i].String(), + MhKey: shared.MultihashKeyFromCID(stateNodeCIDs[i]), + Diff: true, + }) + } + sort.Slice(expectedStateNodes, func(i, j int) bool { + if bytes.Compare(expectedStateNodes[i].Path, expectedStateNodes[j].Path) < 0 { + return true + } else { + return false + } + }) + + // check state nodes for canonical block + stateNodes := make([]models.StateNodeModel, 0) + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64(), mockBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStateNodes), len(stateNodes)) + + for i, expectedStateNode := range expectedStateNodes { + require.Equal(t, expectedStateNode, stateNodes[i]) + } + + // check state nodes for non-canonical block + stateNodes = make([]models.StateNodeModel, 0) + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64(), mockNonCanonicalBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStateNodes), len(stateNodes)) + + for i, expectedStateNode := range expectedStateNodes { + require.Equal(t, expectedStateNode, stateNodes[i]) + } +} + +func testPublishAndIndexStorageNonCanonical(t *testing.T) { + // check indexed storage nodes + pgStr := `SELECT state_path, storage_path, storage_leaf_key, node_type, cid, mh_key, diff + FROM eth.storage_cids + WHERE block_number = $1 + AND header_id = $2 + ORDER BY state_path, storage_path` + + removedNodeCID, _ := cid.Decode(shared.RemovedNodeStorageCID) + storageNodeCIDs := []cid.Cid{storageCID, removedNodeCID, removedNodeCID, removedNodeCID} + + expectedStorageNodes := make([]models.StorageNodeModel, 0) + storageNodeIndex := 0 + for _, stateDiff := range mocks.StateDiffs { + for _, storageNode := range stateDiff.StorageNodes { + expectedStorageNodes = append(expectedStorageNodes, models.StorageNodeModel{ + StatePath: stateDiff.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).Hex(), + NodeType: storageNode.NodeType.Int(), + CID: storageNodeCIDs[storageNodeIndex].String(), + MhKey: shared.MultihashKeyFromCID(storageNodeCIDs[storageNodeIndex]), + Diff: true, + }) + storageNodeIndex++ + } + } + sort.Slice(expectedStorageNodes, func(i, j int) bool { + if bytes.Compare(expectedStorageNodes[i].Path, expectedStorageNodes[j].Path) < 0 { + return true + } else { + return false + } + }) + + // check storage nodes for canonical block + storageNodes := make([]models.StorageNodeModel, 0) + err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64(), mockBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStorageNodes), len(storageNodes)) + + for i, expectedStorageNode := range expectedStorageNodes { + require.Equal(t, expectedStorageNode, storageNodes[i]) + } + + // check storage nodes for non-canonical block + storageNodes = make([]models.StorageNodeModel, 0) + err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64(), mockNonCanonicalBlock.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedStorageNodes), len(storageNodes)) + + for i, expectedStorageNode := range expectedStorageNodes { + require.Equal(t, expectedStorageNode, storageNodes[i]) + } +} diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index 6a147ce7c..168dba5f4 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -623,6 +623,22 @@ func TestPGXIndexerNonCanonical(t *testing.T) { testPublishAndIndexLogsNonCanonical(t) }) + + t.Run("Publish and index state nodes", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexStateNonCanonical(t) + }) + + t.Run("Publish and index storage nodes", func(t *testing.T) { + setupPGXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexStorageNonCanonical(t) + }) } func TestPGXWatchAddressMethods(t *testing.T) { diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index 2d1076bf8..5f96abf60 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -616,6 +616,22 @@ func TestSQLXIndexerNonCanonical(t *testing.T) { testPublishAndIndexLogsNonCanonical(t) }) + + t.Run("Publish and index state nodes", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexStateNonCanonical(t) + }) + + t.Run("Publish and index storage nodes", func(t *testing.T) { + setupSQLXNonCanonical(t) + defer tearDown(t) + defer checkTxClosure(t, 1, 0, 1) + + testPublishAndIndexStorageNonCanonical(t) + }) } func TestSQLXWatchAddressMethods(t *testing.T) { -- 2.45.2 From 01821450de601f8fd125b20b74435893e5bb6e65 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 10 Aug 2022 15:08:33 +0530 Subject: [PATCH 5/9] Add indexer tests for non-canonical block at another height --- .../database/sql/indexer_shared_test.go | 607 +++++++++++------- statediff/indexer/mocks/test_data.go | 25 +- 2 files changed, 398 insertions(+), 234 deletions(-) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 7988aa4fe..0d7f72705 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -30,18 +30,22 @@ var ( ind interfaces.StateDiffIndexer ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` - tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte - nonCanonicalRct1, nonCanonicalRct2 []byte - mockBlock, mockNonCanonicalBlock *types.Block - headerCID, mockNonCanonicalHeaderCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid - rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid - nonCanonicalRct1CID, nonCanonicalRct2CID cid.Cid - rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte - nonCanonicalRctLeaf1, nonCanonicalRctLeaf2 []byte - state1CID, state2CID, storageCID cid.Cid - contract1Address, contract2Address, contract3Address, contract4Address string - contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 - lastFilledAt, watchedAt1, watchedAt2, watchedAt3 uint64 + tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + nonCanonicalBlockRct1, nonCanonicalBlockRct2 []byte + nonCanonicalBlock2Rct1, nonCanonicalBlock2Rct2 []byte + mockBlock, mockNonCanonicalBlock, mockNonCanonicalBlock2 *types.Block + headerCID, mockNonCanonicalHeaderCID, mockNonCanonicalHeader2CID cid.Cid + trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid + rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID cid.Cid + nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID cid.Cid + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte + nonCanonicalBlockRctLeaf1, nonCanonicalBlockRctLeaf2 []byte + nonCanonicalBlock2RctLeaf1, nonCanonicalBlock2RctLeaf2 []byte + state1CID, state2CID, storageCID cid.Cid + contract1Address, contract2Address, contract3Address, contract4Address string + contract1CreatedAt, contract2CreatedAt, contract3CreatedAt, contract4CreatedAt uint64 + lastFilledAt, watchedAt1, watchedAt2, watchedAt3 uint64 ) func init() { @@ -56,6 +60,9 @@ func init() { mockNonCanonicalBlock = mocks.MockNonCanonicalBlock nonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts + mockNonCanonicalBlock2 = mocks.MockNonCanonicalBlock2 + nonCanonicalBlock2Rcts := mocks.MockNonCanonicalBlock2Receipts + buf := new(bytes.Buffer) txs.EncodeIndex(0, buf) tx1 = make([]byte, buf.Len()) @@ -108,17 +115,28 @@ func init() { buf.Reset() nonCanonicalBlockRcts.EncodeIndex(0, buf) - nonCanonicalRct1 = make([]byte, buf.Len()) - copy(nonCanonicalRct1, buf.Bytes()) + nonCanonicalBlockRct1 = make([]byte, buf.Len()) + copy(nonCanonicalBlockRct1, buf.Bytes()) buf.Reset() nonCanonicalBlockRcts.EncodeIndex(1, buf) - nonCanonicalRct2 = make([]byte, buf.Len()) - copy(nonCanonicalRct2, buf.Bytes()) + nonCanonicalBlockRct2 = make([]byte, buf.Len()) + copy(nonCanonicalBlockRct2, buf.Bytes()) + buf.Reset() + + nonCanonicalBlock2Rcts.EncodeIndex(0, buf) + nonCanonicalBlock2Rct1 = make([]byte, buf.Len()) + copy(nonCanonicalBlock2Rct1, buf.Bytes()) + buf.Reset() + + nonCanonicalBlock2Rcts.EncodeIndex(1, buf) + nonCanonicalBlock2Rct2 = make([]byte, buf.Len()) + copy(nonCanonicalBlock2Rct2, buf.Bytes()) buf.Reset() headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256) mockNonCanonicalHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeaderRlp, multihash.KECCAK_256) + mockNonCanonicalHeader2CID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockNonCanonicalHeader2Rlp, multihash.KECCAK_256) trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256) trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx2, multihash.KECCAK_256) trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256) @@ -128,26 +146,7 @@ func init() { state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256) storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256) - receiptTrie := ipld.NewRctTrie() - - receiptTrie.Add(0, rct1) - receiptTrie.Add(1, rct2) - receiptTrie.Add(2, rct3) - receiptTrie.Add(3, rct4) - receiptTrie.Add(4, rct5) - - rctLeafNodes, keys, _ := receiptTrie.GetLeafNodes() - - rctleafNodeCids := make([]cid.Cid, len(rctLeafNodes)) - orderedRctLeafNodes := make([][]byte, len(rctLeafNodes)) - for i, rln := range rctLeafNodes { - var idx uint - - r := bytes.NewReader(keys[i].TrieKey) - rlp.Decode(r, &idx) - rctleafNodeCids[idx] = rln.Cid() - orderedRctLeafNodes[idx] = rln.RawData() - } + rawRctLeafNodes, rctleafNodeCids := createRctTrie([][]byte{rct1, rct2, rct3, rct4, rct5}) rct1CID = rctleafNodeCids[0] rct2CID = rctleafNodeCids[1] @@ -155,35 +154,27 @@ func init() { rct4CID = rctleafNodeCids[3] rct5CID = rctleafNodeCids[4] - rctLeaf1 = orderedRctLeafNodes[0] - rctLeaf2 = orderedRctLeafNodes[1] - rctLeaf3 = orderedRctLeafNodes[2] - rctLeaf4 = orderedRctLeafNodes[3] - rctLeaf5 = orderedRctLeafNodes[4] + rctLeaf1 = rawRctLeafNodes[0] + rctLeaf2 = rawRctLeafNodes[1] + rctLeaf3 = rawRctLeafNodes[2] + rctLeaf4 = rawRctLeafNodes[3] + rctLeaf5 = rawRctLeafNodes[4] - nonCanonicalRctTrie := ipld.NewRctTrie() + nonCanonicalBlockRawRctLeafNodes, nonCanonicalBlockRctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2}) - nonCanonicalRctTrie.Add(0, nonCanonicalRct1) - nonCanonicalRctTrie.Add(1, nonCanonicalRct2) + nonCanonicalBlockRct1CID = nonCanonicalBlockRctLeafNodeCids[0] + nonCanonicalBlockRct2CID = nonCanonicalBlockRctLeafNodeCids[1] - nonCanonicalRctLeafNodes, keys, _ := nonCanonicalRctTrie.GetLeafNodes() + nonCanonicalBlockRctLeaf1 = nonCanonicalBlockRawRctLeafNodes[0] + nonCanonicalBlockRctLeaf2 = nonCanonicalBlockRawRctLeafNodes[1] - nonCanonicalRctLeafNodeCids := make([]cid.Cid, len(nonCanonicalRctLeafNodes)) - nonCanonicalRawRctLeafNodes := make([][]byte, len(nonCanonicalRctLeafNodes)) - for i, rln := range nonCanonicalRctLeafNodes { - var idx uint + nonCanonicalBlock2RawRctLeafNodes, nonCanonicalBlock2RctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2}) - r := bytes.NewReader(keys[i].TrieKey) - rlp.Decode(r, &idx) - nonCanonicalRctLeafNodeCids[idx] = rln.Cid() - nonCanonicalRawRctLeafNodes[idx] = rln.RawData() - } + nonCanonicalBlock2Rct1CID = nonCanonicalBlock2RctLeafNodeCids[0] + nonCanonicalBlock2Rct2CID = nonCanonicalBlock2RctLeafNodeCids[1] - nonCanonicalRct1CID = nonCanonicalRctLeafNodeCids[0] - nonCanonicalRct2CID = nonCanonicalRctLeafNodeCids[1] - - nonCanonicalRctLeaf1 = nonCanonicalRawRctLeafNodes[0] - nonCanonicalRctLeaf2 = nonCanonicalRawRctLeafNodes[1] + nonCanonicalBlock2RctLeaf1 = nonCanonicalBlock2RawRctLeafNodes[0] + nonCanonicalBlock2RctLeaf2 = nonCanonicalBlock2RawRctLeafNodes[1] contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F" contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B" @@ -200,6 +191,53 @@ func init() { watchedAt3 = uint64(20) } +func createRctTrie(rcts [][]byte) ([][]byte, []cid.Cid) { + receiptTrie := ipld.NewRctTrie() + + for i, rct := range rcts { + receiptTrie.Add(i, rct) + } + rctLeafNodes, keys, _ := receiptTrie.GetLeafNodes() + + rctleafNodeCids := make([]cid.Cid, len(rctLeafNodes)) + orderedRctLeafNodes := make([][]byte, len(rctLeafNodes)) + for i, rln := range rctLeafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + rlp.Decode(r, &idx) + rctleafNodeCids[idx] = rln.Cid() + orderedRctLeafNodes[idx] = rln.RawData() + } + + return orderedRctLeafNodes, rctleafNodeCids +} + +func createRctModel(rct *types.Receipt, cid cid.Cid, blockNumber string) models.ReceiptModel { + rctModel := models.ReceiptModel{ + BlockNumber: blockNumber, + HeaderID: rct.BlockHash.String(), + TxID: rct.TxHash.String(), + LeafCID: cid.String(), + LeafMhKey: shared.MultihashKeyFromCID(cid), + LogRoot: rct.LogRoot.String(), + } + + contract := shared.HandleZeroAddr(rct.ContractAddress) + rctModel.Contract = contract + if contract != "" { + rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + } + + if len(rct.PostState) == 0 { + rctModel.PostStatus = rct.Status + } else { + rctModel.PostState = common.Bytes2Hex(rct.PostState) + } + + return rctModel +} + func expectTrue(t *testing.T, value bool) { if !value { t.Fatalf("Assertion failed") @@ -262,14 +300,14 @@ func setupTestDataNonCanonical(t *testing.T) { var tx2 interfaces.Batch tx2, err = ind.PushBlock( - mocks.MockNonCanonicalBlock, + mockNonCanonicalBlock, mocks.MockNonCanonicalBlockReceipts, - mocks.MockNonCanonicalBlock.Difficulty()) + mockNonCanonicalBlock.Difficulty()) if err != nil { t.Fatal(err) } for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx2, node, mocks.MockNonCanonicalBlock.Hash().String()) + err = ind.PushStateNode(tx2, node, mockNonCanonicalBlock.Hash().String()) require.NoError(t, err) } @@ -277,22 +315,41 @@ func setupTestDataNonCanonical(t *testing.T) { if err := tx2.Submit(err); err != nil { t.Fatal(err) } + + var tx3 interfaces.Batch + tx3, err = ind.PushBlock( + mockNonCanonicalBlock2, + mocks.MockNonCanonicalBlock2Receipts, + mockNonCanonicalBlock2.Difficulty()) + if err != nil { + t.Fatal(err) + } + for _, node := range mocks.StateDiffs[:2] { + err = ind.PushStateNode(tx3, node, mockNonCanonicalBlock2.Hash().String()) + require.NoError(t, err) + } + + require.Equal(t, mocks.Block2Number.String(), tx3.(*sql.BatchTx).BlockNumber) + if err := tx3.Submit(err); err != nil { + t.Fatal(err) + } } func testPublishAndIndexHeaderNonCanonical(t *testing.T) { // check indexed headers - pgStr := `SELECT block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT), + pgStr := `SELECT CAST(block_number as TEXT), block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT), tx_root, receipt_root, uncle_root, coinbase FROM eth.header_cids - WHERE block_number = $1` + ORDER BY block_number` headerRes := make([]models.HeaderModel, 0) - err = db.Select(context.Background(), &headerRes, pgStr, mocks.BlockNumber.Uint64()) + err = db.Select(context.Background(), &headerRes, pgStr) if err != nil { t.Fatal(err) } expectedRes := []models.HeaderModel{ { + BlockNumber: mockBlock.Number().String(), BlockHash: mockBlock.Hash().String(), CID: headerCID.String(), TotalDifficulty: mockBlock.Difficulty().String(), @@ -302,6 +359,7 @@ func testPublishAndIndexHeaderNonCanonical(t *testing.T) { Coinbase: mocks.MockHeader.Coinbase.String(), }, { + BlockNumber: mockNonCanonicalBlock.Number().String(), BlockHash: mockNonCanonicalBlock.Hash().String(), CID: mockNonCanonicalHeaderCID.String(), TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(), @@ -310,51 +368,60 @@ func testPublishAndIndexHeaderNonCanonical(t *testing.T) { UncleRoot: mockNonCanonicalBlock.UncleHash().String(), Coinbase: mocks.MockNonCanonicalHeader.Coinbase.String(), }, + { + BlockNumber: mockNonCanonicalBlock2.Number().String(), + BlockHash: mockNonCanonicalBlock2.Hash().String(), + CID: mockNonCanonicalHeader2CID.String(), + TotalDifficulty: mockNonCanonicalBlock2.Difficulty().String(), + TxRoot: mockNonCanonicalBlock2.TxHash().String(), + RctRoot: mockNonCanonicalBlock2.ReceiptHash().String(), + UncleRoot: mockNonCanonicalBlock2.UncleHash().String(), + Coinbase: mocks.MockNonCanonicalHeader2.Coinbase.String(), + }, } expectedRes[0].Reward = shared.CalcEthBlockReward(mockBlock.Header(), mockBlock.Uncles(), mockBlock.Transactions(), mocks.MockReceipts).String() expectedRes[1].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock.Header(), mockNonCanonicalBlock.Uncles(), mockNonCanonicalBlock.Transactions(), mocks.MockNonCanonicalBlockReceipts).String() + expectedRes[2].Reward = shared.CalcEthBlockReward(mockNonCanonicalBlock2.Header(), mockNonCanonicalBlock2.Uncles(), mockNonCanonicalBlock2.Transactions(), mocks.MockNonCanonicalBlock2Receipts).String() require.Equal(t, len(expectedRes), len(headerRes)) require.ElementsMatch(t, - []string{mockBlock.Hash().String(), mocks.MockNonCanonicalBlock.Hash().String()}, - []string{headerRes[0].BlockHash, headerRes[1].BlockHash}, + []string{mockBlock.Hash().String(), mockNonCanonicalBlock.Hash().String(), mockNonCanonicalBlock2.Hash().String()}, + []string{headerRes[0].BlockHash, headerRes[1].BlockHash, headerRes[2].BlockHash}, ) if headerRes[0].BlockHash == mockBlock.Hash().String() { require.Equal(t, expectedRes[0], headerRes[0]) require.Equal(t, expectedRes[1], headerRes[1]) + require.Equal(t, expectedRes[2], headerRes[2]) } else { require.Equal(t, expectedRes[1], headerRes[0]) require.Equal(t, expectedRes[0], headerRes[1]) + require.Equal(t, expectedRes[2], headerRes[2]) } // check indexed IPLD blocks - var data []byte - var prefixedKey string - - prefixedKey = shared.MultihashKeyFromCID(headerCID) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) + headerCIDs := []cid.Cid{headerCID, mockNonCanonicalHeaderCID, mockNonCanonicalHeader2CID} + blockNumbers := []uint64{mocks.BlockNumber.Uint64(), mocks.BlockNumber.Uint64(), mocks.Block2Number.Uint64()} + headerRLPs := [][]byte{mocks.MockHeaderRlp, mocks.MockNonCanonicalHeaderRlp, mocks.MockNonCanonicalHeader2Rlp} + for i := range expectedRes { + var data []byte + prefixedKey := shared.MultihashKeyFromCID(headerCIDs[i]) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, blockNumbers[i]) + if err != nil { + t.Fatal(err) + } + require.Equal(t, headerRLPs[i], data) } - require.Equal(t, mocks.MockHeaderRlp, data) - - prefixedKey = shared.MultihashKeyFromCID(mockNonCanonicalHeaderCID) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) - if err != nil { - t.Fatal(err) - } - require.Equal(t, mocks.MockNonCanonicalHeaderRlp, data) } func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { // check indexed transactions - pgStr := `SELECT header_id, tx_hash, cid, dst, src, index, + pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_hash, cid, dst, src, index, tx_data, tx_type, CAST(value as TEXT) FROM eth.transaction_cids - WHERE block_number = $1` + ORDER BY block_number, index` txRes := make([]models.TxModel, 0) - err = db.Select(context.Background(), &txRes, pgStr, mocks.BlockNumber.Uint64()) + err = db.Select(context.Background(), &txRes, pgStr) if err != nil { t.Fatal(err) } @@ -362,93 +429,130 @@ func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { mockBlockTxs := mocks.MockBlock.Transactions() expectedBlockTxs := []models.TxModel{ { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[0].Hash().String(), - CID: trx1CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()), - Src: mocks.SenderAddr.String(), - Index: 0, - Data: mockBlockTxs[0].Data(), - Type: mockBlockTxs[0].Type(), - Value: mockBlockTxs[0].Value().String(), + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[0].Hash().String(), + CID: trx1CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockBlockTxs[0].Data(), + Type: mockBlockTxs[0].Type(), + Value: mockBlockTxs[0].Value().String(), }, { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[1].Hash().String(), - CID: trx2CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()), - Src: mocks.SenderAddr.String(), - Index: 1, - Data: mockBlockTxs[1].Data(), - Type: mockBlockTxs[1].Type(), - Value: mockBlockTxs[1].Value().String(), + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[1].Hash().String(), + CID: trx2CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockBlockTxs[1].Data(), + Type: mockBlockTxs[1].Type(), + Value: mockBlockTxs[1].Value().String(), }, { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[2].Hash().String(), - CID: trx3CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()), - Src: mocks.SenderAddr.String(), - Index: 2, - Data: mockBlockTxs[2].Data(), - Type: mockBlockTxs[2].Type(), - Value: mockBlockTxs[2].Value().String(), + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[2].Hash().String(), + CID: trx3CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()), + Src: mocks.SenderAddr.String(), + Index: 2, + Data: mockBlockTxs[2].Data(), + Type: mockBlockTxs[2].Type(), + Value: mockBlockTxs[2].Value().String(), }, { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[3].Hash().String(), - CID: trx4CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()), - Src: mocks.SenderAddr.String(), - Index: 3, - Data: mockBlockTxs[3].Data(), - Type: mockBlockTxs[3].Type(), - Value: mockBlockTxs[3].Value().String(), + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[3].Hash().String(), + CID: trx4CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()), + Src: mocks.SenderAddr.String(), + Index: 3, + Data: mockBlockTxs[3].Data(), + Type: mockBlockTxs[3].Type(), + Value: mockBlockTxs[3].Value().String(), }, { - HeaderID: mockBlock.Hash().String(), - TxHash: mockBlockTxs[4].Hash().String(), - CID: trx5CID.String(), - Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()), - Src: mocks.SenderAddr.String(), - Index: 4, - Data: mockBlockTxs[4].Data(), - Type: mockBlockTxs[4].Type(), - Value: mockBlockTxs[4].Value().String(), + BlockNumber: mockBlock.Number().String(), + HeaderID: mockBlock.Hash().String(), + TxHash: mockBlockTxs[4].Hash().String(), + CID: trx5CID.String(), + Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()), + Src: mocks.SenderAddr.String(), + Index: 4, + Data: mockBlockTxs[4].Data(), + Type: mockBlockTxs[4].Type(), + Value: mockBlockTxs[4].Value().String(), }, } - mockNonCanonicalBlockTxs := mocks.MockNonCanonicalBlock.Transactions() + mockNonCanonicalBlockTxs := mockNonCanonicalBlock.Transactions() expectedNonCanonicalBlockTxs := []models.TxModel{ { - HeaderID: mockNonCanonicalBlock.Hash().String(), - TxHash: mockNonCanonicalBlockTxs[0].Hash().String(), - CID: trx2CID.String(), - Dst: mockNonCanonicalBlockTxs[0].To().String(), - Src: mocks.SenderAddr.String(), - Index: 0, - Data: mockNonCanonicalBlockTxs[0].Data(), - Type: mockNonCanonicalBlockTxs[0].Type(), - Value: mockNonCanonicalBlockTxs[0].Value().String(), + BlockNumber: mockNonCanonicalBlock.Number().String(), + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[0].Hash().String(), + CID: trx2CID.String(), + Dst: mockNonCanonicalBlockTxs[0].To().String(), + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockNonCanonicalBlockTxs[0].Data(), + Type: mockNonCanonicalBlockTxs[0].Type(), + Value: mockNonCanonicalBlockTxs[0].Value().String(), }, { - HeaderID: mockNonCanonicalBlock.Hash().String(), - TxHash: mockNonCanonicalBlockTxs[1].Hash().String(), - CID: trx5CID.String(), - Dst: mockNonCanonicalBlockTxs[1].To().String(), - Src: mocks.SenderAddr.String(), - Index: 1, - Data: mockNonCanonicalBlockTxs[1].Data(), - Type: mockNonCanonicalBlockTxs[1].Type(), - Value: mockNonCanonicalBlockTxs[1].Value().String(), + BlockNumber: mockNonCanonicalBlock.Number().String(), + HeaderID: mockNonCanonicalBlock.Hash().String(), + TxHash: mockNonCanonicalBlockTxs[1].Hash().String(), + CID: trx5CID.String(), + Dst: mockNonCanonicalBlockTxs[1].To().String(), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockNonCanonicalBlockTxs[1].Data(), + Type: mockNonCanonicalBlockTxs[1].Type(), + Value: mockNonCanonicalBlockTxs[1].Value().String(), }, } - require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs), len(txRes)) + mockNonCanonicalBlock2Txs := mockNonCanonicalBlock2.Transactions() + expectedNonCanonicalBlock2Txs := []models.TxModel{ + { + BlockNumber: mockNonCanonicalBlock2.Number().String(), + HeaderID: mockNonCanonicalBlock2.Hash().String(), + TxHash: mockNonCanonicalBlock2Txs[0].Hash().String(), + CID: trx3CID.String(), + Dst: "", + Src: mocks.SenderAddr.String(), + Index: 0, + Data: mockNonCanonicalBlock2Txs[0].Data(), + Type: mockNonCanonicalBlock2Txs[0].Type(), + Value: mockNonCanonicalBlock2Txs[0].Value().String(), + }, + { + BlockNumber: mockNonCanonicalBlock2.Number().String(), + HeaderID: mockNonCanonicalBlock2.Hash().String(), + TxHash: mockNonCanonicalBlock2Txs[1].Hash().String(), + CID: trx5CID.String(), + Dst: mockNonCanonicalBlock2Txs[1].To().String(), + Src: mocks.SenderAddr.String(), + Index: 1, + Data: mockNonCanonicalBlock2Txs[1].Data(), + Type: mockNonCanonicalBlock2Txs[1].Type(), + Value: mockNonCanonicalBlock2Txs[1].Value().String(), + }, + } - // sort results such that non-canonical block reciepts come after canonical block ones - sort.Slice(txRes, func(i, j int) bool { - if txRes[i].HeaderID == txRes[j].HeaderID { + require.Equal(t, len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs)+len(expectedNonCanonicalBlock2Txs), len(txRes)) + + // sort results such that non-canonical block transactions come after canonical block ones + sort.SliceStable(txRes, func(i, j int) bool { + if txRes[i].BlockNumber < txRes[j].BlockNumber { + return true + } else if txRes[i].HeaderID == txRes[j].HeaderID { return txRes[i].Index < txRes[j].Index } else if txRes[i].HeaderID == mockBlock.Hash().String() { return true @@ -463,6 +567,9 @@ func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { for i, expectedTx := range expectedNonCanonicalBlockTxs { require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+i]) } + for i, expectedTx := range expectedNonCanonicalBlock2Txs { + require.Equal(t, expectedTx, txRes[len(expectedBlockTxs)+len(expectedNonCanonicalBlockTxs)+i]) + } // check indexed IPLD blocks var data []byte @@ -482,102 +589,78 @@ func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { // check indexed receipts - pgStr := `SELECT header_id, tx_id, leaf_cid, leaf_mh_key, post_status, post_state, contract, contract_hash, log_root + pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_id, leaf_cid, leaf_mh_key, post_status, post_state, contract, contract_hash, log_root FROM eth.receipt_cids - WHERE block_number = $1` + ORDER BY block_number` rctRes := make([]models.ReceiptModel, 0) - err = db.Select(context.Background(), &rctRes, pgStr, mocks.BlockNumber.Uint64()) + err = db.Select(context.Background(), &rctRes, pgStr) if err != nil { t.Fatal(err) } - mockBlockRcts := mocks.MockReceipts rctCids := []cid.Cid{rct1CID, rct2CID, rct3CID, rct4CID, rct5CID} - expectedBlockRcts := make(map[string]models.ReceiptModel, len(mockBlockRcts)) - for i, mockBlockRct := range mockBlockRcts { - rctModel := models.ReceiptModel{ - HeaderID: mockBlock.Hash().String(), - TxID: mockBlockRct.TxHash.String(), - LeafCID: rctCids[i].String(), - LeafMhKey: shared.MultihashKeyFromCID(rctCids[i]), - LogRoot: mockBlockRct.LogRoot.String(), - } - - contract := shared.HandleZeroAddr(mockBlockRct.ContractAddress) - rctModel.Contract = contract - if contract != "" { - rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() - } - - if len(mockBlockRct.PostState) == 0 { - rctModel.PostStatus = mockBlockRct.Status - } else { - rctModel.PostState = common.Bytes2Hex(mockBlockRct.PostState) - } - - expectedBlockRcts[rctCids[i].String()] = rctModel + expectedBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockReceipts)) + for i, mockBlockRct := range mocks.MockReceipts { + rctModel := createRctModel(mockBlockRct, rctCids[i], mockBlock.Number().String()) + expectedBlockRctsMap[rctCids[i].String()] = rctModel } - mockNonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts - nonCanonicalRctCids := []cid.Cid{nonCanonicalRct1CID, nonCanonicalRct2CID} - expectedNonCanonicalBlockRcts := make(map[string]models.ReceiptModel, len(mockNonCanonicalBlockRcts)) - for i, mockNonCanonicalBlockRct := range mockNonCanonicalBlockRcts { - rctModel := models.ReceiptModel{ - HeaderID: mockNonCanonicalBlock.Hash().String(), - TxID: mockNonCanonicalBlockRct.TxHash.String(), - LeafCID: nonCanonicalRctCids[i].String(), - LeafMhKey: shared.MultihashKeyFromCID(nonCanonicalRctCids[i]), - LogRoot: mockNonCanonicalBlockRct.LogRoot.String(), - } - - contract := shared.HandleZeroAddr(mockNonCanonicalBlockRct.ContractAddress) - rctModel.Contract = contract - if contract != "" { - rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() - } - - if len(mockNonCanonicalBlockRct.PostState) == 0 { - rctModel.PostStatus = mockNonCanonicalBlockRct.Status - } else { - rctModel.PostState = common.Bytes2Hex(mockNonCanonicalBlockRct.PostState) - } - - expectedBlockRcts[nonCanonicalRctCids[i].String()] = rctModel + nonCanonicalBlockRctCids := []cid.Cid{nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID} + expectedNonCanonicalBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockNonCanonicalBlockReceipts)) + for i, mockNonCanonicalBlockRct := range mocks.MockNonCanonicalBlockReceipts { + rctModel := createRctModel(mockNonCanonicalBlockRct, nonCanonicalBlockRctCids[i], mockNonCanonicalBlock.Number().String()) + expectedNonCanonicalBlockRctsMap[nonCanonicalBlockRctCids[i].String()] = rctModel } - require.Equal(t, len(expectedBlockRcts)+len(expectedNonCanonicalBlockRcts), len(rctRes)) + nonCanonicalBlock2RctCids := []cid.Cid{nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID} + expectedNonCanonicalBlock2RctsMap := make(map[string]models.ReceiptModel, len(mocks.MockNonCanonicalBlock2Receipts)) + for i, mockNonCanonicalBlock2Rct := range mocks.MockNonCanonicalBlock2Receipts { + rctModel := createRctModel(mockNonCanonicalBlock2Rct, nonCanonicalBlock2RctCids[i], mockNonCanonicalBlock2.Number().String()) + expectedNonCanonicalBlock2RctsMap[nonCanonicalBlock2RctCids[i].String()] = rctModel + } + + require.Equal(t, len(expectedBlockRctsMap)+len(expectedNonCanonicalBlockRctsMap)+len(expectedNonCanonicalBlock2RctsMap), len(rctRes)) // sort results such that non-canonical block reciepts come after canonical block ones - sort.Slice(rctRes, func(i, j int) bool { - if rctRes[i].HeaderID == rctRes[j].HeaderID { + sort.SliceStable(rctRes, func(i, j int) bool { + if rctRes[i].BlockNumber < rctRes[j].BlockNumber { + return true + } else if rctRes[i].HeaderID == rctRes[j].HeaderID { return false + } else if rctRes[i].HeaderID == mockBlock.Hash().String() { + return true } else { - if rctRes[i].HeaderID == mockBlock.Hash().String() { - return true - } else { - return false - } + return false } }) - for i := 0; i < len(expectedBlockRcts); i++ { + for i := 0; i < len(expectedBlockRctsMap); i++ { rct := rctRes[i] - require.Contains(t, expectedBlockRcts, rct.LeafCID) - require.Equal(t, expectedBlockRcts[rct.LeafCID], rct) + require.Contains(t, expectedBlockRctsMap, rct.LeafCID) + require.Equal(t, expectedBlockRctsMap[rct.LeafCID], rct) } - for i := 0; i < len(expectedNonCanonicalBlockRcts); i++ { - rct := rctRes[len(expectedBlockRcts)+i] - require.Contains(t, expectedNonCanonicalBlockRcts, rct.LeafCID) - require.Equal(t, expectedNonCanonicalBlockRcts[rct.LeafCID], rct) + for i := 0; i < len(expectedNonCanonicalBlockRctsMap); i++ { + rct := rctRes[len(expectedBlockRctsMap)+i] + require.Contains(t, expectedNonCanonicalBlockRctsMap, rct.LeafCID) + require.Equal(t, expectedNonCanonicalBlockRctsMap[rct.LeafCID], rct) + } + + for i := 0; i < len(expectedNonCanonicalBlock2RctsMap); i++ { + rct := rctRes[len(expectedBlockRctsMap)+len(expectedNonCanonicalBlockRctsMap)+i] + require.Contains(t, expectedNonCanonicalBlock2RctsMap, rct.LeafCID) + require.Equal(t, expectedNonCanonicalBlock2RctsMap[rct.LeafCID], rct) } // check indexed rct IPLD blocks var data []byte var prefixedKey string - rctRLPs := [][]byte{rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5} - for i, rctCid := range rctCids { + rctRLPs := [][]byte{ + rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5, + nonCanonicalBlockRctLeaf1, nonCanonicalBlockRctLeaf2, + } + for i, rctCid := range append(rctCids, nonCanonicalBlockRctCids...) { prefixedKey = shared.MultihashKeyFromCID(rctCid) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) if err != nil { @@ -586,14 +669,14 @@ func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { require.Equal(t, rctRLPs[i], data) } - nonCanonicalrctRLPs := [][]byte{nonCanonicalRctLeaf1, nonCanonicalRctLeaf2} - for i, rctCid := range nonCanonicalRctCids { + nonCanonicalBlock2RctRLPs := [][]byte{nonCanonicalBlock2RctLeaf1, nonCanonicalBlock2RctLeaf2} + for i, rctCid := range nonCanonicalBlock2RctCids { prefixedKey = shared.MultihashKeyFromCID(rctCid) - err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64()) + err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.Block2Number.Uint64()) if err != nil { t.Fatal(err) } - require.Equal(t, nonCanonicalrctRLPs[i], data) + require.Equal(t, nonCanonicalBlock2RctRLPs[i], data) } } @@ -601,13 +684,14 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { // check indexed logs pgStr := `SELECT address, log_data, topic0, topic1, topic2, topic3, data FROM eth.log_cids - INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key) + INNER JOIN public.blocks ON (log_cids.block_number = blocks.block_number AND log_cids.leaf_mh_key = blocks.key) WHERE log_cids.block_number = $1 AND header_id = $2 AND rct_id = $3 ORDER BY log_cids.index ASC` type rctWithBlockHash struct { - rct *types.Receipt - blockHash string + rct *types.Receipt + blockHash string + blockNumber uint64 } mockRcts := make([]rctWithBlockHash, 0) @@ -615,6 +699,7 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { mockRcts = append(mockRcts, rctWithBlockHash{ mockBlockRct, mockBlock.Hash().String(), + mockBlock.NumberU64(), }) } @@ -622,6 +707,15 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { mockRcts = append(mockRcts, rctWithBlockHash{ mockBlockRct, mockNonCanonicalBlock.Hash().String(), + mockNonCanonicalBlock.NumberU64(), + }) + } + + for _, mockBlockRct := range mocks.MockNonCanonicalBlock2Receipts { + mockRcts = append(mockRcts, rctWithBlockHash{ + mockBlockRct, + mockNonCanonicalBlock2.Hash().String(), + mockNonCanonicalBlock2.NumberU64(), }) } @@ -631,7 +725,7 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { IPLDData []byte `db:"data"` } logRes := make([]logWithIPLD, 0) - err = db.Select(context.Background(), &logRes, pgStr, mocks.BlockNumber.Uint64(), mockRct.blockHash, mockRct.rct.TxHash.String()) + err = db.Select(context.Background(), &logRes, pgStr, mockRct.blockNumber, mockRct.blockHash, mockRct.rct.TxHash.String()) require.NoError(t, err) require.Equal(t, len(mockRct.rct.Logs), len(logRes)) @@ -701,6 +795,18 @@ func testPublishAndIndexStateNonCanonical(t *testing.T) { } }) + expectedNonCanonicalBlock2StateNodes := make([]models.StateNodeModel, 0) + for i, stateDiff := range mocks.StateDiffs[:2] { + expectedNonCanonicalBlock2StateNodes = append(expectedNonCanonicalBlock2StateNodes, models.StateNodeModel{ + Path: stateDiff.Path, + StateKey: common.BytesToHash(stateDiff.LeafKey).Hex(), + NodeType: stateDiff.NodeType.Int(), + CID: stateNodeCIDs[i].String(), + MhKey: shared.MultihashKeyFromCID(stateNodeCIDs[i]), + Diff: true, + }) + } + // check state nodes for canonical block stateNodes := make([]models.StateNodeModel, 0) err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64(), mockBlock.Hash().String()) @@ -724,6 +830,18 @@ func testPublishAndIndexStateNonCanonical(t *testing.T) { for i, expectedStateNode := range expectedStateNodes { require.Equal(t, expectedStateNode, stateNodes[i]) } + + // check state nodes for non-canonical block at another height + stateNodes = make([]models.StateNodeModel, 0) + err = db.Select(context.Background(), &stateNodes, pgStr, mocks.Block2Number.Uint64(), mockNonCanonicalBlock2.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedNonCanonicalBlock2StateNodes), len(stateNodes)) + + for i, expectedStateNode := range expectedNonCanonicalBlock2StateNodes { + require.Equal(t, expectedStateNode, stateNodes[i]) + } } func testPublishAndIndexStorageNonCanonical(t *testing.T) { @@ -761,6 +879,23 @@ func testPublishAndIndexStorageNonCanonical(t *testing.T) { } }) + expectedNonCanonicalBlock2StorageNodes := make([]models.StorageNodeModel, 0) + storageNodeIndex = 0 + for _, stateDiff := range mocks.StateDiffs[:2] { + for _, storageNode := range stateDiff.StorageNodes { + expectedNonCanonicalBlock2StorageNodes = append(expectedNonCanonicalBlock2StorageNodes, models.StorageNodeModel{ + StatePath: stateDiff.Path, + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).Hex(), + NodeType: storageNode.NodeType.Int(), + CID: storageNodeCIDs[storageNodeIndex].String(), + MhKey: shared.MultihashKeyFromCID(storageNodeCIDs[storageNodeIndex]), + Diff: true, + }) + storageNodeIndex++ + } + } + // check storage nodes for canonical block storageNodes := make([]models.StorageNodeModel, 0) err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64(), mockBlock.Hash().String()) @@ -784,4 +919,16 @@ func testPublishAndIndexStorageNonCanonical(t *testing.T) { for i, expectedStorageNode := range expectedStorageNodes { require.Equal(t, expectedStorageNode, storageNodes[i]) } + + // check storage nodes for non-canonical block at another height + storageNodes = make([]models.StorageNodeModel, 0) + err = db.Select(context.Background(), &storageNodes, pgStr, mockNonCanonicalBlock2.NumberU64(), mockNonCanonicalBlock2.Hash().String()) + if err != nil { + t.Fatal(err) + } + require.Equal(t, len(expectedNonCanonicalBlock2StorageNodes), len(storageNodes)) + + for i, expectedStorageNode := range expectedNonCanonicalBlock2StorageNodes { + require.Equal(t, expectedStorageNode, storageNodes[i]) + } } diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index b353a61e2..6be79a88e 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -61,6 +61,23 @@ var ( MockNonCanonicalBlock = types.NewBlock(&MockNonCanonicalHeader, MockNonCanonicalBlockTransactions, nil, MockNonCanonicalBlockReceipts, new(trie.Trie)) MockNonCanonicalHeaderRlp, _ = rlp.EncodeToBytes(MockNonCanonicalBlock.Header()) + Block2Number = big.NewInt(BlockNumber.Int64() + 1) + MockNonCanonicalHeader2 = types.Header{ + Time: 0, + Number: new(big.Int).Set(Block2Number), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(6000000), + Extra: []byte{}, + BaseFee: big.NewInt(params.InitialBaseFee), + Coinbase: common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476777"), + } + MockNonCanonicalBlock2Transactions = types.Transactions{MockTransactions[2], MockTransactions[4]} + MockNonCanonicalBlock2Receipts = createNonCanonicalBlockReceipts(TestConfig, Block2Number, MockNonCanonicalBlock2Transactions) + MockNonCanonicalBlock2 = types.NewBlock(&MockNonCanonicalHeader2, MockNonCanonicalBlock2Transactions, nil, MockNonCanonicalBlock2Receipts, new(trie.Trie)) + MockNonCanonicalHeader2Rlp, _ = rlp.EncodeToBytes(MockNonCanonicalBlock2.Header()) + Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) @@ -466,15 +483,15 @@ func createNonCanonicalBlockReceipts(config *params.ChainConfig, blockNumber *bi log.Crit(err.Error()) } - mockReceipt0 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) - mockReceipt0.Logs = []*types.Log{MockLog2, ShortLog1} + mockReceipt0 := types.NewReceipt(common.HexToHash("0x3").Bytes(), false, 300) + mockReceipt0.Logs = []*types.Log{MockLog1, ShortLog1} mockReceipt0.TxHash = signedTrx0.Hash() mockReceipt1 := &types.Receipt{ Type: types.DynamicFeeTxType, - PostState: common.HexToHash("0x3").Bytes(), + PostState: common.HexToHash("0x4").Bytes(), Status: types.ReceiptStatusSuccessful, - CumulativeGasUsed: 175, + CumulativeGasUsed: 300, Logs: []*types.Log{}, TxHash: signedTrx1.Hash(), } -- 2.45.2 From 9791acf02278ecebd13a3b16c8c74fd714d74d05 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 10 Aug 2022 15:39:55 +0530 Subject: [PATCH 6/9] Avoid passing address of a pointer --- statediff/indexer/database/sql/indexer_shared_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 0d7f72705..8af510abb 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -751,12 +751,12 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { require.NoError(t, err) if len(nodeElements) == 2 { - logRaw, err := rlp.EncodeToBytes(&log) + logRaw, err := rlp.EncodeToBytes(log) require.NoError(t, err) // 2nd element of the leaf node contains the encoded log data. require.Equal(t, nodeElements[1].([]byte), logRaw) } else { - logRaw, err := rlp.EncodeToBytes(&log) + logRaw, err := rlp.EncodeToBytes(log) require.NoError(t, err) // raw log was IPLDized require.Equal(t, logRes[i].IPLDData, logRaw) -- 2.45.2 From 2bab07badef5ab047ea1cdac7a29fc4e63bfccd2 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 10 Aug 2022 15:47:37 +0530 Subject: [PATCH 7/9] Update refs in GitHub workflow --- .github/workflows/tests.yml | 4 ++-- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 913ba07f9..f1fa86a3f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,8 +4,8 @@ on: workflow_call: env: - stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || '382aca8e42bc5e33f301f77cdd2e09cc80602fc3'}} - ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || '65b7bee7a6757c1fc527c8bfdc4f99ab915fcf36' }} + stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'f2fd766f5400fcb9eb47b50675d2e3b1f2753702'}} + ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'be345e0733d2c025e4082c5154e441317ae94cf7' }} GOPATH: /tmp/go jobs: diff --git a/docker-compose.yml b/docker-compose.yml index b3c22e6ef..62a81b0aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - ipld-eth-db - image: vulcanize/ipld-eth-db:v4.2.0-alpha + image: vulcanize/ipld-eth-db:v4.2.1-alpha environment: DATABASE_USER: "vdbm" DATABASE_NAME: "vulcanize_testing" -- 2.45.2 From 2b4f6190048189edfd691a68ef472e67bbc49de9 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 10 Aug 2022 16:09:02 +0530 Subject: [PATCH 8/9] Add genesis file path to stack-orchestrator config in GitHub workflow --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f1fa86a3f..c2de4fc02 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,6 +99,7 @@ jobs: echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > $GITHUB_WORKSPACE/config.sh echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> $GITHUB_WORKSPACE/config.sh echo db_write=true >> $GITHUB_WORKSPACE/config.sh + echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> $GITHUB_WORKSPACE/config.sh cat $GITHUB_WORKSPACE/config.sh - name: Compile Geth -- 2.45.2 From d45c1620787b28efa7430a073a0f787ec51fed53 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Fri, 12 Aug 2022 15:12:46 +0530 Subject: [PATCH 9/9] Add descriptive comments --- .../database/sql/indexer_shared_test.go | 44 ++++++++++++++++--- .../indexer/database/sql/pgx_indexer_test.go | 2 + .../indexer/database/sql/sqlx_indexer_test.go | 2 + statediff/indexer/mocks/test_data.go | 10 ++++- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/statediff/indexer/database/sql/indexer_shared_test.go b/statediff/indexer/database/sql/indexer_shared_test.go index 8af510abb..db301f160 100644 --- a/statediff/indexer/database/sql/indexer_shared_test.go +++ b/statediff/indexer/database/sql/indexer_shared_test.go @@ -54,15 +54,19 @@ func init() { os.Exit(0) } + // canonical block at LondonBlock height mockBlock = mocks.MockBlock txs, rcts := mocks.MockBlock.Transactions(), mocks.MockReceipts + // non-canonical block at LondonBlock height mockNonCanonicalBlock = mocks.MockNonCanonicalBlock nonCanonicalBlockRcts := mocks.MockNonCanonicalBlockReceipts + // non-canonical block at LondonBlock height + 1 mockNonCanonicalBlock2 = mocks.MockNonCanonicalBlock2 nonCanonicalBlock2Rcts := mocks.MockNonCanonicalBlock2Receipts + // encode mock receipts buf := new(bytes.Buffer) txs.EncodeIndex(0, buf) tx1 = make([]byte, buf.Len()) @@ -114,6 +118,7 @@ func init() { copy(rct5, buf.Bytes()) buf.Reset() + // encode mock receipts for non-canonical blocks nonCanonicalBlockRcts.EncodeIndex(0, buf) nonCanonicalBlockRct1 = make([]byte, buf.Len()) copy(nonCanonicalBlockRct1, buf.Bytes()) @@ -146,6 +151,7 @@ func init() { state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256) storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256) + // create raw receipts rawRctLeafNodes, rctleafNodeCids := createRctTrie([][]byte{rct1, rct2, rct3, rct4, rct5}) rct1CID = rctleafNodeCids[0] @@ -160,6 +166,7 @@ func init() { rctLeaf4 = rawRctLeafNodes[3] rctLeaf5 = rawRctLeafNodes[4] + // create raw receipts for non-canonical blocks nonCanonicalBlockRawRctLeafNodes, nonCanonicalBlockRctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2}) nonCanonicalBlockRct1CID = nonCanonicalBlockRctLeafNodeCids[0] @@ -191,6 +198,8 @@ func init() { watchedAt3 = uint64(20) } +// createRctTrie creates a receipt trie from the given raw receipts +// returns receipt leaf nodes and their CIDs func createRctTrie(rcts [][]byte) ([][]byte, []cid.Cid) { receiptTrie := ipld.NewRctTrie() @@ -213,6 +222,7 @@ func createRctTrie(rcts [][]byte) ([][]byte, []cid.Cid) { return orderedRctLeafNodes, rctleafNodeCids } +// createRctModel creates a models.ReceiptModel object from a given ethereum receipt func createRctModel(rct *types.Receipt, cid cid.Cid, blockNumber string) models.ReceiptModel { rctModel := models.ReceiptModel{ BlockNumber: blockNumber, @@ -256,6 +266,7 @@ func tearDown(t *testing.T) { require.NoError(t, err) } +// setupTestData indexes a single mock block along with it's state nodes func setupTestData(t *testing.T) { var tx interfaces.Batch tx, err = ind.PushBlock( @@ -278,9 +289,12 @@ func setupTestData(t *testing.T) { require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber) } +// setupTestData indexes a mock block and a non-canonical mock block at London height +// and a non-canonical block at London height + 1 +// along with their state nodes func setupTestDataNonCanonical(t *testing.T) { + // index a canonical block at London height var tx1 interfaces.Batch - tx1, err = ind.PushBlock( mockBlock, mocks.MockReceipts, @@ -298,6 +312,8 @@ func setupTestDataNonCanonical(t *testing.T) { t.Fatal(err) } + // index a non-canonical block at London height + // has transactions overlapping with that of the canonical block var tx2 interfaces.Batch tx2, err = ind.PushBlock( mockNonCanonicalBlock, @@ -316,6 +332,8 @@ func setupTestDataNonCanonical(t *testing.T) { t.Fatal(err) } + // index a non-canonical block at London height + 1 + // has transactions overlapping with that of the canonical block var tx3 interfaces.Batch tx3, err = ind.PushBlock( mockNonCanonicalBlock2, @@ -347,6 +365,9 @@ func testPublishAndIndexHeaderNonCanonical(t *testing.T) { t.Fatal(err) } + // expect three blocks to be indexed + // a canonical and a non-canonical block at London height, + // non-canonical block at London height + 1 expectedRes := []models.HeaderModel{ { BlockNumber: mockBlock.Number().String(), @@ -426,6 +447,7 @@ func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { t.Fatal(err) } + // expected transactions in the canonical block mockBlockTxs := mocks.MockBlock.Transactions() expectedBlockTxs := []models.TxModel{ { @@ -490,6 +512,7 @@ func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { }, } + // expected transactions in the canonical block at London height mockNonCanonicalBlockTxs := mockNonCanonicalBlock.Transactions() expectedNonCanonicalBlockTxs := []models.TxModel{ { @@ -518,6 +541,7 @@ func testPublishAndIndexTransactionsNonCanonical(t *testing.T) { }, } + // expected transactions in the canonical block at London height + 1 mockNonCanonicalBlock2Txs := mockNonCanonicalBlock2.Transactions() expectedNonCanonicalBlock2Txs := []models.TxModel{ { @@ -598,6 +622,7 @@ func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { t.Fatal(err) } + // expected receipts in the canonical block rctCids := []cid.Cid{rct1CID, rct2CID, rct3CID, rct4CID, rct5CID} expectedBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockReceipts)) for i, mockBlockRct := range mocks.MockReceipts { @@ -605,6 +630,7 @@ func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { expectedBlockRctsMap[rctCids[i].String()] = rctModel } + // expected receipts in the canonical block at London height + 1 nonCanonicalBlockRctCids := []cid.Cid{nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID} expectedNonCanonicalBlockRctsMap := make(map[string]models.ReceiptModel, len(mocks.MockNonCanonicalBlockReceipts)) for i, mockNonCanonicalBlockRct := range mocks.MockNonCanonicalBlockReceipts { @@ -612,6 +638,7 @@ func testPublishAndIndexReceiptsNonCanonical(t *testing.T) { expectedNonCanonicalBlockRctsMap[nonCanonicalBlockRctCids[i].String()] = rctModel } + // expected receipts in the canonical block at London height + 1 nonCanonicalBlock2RctCids := []cid.Cid{nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID} expectedNonCanonicalBlock2RctsMap := make(map[string]models.ReceiptModel, len(mocks.MockNonCanonicalBlock2Receipts)) for i, mockNonCanonicalBlock2Rct := range mocks.MockNonCanonicalBlock2Receipts { @@ -695,6 +722,7 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { } mockRcts := make([]rctWithBlockHash, 0) + // logs in the canonical block for _, mockBlockRct := range mocks.MockReceipts { mockRcts = append(mockRcts, rctWithBlockHash{ mockBlockRct, @@ -703,6 +731,7 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { }) } + // logs in the canonical block at London height + 1 for _, mockBlockRct := range mocks.MockNonCanonicalBlockReceipts { mockRcts = append(mockRcts, rctWithBlockHash{ mockBlockRct, @@ -711,6 +740,7 @@ func testPublishAndIndexLogsNonCanonical(t *testing.T) { }) } + // logs in the canonical block at London height + 1 for _, mockBlockRct := range mocks.MockNonCanonicalBlock2Receipts { mockRcts = append(mockRcts, rctWithBlockHash{ mockBlockRct, @@ -776,6 +806,7 @@ func testPublishAndIndexStateNonCanonical(t *testing.T) { removedNodeCID, _ := cid.Decode(shared.RemovedNodeStateCID) stateNodeCIDs := []cid.Cid{state1CID, state2CID, removedNodeCID, removedNodeCID} + // expected state nodes in the canonical and the non-canonical block at London height expectedStateNodes := make([]models.StateNodeModel, 0) for i, stateDiff := range mocks.StateDiffs { expectedStateNodes = append(expectedStateNodes, models.StateNodeModel{ @@ -795,6 +826,7 @@ func testPublishAndIndexStateNonCanonical(t *testing.T) { } }) + // expected state nodes in the non-canonical block at London height + 1 expectedNonCanonicalBlock2StateNodes := make([]models.StateNodeModel, 0) for i, stateDiff := range mocks.StateDiffs[:2] { expectedNonCanonicalBlock2StateNodes = append(expectedNonCanonicalBlock2StateNodes, models.StateNodeModel{ @@ -819,7 +851,7 @@ func testPublishAndIndexStateNonCanonical(t *testing.T) { require.Equal(t, expectedStateNode, stateNodes[i]) } - // check state nodes for non-canonical block + // check state nodes for non-canonical block at London height stateNodes = make([]models.StateNodeModel, 0) err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64(), mockNonCanonicalBlock.Hash().String()) if err != nil { @@ -831,7 +863,7 @@ func testPublishAndIndexStateNonCanonical(t *testing.T) { require.Equal(t, expectedStateNode, stateNodes[i]) } - // check state nodes for non-canonical block at another height + // check state nodes for non-canonical block at London height + 1 stateNodes = make([]models.StateNodeModel, 0) err = db.Select(context.Background(), &stateNodes, pgStr, mocks.Block2Number.Uint64(), mockNonCanonicalBlock2.Hash().String()) if err != nil { @@ -855,6 +887,7 @@ func testPublishAndIndexStorageNonCanonical(t *testing.T) { removedNodeCID, _ := cid.Decode(shared.RemovedNodeStorageCID) storageNodeCIDs := []cid.Cid{storageCID, removedNodeCID, removedNodeCID, removedNodeCID} + // expected state nodes in the canonical and the non-canonical block at London height expectedStorageNodes := make([]models.StorageNodeModel, 0) storageNodeIndex := 0 for _, stateDiff := range mocks.StateDiffs { @@ -879,6 +912,7 @@ func testPublishAndIndexStorageNonCanonical(t *testing.T) { } }) + // expected state nodes in the non-canonical block at London height + 1 expectedNonCanonicalBlock2StorageNodes := make([]models.StorageNodeModel, 0) storageNodeIndex = 0 for _, stateDiff := range mocks.StateDiffs[:2] { @@ -908,7 +942,7 @@ func testPublishAndIndexStorageNonCanonical(t *testing.T) { require.Equal(t, expectedStorageNode, storageNodes[i]) } - // check storage nodes for non-canonical block + // check storage nodes for non-canonical block at London height storageNodes = make([]models.StorageNodeModel, 0) err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64(), mockNonCanonicalBlock.Hash().String()) if err != nil { @@ -920,7 +954,7 @@ func testPublishAndIndexStorageNonCanonical(t *testing.T) { require.Equal(t, expectedStorageNode, storageNodes[i]) } - // check storage nodes for non-canonical block at another height + // check storage nodes for non-canonical block at London height + 1 storageNodes = make([]models.StorageNodeModel, 0) err = db.Select(context.Background(), &storageNodes, pgStr, mockNonCanonicalBlock2.NumberU64(), mockNonCanonicalBlock2.Hash().String()) if err != nil { diff --git a/statediff/indexer/database/sql/pgx_indexer_test.go b/statediff/indexer/database/sql/pgx_indexer_test.go index 168dba5f4..79d3ae252 100644 --- a/statediff/indexer/database/sql/pgx_indexer_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_test.go @@ -57,6 +57,7 @@ func setupPGXNonCanonical(t *testing.T) { setupTestDataNonCanonical(t) } +// Test indexer for a canonical block func TestPGXIndexer(t *testing.T) { t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { setupPGX(t) @@ -591,6 +592,7 @@ func TestPGXIndexer(t *testing.T) { }) } +// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1 func TestPGXIndexerNonCanonical(t *testing.T) { t.Run("Publish and index header", func(t *testing.T) { setupPGXNonCanonical(t) diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index 5f96abf60..a2bf0afd4 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -58,6 +58,7 @@ func setupSQLXNonCanonical(t *testing.T) { setupTestDataNonCanonical(t) } +// Test indexer for a canonical block func TestSQLXIndexer(t *testing.T) { t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { setupSQLX(t) @@ -584,6 +585,7 @@ func TestSQLXIndexer(t *testing.T) { }) } +// Test indexer for a canonical + a non-canonical block at London height + a non-canonical block at London height + 1 func TestSQLXIndexerNonCanonical(t *testing.T) { t.Run("Publish and index header", func(t *testing.T) { setupSQLXNonCanonical(t) diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index 6be79a88e..ceb45233c 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -40,7 +40,10 @@ var ( // TODO: Update this to `MainnetChainConfig` when `LondonBlock` is added TestConfig = params.RopstenChainConfig BlockNumber = TestConfig.LondonBlock - MockHeader = types.Header{ + + // canonical block at London height + // includes 5 transactions: 3 Legacy + 1 EIP-2930 + 1 EIP-1559 + MockHeader = types.Header{ Time: 0, Number: new(big.Int).Set(BlockNumber), Root: common.HexToHash("0x0"), @@ -55,12 +58,16 @@ var ( MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) + // non-canonical block at London height + // includes 2nd and 5th transactions from the canonical block MockNonCanonicalHeader = MockHeader MockNonCanonicalBlockTransactions = types.Transactions{MockTransactions[1], MockTransactions[4]} MockNonCanonicalBlockReceipts = createNonCanonicalBlockReceipts(TestConfig, BlockNumber, MockNonCanonicalBlockTransactions) MockNonCanonicalBlock = types.NewBlock(&MockNonCanonicalHeader, MockNonCanonicalBlockTransactions, nil, MockNonCanonicalBlockReceipts, new(trie.Trie)) MockNonCanonicalHeaderRlp, _ = rlp.EncodeToBytes(MockNonCanonicalBlock.Header()) + // non-canonical block at London height + 1 + // includes 3rd and 5th transactions from the canonical block Block2Number = big.NewInt(BlockNumber.Int64() + 1) MockNonCanonicalHeader2 = types.Header{ Time: 0, @@ -465,6 +472,7 @@ func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big. return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4, signedTrx5}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4, mockReceipt5}, senderAddr } +// createNonCanonicalBlockReceipts is a helper function to generate mock receipts with mock logs for non-canonical blocks func createNonCanonicalBlockReceipts(config *params.ChainConfig, blockNumber *big.Int, transactions types.Transactions) types.Receipts { transactionSigner := types.MakeSigner(config, blockNumber) mockCurve := elliptic.P256() -- 2.45.2