diff --git a/params/version.go b/params/version.go index 95b482ffe..0710ccdad 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release VersionPatch = 8 // Patch version component of the current release - VersionMeta = "statediff-0.0.26" // Version metadata to append to the version string + VersionMeta = "statediff-0.0.27" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/statediff/db/migrations/00007_create_eth_receipt_cids_table.sql b/statediff/db/migrations/00007_create_eth_receipt_cids_table.sql index 856322f32..ba5d58ffe 100644 --- a/statediff/db/migrations/00007_create_eth_receipt_cids_table.sql +++ b/statediff/db/migrations/00007_create_eth_receipt_cids_table.sql @@ -2,8 +2,8 @@ CREATE TABLE eth.receipt_cids ( id SERIAL PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - cid TEXT NOT NULL, - mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + leaf_cid TEXT NOT NULL, + leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, contract VARCHAR(66), contract_hash VARCHAR(66), post_state VARCHAR(66), diff --git a/statediff/db/migrations/00013_create_cid_indexes.sql b/statediff/db/migrations/00013_create_cid_indexes.sql index e123510af..1741e9713 100644 --- a/statediff/db/migrations/00013_create_cid_indexes.sql +++ b/statediff/db/migrations/00013_create_cid_indexes.sql @@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); -- receipt indexes CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id); -CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); +CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); -CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); +CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key); CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); @@ -91,8 +91,8 @@ DROP INDEX eth.state_header_id_index; -- receipt indexes DROP INDEX eth.rct_contract_hash_index; DROP INDEX eth.rct_contract_index; -DROP INDEX eth.rct_mh_index; -DROP INDEX eth.rct_cid_index; +DROP INDEX eth.rct_leaf_mh_index; +DROP INDEX eth.rct_leaf_cid_index; DROP INDEX eth.rct_tx_id_index; -- transaction indexes diff --git a/statediff/db/schema.sql b/statediff/db/schema.sql index 33031744a..d4f592eab 100644 --- a/statediff/db/schema.sql +++ b/statediff/db/schema.sql @@ -363,8 +363,8 @@ ALTER SEQUENCE eth.log_cids_id_seq OWNED BY eth.log_cids.id; CREATE TABLE eth.receipt_cids ( id integer NOT NULL, tx_id integer NOT NULL, - cid text NOT NULL, - mh_key text NOT NULL, + leaf_cid text NOT NULL, + leaf_mh_key text NOT NULL, contract character varying(66), contract_hash character varying(66), post_state character varying(66), @@ -1012,10 +1012,10 @@ CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3); -- --- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: - +-- Name: rct_leaf_cid_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); +CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); -- @@ -1033,10 +1033,10 @@ CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); -- --- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: - +-- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: - -- -CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); +CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key); -- @@ -1287,7 +1287,7 @@ ALTER TABLE ONLY eth.log_cids -- ALTER TABLE ONLY eth.receipt_cids - ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; + ADD CONSTRAINT receipt_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; -- diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index eaf3a92d1..761bd3c29 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -110,13 +110,13 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip } // Generate the block iplds - headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, err := ipld.FromBlockAndReceipts(block, receipts) + headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld.FromBlockAndReceipts(block, receipts) if err != nil { return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err) } - if len(txNodes) != len(txTrieNodes) && len(rctNodes) != len(rctTrieNodes) && len(txNodes) != len(rctNodes) { - return nil, fmt.Errorf("expected number of transactions (%d), transaction trie nodes (%d), receipts (%d), and receipt trie nodes (%d)to be equal", len(txNodes), len(txTrieNodes), len(rctNodes), len(rctTrieNodes)) + if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) { + return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d)to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs)) } // Calculate reward @@ -202,6 +202,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip txTrieNodes: txTrieNodes, logTrieNodes: logTrieNodes, logLeafNodeCIDs: logLeafNodeCIDs, + rctLeafNodeCIDs: rctLeafNodeCIDs, }) if err != nil { return nil, err @@ -288,6 +289,7 @@ type processArgs struct { txTrieNodes []*ipld.EthTxTrie logTrieNodes [][]*ipld.EthLogTrie logLeafNodeCIDs [][]cid.Cid + rctLeafNodeCIDs []cid.Cid } // processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres @@ -309,13 +311,10 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs } // publish the txs and receipts - txNode, rctNode := args.txNodes[i], args.rctNodes[i] + txNode := args.txNodes[i] if err := shared.PublishIPLD(tx, txNode); err != nil { return fmt.Errorf("error publishing tx IPLD: %v", err) } - if err := shared.PublishIPLD(tx, rctNode); err != nil { - return fmt.Errorf("error publishing rct IPLD: %v", err) - } // Indexing // extract topic and contract data from the receipt for indexing @@ -390,13 +389,18 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs return err } } + // index the receipt + if !args.rctLeafNodeCIDs[i].Defined() { + return fmt.Errorf("invalid receipt leaf node cid") + } + rctModel := &models.ReceiptModel{ Contract: contract, ContractHash: contractHash, - CID: rctNode.Cid().String(), - MhKey: shared.MultihashKeyFromCID(rctNode.Cid()), - LogRoot: rctNode.LogRoot.String(), + LeafCID: args.rctLeafNodeCIDs[i].String(), + LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]), + LogRoot: args.rctNodes[i].LogRoot.String(), } if len(receipt.PostState) == 0 { rctModel.PostStatus = receipt.Status diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go index 7efbcc3fd..70ab4cd90 100644 --- a/statediff/indexer/indexer_test.go +++ b/statediff/indexer/indexer_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/models" @@ -310,9 +311,21 @@ func TestPublishAndIndexer(t *testing.T) { } }) - t.Run("Publish and index log IPLDs for single receipt", func(t *testing.T) { + t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) { setup(t) defer tearDown(t) + + rcts := make([]string, 0) + pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.id + AND transaction_cids.header_id = header_cids.id + AND header_cids.block_number = $1 + ORDER BY transaction_cids.index` + err = db.Select(&rcts, pgStr, mocks.BlockNumber.Uint64()) + if err != nil { + t.Fatal(err) + } + type logIPLD struct { Index int `db:"index"` Address string `db:"address"` @@ -320,52 +333,72 @@ func TestPublishAndIndexer(t *testing.T) { Topic0 string `db:"topic0"` Topic1 string `db:"topic1"` } - - results := make([]logIPLD, 0) - pgStr := `SELECT log_cids.index, log_cids.address, log_cids.Topic0, log_cids.Topic1, data FROM eth.log_cids + for i := range rcts { + results := make([]logIPLD, 0) + pgStr = `SELECT log_cids.index, log_cids.address, log_cids.Topic0, log_cids.Topic1, data FROM eth.log_cids INNER JOIN eth.receipt_cids ON (log_cids.receipt_id = receipt_cids.id) INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key) - WHERE receipt_cids.cid = $1 ORDER BY eth.log_cids.index ASC` - err = db.Select(&results, pgStr, rct4CID.String()) - require.NoError(t, err) - - // expecting MockLog1 and MockLog2 for mockReceipt4 - expectedLogs := mocks.MockReceipts[3].Logs - shared.ExpectEqual(t, len(results), len(expectedLogs)) - - var nodeElements []interface{} - for idx, r := range results { - // Decode the log leaf node. - err = rlp.DecodeBytes(r.Data, &nodeElements) + WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC` + err = db.Select(&results, pgStr, rcts[i]) require.NoError(t, err) - logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) - require.NoError(t, err) + // expecting MockLog1 and MockLog2 for mockReceipt4 + expectedLogs := mocks.MockReceipts[i].Logs + shared.ExpectEqual(t, len(results), len(expectedLogs)) - // 2nd element of the leaf node contains the encoded log data. - shared.ExpectEqual(t, logRaw, nodeElements[1].([]byte)) + var nodeElements []interface{} + for idx, r := range results { + // Decode the log leaf node. + err = rlp.DecodeBytes(r.Data, &nodeElements) + require.NoError(t, err) + + logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) + require.NoError(t, err) + + // 2nd element of the leaf node contains the encoded log data. + shared.ExpectEqual(t, logRaw, nodeElements[1].([]byte)) + } } }) t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) { setup(t) defer tearDown(t) + // check receipts were properly indexed rcts := make([]string, 0) - pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id - AND header_cids.block_number = $1` + AND header_cids.block_number = $1 order by transaction_cids.id` err = db.Select(&rcts, pgStr, mocks.BlockNumber.Uint64()) if err != nil { t.Fatal(err) } shared.ExpectEqual(t, len(rcts), 5) - expectTrue(t, shared.ListContainsString(rcts, rct1CID.String())) - expectTrue(t, shared.ListContainsString(rcts, rct2CID.String())) - expectTrue(t, shared.ListContainsString(rcts, rct3CID.String())) - expectTrue(t, shared.ListContainsString(rcts, rct4CID.String())) - expectTrue(t, shared.ListContainsString(rcts, rct5CID.String())) + + for idx, rctLeafCID := range rcts { + result := make([]ipfs.BlockModel, 0) + pgStr = `SELECT data + FROM eth.receipt_cids + INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key) + WHERE receipt_cids.leaf_cid = $1` + err = db.Select(&result, pgStr, rctLeafCID) + if err != nil { + t.Fatal(err) + } + + // Decode the log leaf node. + var nodeElements []interface{} + err = rlp.DecodeBytes(result[0].Data, &nodeElements) + require.NoError(t, err) + + expectedRct, err := mocks.MockReceipts[idx].MarshalBinary() + require.NoError(t, err) + + shared.ExpectEqual(t, expectedRct, nodeElements[1].([]byte)) + } + // and published for _, c := range rcts { dc, err := cid.Decode(c) @@ -379,11 +412,12 @@ func TestPublishAndIndexer(t *testing.T) { if err != nil { t.Fatal(err) } + switch c { case rct1CID.String(): shared.ExpectEqual(t, data, rct1) var postStatus uint64 - pgStr = `SELECT post_status FROM eth.receipt_cids WHERE cid = $1` + pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1` err = db.Get(&postStatus, pgStr, c) if err != nil { t.Fatal(err) @@ -392,7 +426,7 @@ func TestPublishAndIndexer(t *testing.T) { case rct2CID.String(): shared.ExpectEqual(t, data, rct2) var postState string - pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1` + pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` err = db.Get(&postState, pgStr, c) if err != nil { t.Fatal(err) @@ -401,7 +435,7 @@ func TestPublishAndIndexer(t *testing.T) { case rct3CID.String(): shared.ExpectEqual(t, data, rct3) var postState string - pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1` + pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` err = db.Get(&postState, pgStr, c) if err != nil { t.Fatal(err) @@ -410,7 +444,7 @@ func TestPublishAndIndexer(t *testing.T) { case rct4CID.String(): shared.ExpectEqual(t, data, rct4) var postState string - pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1` + pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` err = db.Get(&postState, pgStr, c) if err != nil { t.Fatal(err) @@ -419,7 +453,7 @@ func TestPublishAndIndexer(t *testing.T) { case rct5CID.String(): shared.ExpectEqual(t, data, rct5) var postState string - pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1` + pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` err = db.Get(&postState, pgStr, c) if err != nil { t.Fatal(err) diff --git a/statediff/indexer/ipfs/ipld/eth_parser.go b/statediff/indexer/ipfs/ipld/eth_parser.go index 1e9f07888..0b4780f8a 100644 --- a/statediff/indexer/ipfs/ipld/eth_parser.go +++ b/statediff/indexer/ipfs/ipld/eth_parser.go @@ -123,11 +123,11 @@ func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) { // FromBlockAndReceipts takes a block and processes it // to return it a set of IPLD nodes for further processing. -func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, error) { +func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) { // Process the header headerNode, err := NewEthHeader(block.Header()) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, nil, nil, nil, err } // Process the uncles @@ -135,7 +135,7 @@ func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHe for i, uncle := range block.Uncles() { uncleNode, err := NewEthHeader(uncle) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, nil, nil, nil, err } uncleNodes[i] = uncleNode } @@ -144,14 +144,14 @@ func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHe txNodes, txTrieNodes, err := processTransactions(block.Transactions(), block.Header().TxHash[:]) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, nil, nil, nil, err } // Process the receipts and logs - rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, err := processReceiptsAndLogs(receipts, + rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts, block.Header().ReceiptHash[:]) - return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, err + return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err } // processTransactions will take the found transactions in a parsed block body @@ -166,7 +166,7 @@ func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*Et return nil, nil, err } ethTxNodes = append(ethTxNodes, ethTx) - if err := transactionTrie.add(idx, ethTx.RawData()); err != nil { + if err := transactionTrie.Add(idx, ethTx.RawData()); err != nil { return nil, nil, err } } @@ -179,20 +179,20 @@ func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*Et } // processReceiptsAndLogs will take in receipts -// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie -func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, error) { +// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID +func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) { // Pre allocating memory. ethRctNodes := make([]*EthReceipt, 0, len(rcts)) ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts)) ethLogTrieNodes := make([][]*EthLogTrie, 0, len(rcts)) - receiptTrie := newRctTrie() + receiptTrie := NewRctTrie() for idx, rct := range rcts { // Process logs for each receipt. logTrieNodes, leafNodeCids, logTrieHash, err := processLogs(rct.Logs) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err } rct.LogRoot = logTrieHash ethLogTrieNodes = append(ethLogTrieNodes, logTrieNodes) @@ -200,20 +200,42 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E ethRct, err := NewReceipt(rct) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err } ethRctNodes = append(ethRctNodes, ethRct) - if err = receiptTrie.add(idx, ethRct.RawData()); err != nil { - return nil, nil, nil, nil, err + if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil { + return nil, nil, nil, nil, nil, err } } if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) { - return nil, nil, nil, nil, fmt.Errorf("wrong receipt hash computed") + return nil, nil, nil, nil, nil, fmt.Errorf("wrong receipt hash computed") } - rctTrieNodes, err := receiptTrie.getNodes() - return ethRctNodes, rctTrieNodes, ethLogTrieNodes, ethLogleafNodeCids, err + + rctTrieNodes, err := receiptTrie.GetNodes() + if err != nil { + return nil, nil, nil, nil, nil, err + } + + rctLeafNodes, keys, err := receiptTrie.GetLeafNodes() + if err != nil { + return nil, nil, nil, nil, nil, err + } + + ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes)) + for i, rln := range rctLeafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + err = rlp.Decode(r, &idx) + if err != nil { + return nil, nil, nil, nil, nil, err + } + ethRctleafNodeCids[idx] = rln.Cid() + } + + return ethRctNodes, rctTrieNodes, ethLogTrieNodes, ethLogleafNodeCids, ethRctleafNodeCids, err } func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, error) { @@ -223,7 +245,7 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro if err != nil { return nil, nil, common.Hash{}, err } - if err = logTr.add(idx, ethLog.RawData()); err != nil { + if err = logTr.Add(idx, ethLog.RawData()); err != nil { return nil, nil, common.Hash{}, err } } @@ -242,7 +264,7 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro for i, ln := range leafNodes { var idx uint - r := bytes.NewReader(keys[i].trieKey) + r := bytes.NewReader(keys[i].TrieKey) err = rlp.Decode(r, &idx) if err != nil { return nil, nil, common.Hash{}, err diff --git a/statediff/indexer/ipfs/ipld/eth_parser_test.go b/statediff/indexer/ipfs/ipld/eth_parser_test.go index 2b4050562..9cb8d4e46 100644 --- a/statediff/indexer/ipfs/ipld/eth_parser_test.go +++ b/statediff/indexer/ipfs/ipld/eth_parser_test.go @@ -91,7 +91,7 @@ func loadBlockData(t *testing.T) []testCase { func TestFromBlockAndReceipts(t *testing.T) { testCases := loadBlockData(t) for _, tc := range testCases { - _, _, _, _, _, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts) + _, _, _, _, _, _, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts) if err != nil { t.Fatalf("error generating IPLDs from block and receipts, err %v, kind %s, block hash %s", err, tc.kind, tc.block.Hash()) } diff --git a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go index ee371c15a..fc1480703 100644 --- a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go +++ b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go @@ -113,17 +113,17 @@ type rctTrie struct { *localTrie } -// newRctTrie initializes and returns a rctTrie. -func newRctTrie() *rctTrie { +// NewRctTrie initializes and returns a rctTrie. +func NewRctTrie() *rctTrie { return &rctTrie{ localTrie: newLocalTrie(), } } -// getNodes invokes the localTrie, which computes the root hash of the +// GetNodes invokes the localTrie, which computes the root hash of the // transaction trie and returns its database keys, to return a slice // of EthRctTrie nodes. -func (rt *rctTrie) getNodes() ([]*EthRctTrie, error) { +func (rt *rctTrie) GetNodes() ([]*EthRctTrie, error) { keys, err := rt.getKeys() if err != nil { return nil, err @@ -131,20 +131,51 @@ func (rt *rctTrie) getNodes() ([]*EthRctTrie, error) { var out []*EthRctTrie for _, k := range keys { - rawdata, err := rt.db.Get(k) + n, err := rt.getNodeFromDB(k) if err != nil { return nil, err } - c, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256) - if err != nil { - return nil, err - } - tn := &TrieNode{ - cid: c, - rawdata: rawdata, - } - out = append(out, &EthRctTrie{TrieNode: tn}) + out = append(out, n) } return out, nil } + +// GetLeafNodes invokes the localTrie, which returns a slice +// of EthRctTrie leaf nodes. +func (rt *rctTrie) GetLeafNodes() ([]*EthRctTrie, []*nodeKey, error) { + keys, err := rt.getLeafKeys() + if err != nil { + return nil, nil, err + } + + out := make([]*EthRctTrie, 0, len(keys)) + for _, k := range keys { + n, err := rt.getNodeFromDB(k.dbKey) + if err != nil { + return nil, nil, err + } + out = append(out, n) + } + + return out, keys, nil +} + +func (rt *rctTrie) getNodeFromDB(key []byte) (*EthRctTrie, error) { + rawdata, err := rt.db.Get(key) + if err != nil { + return nil, err + } + + cid, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256) + if err != nil { + return nil, err + } + + tn := &TrieNode{ + cid: cid, + rawdata: rawdata, + } + + return &EthRctTrie{TrieNode: tn}, nil +} diff --git a/statediff/indexer/ipfs/ipld/shared.go b/statediff/indexer/ipfs/ipld/shared.go index a6f4dd492..41887be2d 100644 --- a/statediff/indexer/ipfs/ipld/shared.go +++ b/statediff/indexer/ipfs/ipld/shared.go @@ -112,9 +112,9 @@ func newLocalTrie() *localTrie { return lt } -// add receives the index of an object and its rawdata value +// Add receives the index of an object and its rawdata value // and includes it into the localTrie -func (lt *localTrie) add(idx int, rawdata []byte) error { +func (lt *localTrie) Add(idx int, rawdata []byte) error { key, err := rlp.EncodeToBytes(uint(idx)) if err != nil { panic(err) @@ -128,19 +128,27 @@ func (lt *localTrie) rootHash() []byte { return lt.trie.Hash().Bytes() } -// getKeys returns the stored keys of the memory database -// of the localTrie for further processing. -func (lt *localTrie) getKeys() ([][]byte, error) { +func (lt *localTrie) commit() error { // commit trie nodes to trieDB var err error _, err = lt.trie.Commit(nil) if err != nil { - return nil, err + return err } // commit trieDB to the underlying ethdb.Database if err := lt.trieDB.Commit(lt.trie.Hash(), false, nil); err != nil { + return err + } + return nil +} + +// getKeys returns the stored keys of the memory database +// of the localTrie for further processing. +func (lt *localTrie) getKeys() ([][]byte, error) { + if err := lt.commit(); err != nil { return nil, err } + // collect all of the node keys it := lt.trie.NodeIterator([]byte{}) keyBytes := make([][]byte, 0) @@ -155,14 +163,17 @@ func (lt *localTrie) getKeys() ([][]byte, error) { type nodeKey struct { dbKey []byte - trieKey []byte + TrieKey []byte } // getLeafKeys returns the stored leaf keys from the memory database // of the localTrie for further processing. func (lt *localTrie) getLeafKeys() ([]*nodeKey, error) { - it := lt.trie.NodeIterator([]byte{}) + if err := lt.commit(); err != nil { + return nil, err + } + it := lt.trie.NodeIterator([]byte{}) leafKeys := make([]*nodeKey, 0) for it.Next(true) { if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { @@ -183,7 +194,7 @@ func (lt *localTrie) getLeafKeys() ([]*nodeKey, error) { encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - leafKeys = append(leafKeys, &nodeKey{dbKey: it.Hash().Bytes(), trieKey: leafKey}) + leafKeys = append(leafKeys, &nodeKey{dbKey: it.Hash().Bytes(), TrieKey: leafKey}) } return leafKeys, nil } diff --git a/statediff/indexer/ipfs/ipld/test_data/tx_data b/statediff/indexer/ipfs/ipld/test_data/tx_data new file mode 100644 index 000000000..1ec58bd90 Binary files /dev/null and b/statediff/indexer/ipfs/ipld/test_data/tx_data differ diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index cecf18915..cb9404385 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -77,8 +77,8 @@ type AccessListElementModel struct { type ReceiptModel struct { ID int64 `db:"id"` TxID int64 `db:"tx_id"` - CID string `db:"cid"` - MhKey string `db:"mh_key"` + LeafCID string `db:"leaf_cid"` + LeafMhKey string `db:"leaf_mh_key"` PostStatus uint64 `db:"post_status"` PostState string `db:"post_state"` Contract string `db:"contract"` diff --git a/statediff/indexer/writer.go b/statediff/indexer/writer.go index 6b730e261..60ed4fbc0 100644 --- a/statediff/indexer/writer.go +++ b/statediff/indexer/writer.go @@ -93,10 +93,10 @@ func (in *PostgresCIDWriter) upsertAccessListElement(tx *sqlx.Tx, accessListElem func (in *PostgresCIDWriter) upsertReceiptCID(tx *sqlx.Tx, rct *models.ReceiptModel, txID int64) (int64, error) { var receiptID int64 - err := tx.QueryRowx(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, mh_key, post_state, post_status, log_root) = ($2, $3, $4, $5, $6, $7, $8) + err := tx.QueryRowx(`INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (tx_id) DO UPDATE SET (leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) = ($2, $3, $4, $5, $6, $7, $8) RETURNING id`, - txID, rct.CID, rct.Contract, rct.ContractHash, rct.MhKey, rct.PostState, rct.PostStatus, rct.LogRoot).Scan(&receiptID) + txID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState, rct.PostStatus, rct.LogRoot).Scan(&receiptID) if err != nil { return 0, fmt.Errorf("error upserting receipt_cids entry: %w", err) }