diff --git a/db/migrations/00007_create_eth_receipt_cids_table.sql b/db/migrations/00007_create_eth_receipt_cids_table.sql index 5ee8c92c..ee1cc350 100644 --- a/db/migrations/00007_create_eth_receipt_cids_table.sql +++ b/db/migrations/00007_create_eth_receipt_cids_table.sql @@ -2,8 +2,8 @@ CREATE TABLE eth.receipt_cids ( id SERIAL PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - cid TEXT NOT NULL, - mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + leaf_cid TEXT NOT NULL, + leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, contract VARCHAR(66), contract_hash VARCHAR(66), post_state VARCHAR(66), diff --git a/db/migrations/00013_create_cid_indexes.sql b/db/migrations/00013_create_cid_indexes.sql index 2038c867..58ab67e5 100644 --- a/db/migrations/00013_create_cid_indexes.sql +++ b/db/migrations/00013_create_cid_indexes.sql @@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src); -- receipt indexes CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id); -CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); +CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid); -CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); +CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key); CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); diff --git a/docker-compose.yml b/docker-compose.yml index 28a62930..0eaeec95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: unless-stopped depends_on: - statediff-migrations - image: vulcanize/dapptools:v0.29.0-v1.10.8-statediff-0.0.26 + image: vulcanize/dapptools:latest environment: DB_USER: vdbm DB_NAME: vulcanize_public @@ -20,7 +20,7 @@ services: restart: on-failure depends_on: - db - image: vulcanize/statediff-migrations:v0.6.0 + image: vulcanize/statediff-migrations:latest environment: DATABASE_USER: vdbm DATABASE_NAME: vulcanize_public diff --git a/pkg/.DS_Store b/pkg/.DS_Store new file mode 100644 index 00000000..19007879 Binary files /dev/null and b/pkg/.DS_Store differ diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 66cd7bb6..c2632216 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -376,7 +376,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber var receipts []*types.Receipt for _, rctIPLD := range rctIPLDs { var receipt types.Receipt - err = receipt.UnmarshalBinary(rctIPLD.Data) + nodeVal, err := DecodeLeafNode(rctIPLD.Data) + if err != nil { + return nil, err + } + + err = receipt.UnmarshalBinary(nodeVal) if err != nil { return nil, err } @@ -476,7 +481,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo var receipts []*types.Receipt for _, rctIPLD := range rctIPLDs { var receipt types.Receipt - err = receipt.UnmarshalBinary(rctIPLD.Data) + nodeVal, err := DecodeLeafNode(rctIPLD.Data) + if err != nil { + return nil, err + } + + err = receipt.UnmarshalBinary(nodeVal) if err != nil { return nil, err } diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index dac04955..c0167a39 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -292,7 +292,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) { log.Debug("retrieving receipt cids for header id ", headerID) args := make([]interface{}, 0, 4) - pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key, + pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.contract, receipt_cids.contract_hash FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE receipt_cids.tx_id = transaction_cids.id @@ -316,12 +316,12 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF id := 1 pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, - eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.cid, eth.receipt_cids.post_status + eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid, eth.receipt_cids.post_status FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE eth.log_cids.receipt_id = receipt_cids.id AND receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id - AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1` + AND receipt_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1` args = append(args, blockHash.String()) id++ @@ -380,7 +380,7 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilte func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) { log.Debug("retrieving receipt cids for block ", blockNumber) args := make([]interface{}, 0, 5) - pgStr := `SELECT receipt_cids.id, receipt_cids.cid, receipt_cids.mh_key, receipt_cids.tx_id + pgStr := `SELECT receipt_cids.id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE receipt_cids.tx_id = transaction_cids.id AND transaction_cids.header_id = header_cids.id` @@ -595,7 +595,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ( // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) { log.Debugf("retrieving receipt cids for tx ids %v", txIDs) - pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key, + pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.contract, receipt_cids.contract_hash FROM eth.receipt_cids, eth.transaction_cids WHERE tx_id = ANY($1::INTEGER[]) diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/cid_retriever_test.go index e972aee3..c575e3cc 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/cid_retriever_test.go @@ -239,6 +239,17 @@ var _ = Describe("Retriever", func() { Expect(err).ToNot(HaveOccurred()) }) It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { + type rctCIDAndMHKeyResult struct { + LeafCID string `db:"leaf_cid"` + LeafMhKey string `db:"leaf_mh_key"` + } + expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0) + pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.id + AND transaction_cids.header_id = header_cids.id + AND header_cids.block_number = $1 + ORDER BY transaction_cids.index` + err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) cids, empty, err := retriever.Retrieve(openFilter, 1) Expect(err).ToNot(HaveOccurred()) Expect(empty).ToNot(BeTrue()) @@ -253,10 +264,10 @@ var _ = Describe("Retriever", func() { Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue()) Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue()) Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue()) - Expect(len(cids[0].Receipts)).To(Equal(4)) - Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue()) - Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue()) - Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue()) + Expect(len(cids[0].Receipts)).To(Equal(3)) + Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue()) + Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue()) + Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue()) Expect(len(cids[0].StateNodes)).To(Equal(2)) for _, stateNode := range cids[0].StateNodes { @@ -279,6 +290,17 @@ var _ = Describe("Retriever", func() { }) It("Applies filters from the provided config.Subscription", func() { + type rctCIDAndMHKeyResult struct { + LeafCID string `db:"leaf_cid"` + LeafMhKey string `db:"leaf_mh_key"` + } + expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0) + pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.id + AND transaction_cids.header_id = header_cids.id + AND header_cids.block_number = $1 + ORDER BY transaction_cids.index` + err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1) Expect(err).ToNot(HaveOccurred()) Expect(empty).ToNot(BeTrue()) @@ -292,6 +314,8 @@ var _ = Describe("Retriever", func() { expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0] expectedReceiptCID.ID = cids1[0].Receipts[0].ID expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID + expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID + expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID)) cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1) @@ -307,6 +331,8 @@ var _ = Describe("Retriever", func() { expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0] expectedReceiptCID.ID = cids2[0].Receipts[0].ID expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID + expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID + expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID)) cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1) @@ -322,6 +348,8 @@ var _ = Describe("Retriever", func() { expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0] expectedReceiptCID.ID = cids3[0].Receipts[0].ID expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID + expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID + expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID)) cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1) @@ -337,6 +365,8 @@ var _ = Describe("Retriever", func() { expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1] expectedReceiptCID.ID = cids4[0].Receipts[0].ID expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID + expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID + expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID)) cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1) @@ -351,10 +381,10 @@ var _ = Describe("Retriever", func() { Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue()) Expect(len(cids5[0].StateNodes)).To(Equal(0)) Expect(len(cids5[0].StorageNodes)).To(Equal(0)) - Expect(len(cids5[0].Receipts)).To(Equal(4)) - Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue()) - Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue()) - Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue()) + Expect(len(cids5[0].Receipts)).To(Equal(3)) + Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue()) + Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue()) + Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue()) cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1) Expect(err).ToNot(HaveOccurred()) @@ -373,6 +403,8 @@ var _ = Describe("Retriever", func() { expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1] expectedReceiptCID.ID = cids6[0].Receipts[0].ID expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID + expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID + expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID)) cids7, empty, err := retriever.Retrieve(stateFilter, 1) diff --git a/pkg/eth/filterer.go b/pkg/eth/filterer.go index 93923c43..b9dac454 100644 --- a/pkg/eth/filterer.go +++ b/pkg/eth/filterer.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" @@ -164,7 +165,8 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error { if !receiptFilter.Off { response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts)) - for _, receipt := range payload.Receipts { + rctLeafCID, rctIPLDData := FetchRctLeafNodeData(payload.Receipts) + for idx, receipt := range payload.Receipts { // topics is always length 4 topics := make([][]string, 4) contracts := make([]string, len(receipt.Logs)) @@ -177,18 +179,9 @@ func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response * // TODO: Verify this filter logic. if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) { - receiptBuffer := new(bytes.Buffer) - if err := receipt.EncodeRLP(receiptBuffer); err != nil { - return err - } - data := receiptBuffer.Bytes() - cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256) - if err != nil { - return err - } response.Receipts = append(response.Receipts, ipfs.BlockModel{ - Data: data, - CID: cid.String(), + Data: rctIPLDData[idx], + CID: rctLeafCID[idx].String(), }) } } @@ -329,3 +322,44 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool { } return false } + +// FetchRctLeafNodeData fetches receipt leaf node IPLD data and CIDs +func FetchRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte) { + receiptTrie := ipld.NewRctTrie() + for idx, rct := range rcts { + ethRct, err := ipld.NewReceipt(rct) + if err != nil { + return nil, nil + } + if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil { + return nil, nil + } + } + + _, err := receiptTrie.GetNodes() + if err != nil { + return nil, nil + } + + rctLeafNodes, keys, err := receiptTrie.GetLeafNodes() + if err != nil { + return nil, nil + } + + ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes)) + ethRctleafNodeData := make([][]byte, len(rctLeafNodes)) + for i, rln := range rctLeafNodes { + var idx uint + + r := bytes.NewReader(keys[i].TrieKey) + err = rlp.Decode(r, &idx) + if err != nil { + return nil, nil + } + + ethRctleafNodeCids[idx] = rln.Cid() + ethRctleafNodeData[idx] = rln.RawData() + } + + return ethRctleafNodeCids, ethRctleafNodeData +} diff --git a/pkg/eth/filterer_test.go b/pkg/eth/filterer_test.go index 950b281d..eb09dd5e 100644 --- a/pkg/eth/filterer_test.go +++ b/pkg/eth/filterer_test.go @@ -52,10 +52,10 @@ var _ = Describe("Filterer", func() { Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx1)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue()) - Expect(len(iplds.Receipts)).To(Equal(4)) - Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1)).To(BeTrue()) - Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2)).To(BeTrue()) - Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3)).To(BeTrue()) + Expect(len(iplds.Receipts)).To(Equal(3)) + Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1IPLD)).To(BeTrue()) + Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2IPLD)).To(BeTrue()) + Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3IPLD)).To(BeTrue()) Expect(len(iplds.StateNodes)).To(Equal(2)) for _, stateNode := range iplds.StateNodes { Expect(stateNode.Type).To(Equal(sdtypes.Leaf)) @@ -87,8 +87,8 @@ var _ = Describe("Filterer", func() { Expect(len(iplds1.StateNodes)).To(Equal(0)) Expect(len(iplds1.Receipts)).To(Equal(1)) Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{ - Data: test_helpers.Rct1IPLD.RawData(), - CID: test_helpers.Rct1IPLD.Cid().String(), + Data: test_helpers.Rct1IPLD, + CID: test_helpers.Rct1CID.String(), })) iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload) @@ -102,8 +102,8 @@ var _ = Describe("Filterer", func() { Expect(len(iplds2.StateNodes)).To(Equal(0)) Expect(len(iplds2.Receipts)).To(Equal(1)) Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{ - Data: test_helpers.Rct1IPLD.RawData(), - CID: test_helpers.Rct1IPLD.Cid().String(), + Data: test_helpers.Rct1IPLD, + CID: test_helpers.Rct1CID.String(), })) iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload) @@ -117,8 +117,8 @@ var _ = Describe("Filterer", func() { Expect(len(iplds3.StateNodes)).To(Equal(0)) Expect(len(iplds3.Receipts)).To(Equal(1)) Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{ - Data: test_helpers.Rct1IPLD.RawData(), - CID: test_helpers.Rct1IPLD.Cid().String(), + Data: test_helpers.Rct1IPLD, + CID: test_helpers.Rct1CID.String(), })) iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload) @@ -132,8 +132,8 @@ var _ = Describe("Filterer", func() { Expect(len(iplds4.StateNodes)).To(Equal(0)) Expect(len(iplds4.Receipts)).To(Equal(1)) Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{ - Data: test_helpers.Rct2IPLD.RawData(), - CID: test_helpers.Rct2IPLD.Cid().String(), + Data: test_helpers.Rct2IPLD, + CID: test_helpers.Rct2CID.String(), })) iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload) @@ -148,10 +148,10 @@ var _ = Describe("Filterer", func() { Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx3)).To(BeTrue()) Expect(len(iplds5.StorageNodes)).To(Equal(0)) Expect(len(iplds5.StateNodes)).To(Equal(0)) - Expect(len(iplds5.Receipts)).To(Equal(4)) - Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1)).To(BeTrue()) - Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2)).To(BeTrue()) - Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3)).To(BeTrue()) + Expect(len(iplds5.Receipts)).To(Equal(3)) + Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1IPLD)).To(BeTrue()) + Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2IPLD)).To(BeTrue()) + Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3IPLD)).To(BeTrue()) iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload) Expect(err).ToNot(HaveOccurred()) @@ -165,8 +165,8 @@ var _ = Describe("Filterer", func() { Expect(len(iplds6.StateNodes)).To(Equal(0)) Expect(len(iplds6.Receipts)).To(Equal(1)) Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{ - Data: test_helpers.Rct2IPLD.RawData(), - CID: test_helpers.Rct2IPLD.Cid().String(), + Data: test_helpers.Rct2IPLD, + CID: test_helpers.Rct2CID.String(), })) iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload) diff --git a/pkg/eth/ipld_fetcher.go b/pkg/eth/ipld_fetcher.go index 18330143..94047004 100644 --- a/pkg/eth/ipld_fetcher.go +++ b/pkg/eth/ipld_fetcher.go @@ -153,13 +153,14 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs log.Debug("fetching receipt iplds") rctIPLDs := make([]ipfs.BlockModel, len(cids)) for i, c := range cids { - rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) + rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey) if err != nil { return nil, err } + //nodeVal, err := DecodeLeafNode(rctBytes) rctIPLDs[i] = ipfs.BlockModel{ Data: rctBytes, - CID: c.CID, + CID: c.LeafCID, } } return rctIPLDs, nil diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index a8b0156d..6609865d 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -19,6 +19,9 @@ package eth import ( "fmt" + "github.com/ethereum/go-ethereum/statediff/trie" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -81,29 +84,29 @@ const ( FROM eth.transaction_cids INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) WHERE tx_hash = $1` - RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data + RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) - INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE tx_hash = ANY($1::VARCHAR(66)[])` - RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) - INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` - RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data + RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) - INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE block_number = $1 ORDER BY eth.transaction_cids.index ASC` - RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data + RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) - INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) WHERE tx_hash = $1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids @@ -149,6 +152,12 @@ const ( LIMIT 1` ) +type rctIpldResult struct { + LeafCID string `db:"leaf_cid"` + Data []byte `db:"data"` + TxHash string `db:"tx_hash"` +} + type ipldResult struct { CID string `db:"cid"` Data []byte `db:"data"` @@ -316,9 +325,26 @@ func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, [ return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex()) } +// DecodeLeafNode decodes the leaf node data +func DecodeLeafNode(node []byte) ([]byte, error) { + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return nil, err + } + ty, err := trie.CheckKeyType(nodeElements) + if err != nil { + return nil, err + } + + if ty != sdtypes.Leaf { + return nil, fmt.Errorf("expected leaf node but found %s", ty) + } + return nodeElements[1].([]byte), nil +} + // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) { - rctResults := make([]ipldResult, 0) + rctResults := make([]rctIpldResult, 0) hashStrs := make([]string, len(hashes)) for i, hash := range hashes { hashStrs[i] = hash.Hex() @@ -329,15 +355,20 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri cids := make([]string, len(rctResults)) rcts := make([][]byte, len(rctResults)) for i, res := range rctResults { - cids[i] = res.CID - rcts[i] = res.Data + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, err + } + rcts[i] = nodeVal + //rcts[i] = res.Data } return cids, rcts, nil } // RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) { - rctResults := make([]ipldResult, 0) + rctResults := make([]rctIpldResult, 0) if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil { return nil, nil, nil, err } @@ -346,8 +377,13 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, txs := make([]common.Hash, len(rctResults)) for i, res := range rctResults { - cids[i] = res.CID - rcts[i] = res.Data + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, nil, err + } + rcts[i] = nodeVal + //rcts[i] = res.Data txs[i] = common.HexToHash(res.TxHash) } @@ -356,23 +392,28 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, // RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) { - rctResults := make([]ipldResult, 0) + rctResults := make([]rctIpldResult, 0) if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil { return nil, nil, err } cids := make([]string, len(rctResults)) rcts := make([][]byte, len(rctResults)) for i, res := range rctResults { - cids[i] = res.CID - rcts[i] = res.Data + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, err + } + rcts[i] = nodeVal + //rcts[i] = res.Data } return cids, rcts, nil } // RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) { - rctResult := new(ipldResult) - return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) + rctResult := new(rctIpldResult) + return rctResult.LeafCID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) } type nodeInfo struct { diff --git a/pkg/eth/test_helpers.go b/pkg/eth/test_helpers.go index 219743a7..491f837b 100644 --- a/pkg/eth/test_helpers.go +++ b/pkg/eth/test_helpers.go @@ -59,7 +59,7 @@ func TxModelsContainsCID(txs []models.TxModel, cid string) bool { // ListContainsBytes used to check if a list of byte arrays contains a particular byte array func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool { for _, rct := range rcts { - if rct.CID == cid { + if rct.LeafCID == cid { return true } } diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index f769315c..5d7bd84e 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipfs" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/testhelpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" @@ -47,6 +48,7 @@ import ( // Test variables var ( // block data + db *postgres.DB BlockNumber = big.NewInt(1) MockHeader = types.Header{ Time: 0, @@ -165,36 +167,36 @@ var ( Tx3 = GetTxnRlp(2, MockTransactions) Tx4 = GetTxnRlp(3, MockTransactions) - Rct1 = GetRctRlp(0, MockReceipts) - Rct2 = GetRctRlp(1, MockReceipts) - Rct3 = GetRctRlp(2, MockReceipts) - Rct4 = GetRctRlp(3, MockReceipts) - - HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) - HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID) - Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256) - Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID) - Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256) - Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID) - Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256) - Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID) - Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256) - Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID) - Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct1, multihash.KECCAK_256) - Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID) - Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct2, multihash.KECCAK_256) - Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID) - Rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct3, multihash.KECCAK_256) - Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID) - Rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct4, multihash.KECCAK_256) - Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID) - State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) - State1MhKey = shared.MultihashKeyFromCID(State1CID) - State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) - State2MhKey = shared.MultihashKeyFromCID(State2CID) - StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) - StorageMhKey = shared.MultihashKeyFromCID(StorageCID) - MockTrxMeta = []models.TxModel{ + rctCIDs, rctIPLDData = eth.FetchRctLeafNodeData(MockReceipts) + HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) + HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID) + Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256) + Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID) + Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256) + Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID) + Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256) + Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID) + Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256) + Trx4MhKey = shared.MultihashKeyFromCID(Trx3CID) + Rct1CID = rctCIDs[0] + Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID) + Rct2CID = rctCIDs[1] + Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID) + Rct3CID = rctCIDs[2] + Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID) + Rct4CID = rctCIDs[3] + Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID) + State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) + State1MhKey = shared.MultihashKeyFromCID(State1CID) + State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) + State2MhKey = shared.MultihashKeyFromCID(State2CID) + StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) + StorageMhKey = shared.MultihashKeyFromCID(StorageCID) + Rct1IPLD = rctIPLDData[0] + Rct2IPLD = rctIPLDData[1] + Rct3IPLD = rctIPLDData[2] + Rct4IPLD = rctIPLDData[3] + MockTrxMeta = []models.TxModel{ { CID: "", // This is empty until we go to publish to ipfs MhKey: "", @@ -272,52 +274,53 @@ var ( } MockRctMeta = []models.ReceiptModel{ { - CID: "", - MhKey: "", + LeafCID: "", + LeafMhKey: "", Contract: "", ContractHash: "", }, { - CID: "", - MhKey: "", + LeafCID: "", + LeafMhKey: "", Contract: "", ContractHash: "", }, { - CID: "", - MhKey: "", + LeafCID: "", + LeafMhKey: "", Contract: ContractAddress.String(), ContractHash: ContractHash, }, { - CID: "", - MhKey: "", + LeafCID: "", + LeafMhKey: "", Contract: "", ContractHash: "", }, } + MockRctMetaPostPublish = []models.ReceiptModel{ { - CID: Rct1CID.String(), - MhKey: Rct1MhKey, + LeafCID: Rct1CID.String(), + LeafMhKey: Rct1MhKey, Contract: "", ContractHash: "", }, { - CID: Rct2CID.String(), - MhKey: Rct2MhKey, + LeafCID: Rct2CID.String(), + LeafMhKey: Rct2MhKey, Contract: "", ContractHash: "", }, { - CID: Rct3CID.String(), - MhKey: Rct3MhKey, + LeafCID: Rct3CID.String(), + LeafMhKey: Rct3MhKey, Contract: ContractAddress.String(), ContractHash: ContractHash, }, { - CID: Rct4CID.String(), - MhKey: Rct4MhKey, + LeafCID: Rct4CID.String(), + LeafMhKey: Rct4MhKey, Contract: "", ContractHash: "", }, @@ -478,10 +481,6 @@ var ( Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID) Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID) Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID) - Rct1IPLD, _ = blocks.NewBlockWithCid(Rct1, Rct1CID) - Rct2IPLD, _ = blocks.NewBlockWithCid(Rct2, Rct2CID) - Rct3IPLD, _ = blocks.NewBlockWithCid(Rct3, Rct3CID) - Rct4IPLD, _ = blocks.NewBlockWithCid(Rct4, Rct4CID) State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID) State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID) StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID) @@ -512,20 +511,20 @@ var ( }, Receipts: []ipfs.BlockModel{ { - Data: Rct1IPLD.RawData(), - CID: Rct1IPLD.Cid().String(), + Data: Rct1IPLD, + CID: Rct1CID.String(), }, { - Data: Rct2IPLD.RawData(), - CID: Rct2IPLD.Cid().String(), + Data: Rct2IPLD, + CID: Rct2CID.String(), }, { - Data: Rct3IPLD.RawData(), - CID: Rct3IPLD.Cid().String(), + Data: Rct3IPLD, + CID: Rct3CID.String(), }, { - Data: Rct4IPLD.RawData(), - CID: Rct4IPLD.Cid().String(), + Data: Rct4IPLD, + CID: Rct4CID.String(), }, }, StateNodes: []eth.StateNode{