Dedup receipt #103

Merged
arijitAD merged 6 commits from dedup-receipt into master 2021-09-22 10:22:02 +00:00
16 changed files with 272 additions and 147 deletions

View File

@ -2,8 +2,8 @@
CREATE TABLE eth.receipt_cids ( CREATE TABLE eth.receipt_cids (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
cid TEXT NOT NULL, leaf_cid TEXT NOT NULL,
mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
contract VARCHAR(66), contract VARCHAR(66),
contract_hash VARCHAR(66), contract_hash VARCHAR(66),
post_state VARCHAR(66), post_state VARCHAR(66),

View File

@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-- receipt indexes -- receipt indexes
CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id); CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract); CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
@ -91,8 +91,8 @@ DROP INDEX eth.state_header_id_index;
-- receipt indexes -- receipt indexes
DROP INDEX eth.rct_contract_hash_index; DROP INDEX eth.rct_contract_hash_index;
DROP INDEX eth.rct_contract_index; DROP INDEX eth.rct_contract_index;
DROP INDEX eth.rct_mh_index; DROP INDEX eth.rct_leaf_mh_index;
DROP INDEX eth.rct_cid_index; DROP INDEX eth.rct_leaf_cid_index;
DROP INDEX eth.rct_tx_id_index; DROP INDEX eth.rct_tx_id_index;
-- transaction indexes -- transaction indexes

View File

@ -248,8 +248,8 @@ ALTER SEQUENCE eth.header_cids_id_seq OWNED BY eth.header_cids.id;
CREATE TABLE eth.receipt_cids ( CREATE TABLE eth.receipt_cids (
id integer NOT NULL, id integer NOT NULL,
tx_id integer NOT NULL, tx_id integer NOT NULL,
cid text NOT NULL, leaf_cid text NOT NULL,
mh_key text NOT NULL, leaf_mh_key text NOT NULL,
contract character varying(66), contract character varying(66),
contract_hash character varying(66), contract_hash character varying(66),
topic0s character varying(66)[], topic0s character varying(66)[],
@ -798,10 +798,10 @@ CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
-- --
-- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: - -- Name: rct_leaf_cid_index; Type: INDEX; Schema: eth; Owner: -
-- --
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid); CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
-- --
@ -826,10 +826,10 @@ CREATE INDEX rct_log_contract_index ON eth.receipt_cids USING gin (log_contracts
-- --
-- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: - -- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: -
-- --
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key); CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
-- --
@ -1066,11 +1066,11 @@ ALTER TABLE ONLY eth.header_cids
-- --
-- Name: receipt_cids receipt_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: - -- Name: receipt_cids receipt_leaf_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
-- --
ALTER TABLE ONLY eth.receipt_cids ALTER TABLE ONLY eth.receipt_cids
ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; ADD CONSTRAINT receipt_leaf_cids_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-- --

View File

@ -5,7 +5,7 @@ services:
restart: unless-stopped restart: unless-stopped
depends_on: depends_on:
- statediff-migrations - statediff-migrations
image: vulcanize/dapptools:v0.29.0-v1.10.8-statediff-0.0.26 image: vulcanize/dapptools:v0.29.0-v1.10.8-statediff-0.0.27
environment: environment:
DB_USER: vdbm DB_USER: vdbm
DB_NAME: vulcanize_public DB_NAME: vulcanize_public
@ -20,7 +20,7 @@ services:
restart: on-failure restart: on-failure
depends_on: depends_on:
- db - db
image: vulcanize/statediff-migrations:v0.6.0 image: vulcanize/statediff-migrations:v0.7.0
environment: environment:
DATABASE_USER: vdbm DATABASE_USER: vdbm
DATABASE_NAME: vulcanize_public DATABASE_NAME: vulcanize_public

2
go.mod
View File

@ -29,4 +29,4 @@ require (
github.com/vulcanize/ipfs-ethdb v0.0.4 github.com/vulcanize/ipfs-ethdb v0.0.4
) )
replace github.com/ethereum/go-ethereum v1.10.8 => github.com/vulcanize/go-ethereum v1.10.8-statediff-0.0.26 replace github.com/ethereum/go-ethereum v1.10.8 => github.com/vulcanize/go-ethereum v1.10.8-statediff-0.0.26.0.20210920070539-b1f2fcf09700

4
go.sum
View File

@ -1315,8 +1315,8 @@ github.com/vulcanize/eth-ipfs-state-validator v0.0.1 h1:CsKerIVD7Dq1SvWK93b3LOVY
github.com/vulcanize/eth-ipfs-state-validator v0.0.1/go.mod h1:NxzWqnHkHG4a1gsQJ0MJgkDU2vmy5w4iGOX693HMbIU= github.com/vulcanize/eth-ipfs-state-validator v0.0.1/go.mod h1:NxzWqnHkHG4a1gsQJ0MJgkDU2vmy5w4iGOX693HMbIU=
github.com/vulcanize/gap-filler v0.3.1 h1:N5d+jCJo/VTWFvBSbTD7biRhK/OqDZzi1tgA85SIBKs= github.com/vulcanize/gap-filler v0.3.1 h1:N5d+jCJo/VTWFvBSbTD7biRhK/OqDZzi1tgA85SIBKs=
github.com/vulcanize/gap-filler v0.3.1/go.mod h1:qowG1cgshVpBqMokiWro/1xhh0uypw7oAu8FQ42JMy4= github.com/vulcanize/gap-filler v0.3.1/go.mod h1:qowG1cgshVpBqMokiWro/1xhh0uypw7oAu8FQ42JMy4=
github.com/vulcanize/go-ethereum v1.10.8-statediff-0.0.26 h1:1UBVQpeJnHkmSKxXanbNGE8w+LR0iZhfGr0QrQ62+C4= github.com/vulcanize/go-ethereum v1.10.8-statediff-0.0.26.0.20210920070539-b1f2fcf09700 h1:DfOrj4ePv++iP2RBibMmRmswIhpU1MoJcGFbqsWCRTY=
github.com/vulcanize/go-ethereum v1.10.8-statediff-0.0.26/go.mod h1:nXs5fPBjAVzBmIGtrc0f7akQwkXI5Mi+6I1QcbD2br0= github.com/vulcanize/go-ethereum v1.10.8-statediff-0.0.26.0.20210920070539-b1f2fcf09700/go.mod h1:nXs5fPBjAVzBmIGtrc0f7akQwkXI5Mi+6I1QcbD2br0=
github.com/vulcanize/ipfs-ethdb v0.0.3/go.mod h1:VHyO1HhUYRtxsQ0EpF06VGGP8w4zGjvd+hXL6GK35xw= github.com/vulcanize/ipfs-ethdb v0.0.3/go.mod h1:VHyO1HhUYRtxsQ0EpF06VGGP8w4zGjvd+hXL6GK35xw=
github.com/vulcanize/ipfs-ethdb v0.0.4 h1:rAnkju1NUSBFZgRVUZQOXebA9Kkyq4L8sCzJfeXPjPY= github.com/vulcanize/ipfs-ethdb v0.0.4 h1:rAnkju1NUSBFZgRVUZQOXebA9Kkyq4L8sCzJfeXPjPY=
github.com/vulcanize/ipfs-ethdb v0.0.4/go.mod h1:IueWysMbZu0uFmu+ia6mEnyWsTvwe2q2lbYdy2muRUM= github.com/vulcanize/ipfs-ethdb v0.0.4/go.mod h1:IueWysMbZu0uFmu+ia6mEnyWsTvwe2q2lbYdy2muRUM=

BIN
pkg/.DS_Store vendored Normal file

Binary file not shown.

View File

@ -376,7 +376,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
var receipts []*types.Receipt var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs { for _, rctIPLD := range rctIPLDs {
var receipt types.Receipt var receipt types.Receipt
err = receipt.UnmarshalBinary(rctIPLD.Data) nodeVal, err := DecodeLeafNode(rctIPLD.Data)
if err != nil {
return nil, err
}
err = receipt.UnmarshalBinary(nodeVal)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -476,7 +481,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
var receipts []*types.Receipt var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs { for _, rctIPLD := range rctIPLDs {
var receipt types.Receipt var receipt types.Receipt
err = receipt.UnmarshalBinary(rctIPLD.Data) nodeVal, err := DecodeLeafNode(rctIPLD.Data)
if err != nil {
return nil, err
}
err = receipt.UnmarshalBinary(nodeVal)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -292,7 +292,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte
func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) { func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for header id ", headerID) log.Debug("retrieving receipt cids for header id ", headerID)
args := make([]interface{}, 0, 4) args := make([]interface{}, 0, 4)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key, pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
receipt_cids.contract, receipt_cids.contract_hash receipt_cids.contract, receipt_cids.contract_hash
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id WHERE receipt_cids.tx_id = transaction_cids.id
@ -316,7 +316,7 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF
id := 1 id := 1
pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id, pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id,
eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.cid, eth.receipt_cids.post_status eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status
FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
WHERE eth.log_cids.receipt_id = receipt_cids.id WHERE eth.log_cids.receipt_id = receipt_cids.id
AND receipt_cids.tx_id = transaction_cids.id AND receipt_cids.tx_id = transaction_cids.id
@ -380,7 +380,7 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilte
func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) { func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for block ", blockNumber) log.Debug("retrieving receipt cids for block ", blockNumber)
args := make([]interface{}, 0, 5) args := make([]interface{}, 0, 5)
pgStr := `SELECT receipt_cids.id, receipt_cids.cid, receipt_cids.mh_key, receipt_cids.tx_id pgStr := `SELECT receipt_cids.id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id` AND transaction_cids.header_id = header_cids.id`
@ -595,7 +595,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) (
// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) { func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) {
log.Debugf("retrieving receipt cids for tx ids %v", txIDs) log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key, pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
receipt_cids.contract, receipt_cids.contract_hash receipt_cids.contract, receipt_cids.contract_hash
FROM eth.receipt_cids, eth.transaction_cids FROM eth.receipt_cids, eth.transaction_cids
WHERE tx_id = ANY($1::INTEGER[]) WHERE tx_id = ANY($1::INTEGER[])

View File

@ -239,6 +239,17 @@ var _ = Describe("Retriever", func() {
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
}) })
It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() { It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
type rctCIDAndMHKeyResult struct {
LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"`
}
expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
cids, empty, err := retriever.Retrieve(openFilter, 1) cids, empty, err := retriever.Retrieve(openFilter, 1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue()) Expect(empty).ToNot(BeTrue())
@ -254,9 +265,9 @@ var _ = Describe("Retriever", func() {
Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue()) Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue()) Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
Expect(len(cids[0].Receipts)).To(Equal(4)) Expect(len(cids[0].Receipts)).To(Equal(4))
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue()) Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue()) Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue()) Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
Expect(len(cids[0].StateNodes)).To(Equal(2)) Expect(len(cids[0].StateNodes)).To(Equal(2))
for _, stateNode := range cids[0].StateNodes { for _, stateNode := range cids[0].StateNodes {
@ -279,6 +290,17 @@ var _ = Describe("Retriever", func() {
}) })
It("Applies filters from the provided config.Subscription", func() { It("Applies filters from the provided config.Subscription", func() {
type rctCIDAndMHKeyResult struct {
LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"`
}
expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1) cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue()) Expect(empty).ToNot(BeTrue())
@ -292,6 +314,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0] expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
expectedReceiptCID.ID = cids1[0].Receipts[0].ID expectedReceiptCID.ID = cids1[0].Receipts[0].ID
expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID)) Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1) cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
@ -307,6 +331,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0] expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
expectedReceiptCID.ID = cids2[0].Receipts[0].ID expectedReceiptCID.ID = cids2[0].Receipts[0].ID
expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID)) Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1) cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
@ -322,6 +348,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0] expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
expectedReceiptCID.ID = cids3[0].Receipts[0].ID expectedReceiptCID.ID = cids3[0].Receipts[0].ID
expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID)) Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1) cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
@ -337,6 +365,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1] expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
expectedReceiptCID.ID = cids4[0].Receipts[0].ID expectedReceiptCID.ID = cids4[0].Receipts[0].ID
expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID)) Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1) cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
@ -352,9 +382,9 @@ var _ = Describe("Retriever", func() {
Expect(len(cids5[0].StateNodes)).To(Equal(0)) Expect(len(cids5[0].StateNodes)).To(Equal(0))
Expect(len(cids5[0].StorageNodes)).To(Equal(0)) Expect(len(cids5[0].StorageNodes)).To(Equal(0))
Expect(len(cids5[0].Receipts)).To(Equal(4)) Expect(len(cids5[0].Receipts)).To(Equal(4))
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue()) Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue()) Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue()) Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1) cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -373,6 +403,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1] expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
expectedReceiptCID.ID = cids6[0].Receipts[0].ID expectedReceiptCID.ID = cids6[0].Receipts[0].ID
expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID)) Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids7, empty, err := retriever.Retrieve(stateFilter, 1) cids7, empty, err := retriever.Retrieve(stateFilter, 1)

View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash" "github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
@ -164,7 +165,12 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error { func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
if !receiptFilter.Off { if !receiptFilter.Off {
response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts)) response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
for _, receipt := range payload.Receipts { rctLeafCID, rctIPLDData, err := GetRctLeafNodeData(payload.Receipts)
if err != nil {
return err
}
for idx, receipt := range payload.Receipts {
// topics is always length 4 // topics is always length 4
topics := make([][]string, 4) topics := make([][]string, 4)
contracts := make([]string, len(receipt.Logs)) contracts := make([]string, len(receipt.Logs))
@ -177,18 +183,9 @@ func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *
// TODO: Verify this filter logic. // TODO: Verify this filter logic.
if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) { if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) {
receiptBuffer := new(bytes.Buffer)
if err := receipt.EncodeRLP(receiptBuffer); err != nil {
return err
}
data := receiptBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
if err != nil {
return err
}
response.Receipts = append(response.Receipts, ipfs.BlockModel{ response.Receipts = append(response.Receipts, ipfs.BlockModel{
Data: data, Data: rctIPLDData[idx],
CID: cid.String(), CID: rctLeafCID[idx].String(),
}) })
} }
} }
@ -329,3 +326,40 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool {
} }
return false return false
} }
// GetRctLeafNodeData converts the receipts to receipt trie and returns the receipt leaf node IPLD data and
// corresponding CIDs
func GetRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte, error) {
receiptTrie := ipld.NewRctTrie()
for idx, rct := range rcts {
ethRct, err := ipld.NewReceipt(rct)
if err != nil {
return nil, nil, err
}
if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
return nil, nil, err
}
}
rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
if err != nil {
return nil, nil, err
}
ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
ethRctleafNodeData := make([][]byte, len(rctLeafNodes))
for i, rln := range rctLeafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
err = rlp.Decode(r, &idx)
if err != nil {
return nil, nil, err
}
ethRctleafNodeCids[idx] = rln.Cid()
ethRctleafNodeData[idx] = rln.RawData()
}
return ethRctleafNodeCids, ethRctleafNodeData, nil
}

View File

@ -53,9 +53,9 @@ var _ = Describe("Filterer", func() {
Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue())
Expect(len(iplds.Receipts)).To(Equal(4)) Expect(len(iplds.Receipts)).To(Equal(4))
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
Expect(len(iplds.StateNodes)).To(Equal(2)) Expect(len(iplds.StateNodes)).To(Equal(2))
for _, stateNode := range iplds.StateNodes { for _, stateNode := range iplds.StateNodes {
Expect(stateNode.Type).To(Equal(sdtypes.Leaf)) Expect(stateNode.Type).To(Equal(sdtypes.Leaf))
@ -87,8 +87,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds1.StateNodes)).To(Equal(0)) Expect(len(iplds1.StateNodes)).To(Equal(0))
Expect(len(iplds1.Receipts)).To(Equal(1)) Expect(len(iplds1.Receipts)).To(Equal(1))
Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{ Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct1IPLD.RawData(), Data: test_helpers.Rct1IPLD,
CID: test_helpers.Rct1IPLD.Cid().String(), CID: test_helpers.Rct1CID.String(),
})) }))
iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload) iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
@ -102,8 +102,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds2.StateNodes)).To(Equal(0)) Expect(len(iplds2.StateNodes)).To(Equal(0))
Expect(len(iplds2.Receipts)).To(Equal(1)) Expect(len(iplds2.Receipts)).To(Equal(1))
Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{ Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct1IPLD.RawData(), Data: test_helpers.Rct1IPLD,
CID: test_helpers.Rct1IPLD.Cid().String(), CID: test_helpers.Rct1CID.String(),
})) }))
iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload) iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
@ -117,8 +117,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds3.StateNodes)).To(Equal(0)) Expect(len(iplds3.StateNodes)).To(Equal(0))
Expect(len(iplds3.Receipts)).To(Equal(1)) Expect(len(iplds3.Receipts)).To(Equal(1))
Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{ Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct1IPLD.RawData(), Data: test_helpers.Rct1IPLD,
CID: test_helpers.Rct1IPLD.Cid().String(), CID: test_helpers.Rct1CID.String(),
})) }))
iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload) iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
@ -132,8 +132,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds4.StateNodes)).To(Equal(0)) Expect(len(iplds4.StateNodes)).To(Equal(0))
Expect(len(iplds4.Receipts)).To(Equal(1)) Expect(len(iplds4.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{ Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct2IPLD.RawData(), Data: test_helpers.Rct2IPLD,
CID: test_helpers.Rct2IPLD.Cid().String(), CID: test_helpers.Rct2CID.String(),
})) }))
iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload) iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
@ -149,9 +149,9 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds5.StorageNodes)).To(Equal(0)) Expect(len(iplds5.StorageNodes)).To(Equal(0))
Expect(len(iplds5.StateNodes)).To(Equal(0)) Expect(len(iplds5.StateNodes)).To(Equal(0))
Expect(len(iplds5.Receipts)).To(Equal(4)) Expect(len(iplds5.Receipts)).To(Equal(4))
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3)).To(BeTrue()) Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload) iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
@ -165,8 +165,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds6.StateNodes)).To(Equal(0)) Expect(len(iplds6.StateNodes)).To(Equal(0))
Expect(len(iplds6.Receipts)).To(Equal(1)) Expect(len(iplds6.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{ Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct2IPLD.RawData(), Data: test_helpers.Rct2IPLD,
CID: test_helpers.Rct2IPLD.Cid().String(), CID: test_helpers.Rct2CID.String(),
})) }))
iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload) iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)

View File

@ -153,13 +153,14 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs
log.Debug("fetching receipt iplds") log.Debug("fetching receipt iplds")
rctIPLDs := make([]ipfs.BlockModel, len(cids)) rctIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids { for i, c := range cids {
rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
//nodeVal, err := DecodeLeafNode(rctBytes)
rctIPLDs[i] = ipfs.BlockModel{ rctIPLDs[i] = ipfs.BlockModel{
Data: rctBytes, Data: rctBytes,
CID: c.CID, CID: c.LeafCID,
} }
} }
return rctIPLDs, nil return rctIPLDs, nil

View File

@ -19,6 +19,9 @@ package eth
import ( import (
"fmt" "fmt"
"github.com/ethereum/go-ethereum/statediff/trie"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
@ -81,29 +84,29 @@ const (
FROM eth.transaction_cids FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE tx_hash = $1` WHERE tx_hash = $1`
RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data
FROM eth.receipt_cids FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE tx_hash = ANY($1::VARCHAR(66)[])` WHERE tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
FROM eth.receipt_cids FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE block_hash = $1 WHERE block_hash = $1
ORDER BY eth.transaction_cids.index ASC` ORDER BY eth.transaction_cids.index ASC`
RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data
FROM eth.receipt_cids FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE block_number = $1 WHERE block_number = $1
ORDER BY eth.transaction_cids.index ASC` ORDER BY eth.transaction_cids.index ASC`
RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data
FROM eth.receipt_cids FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id) INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key) INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE tx_hash = $1` WHERE tx_hash = $1`
RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type
FROM eth.state_cids FROM eth.state_cids
@ -149,6 +152,12 @@ const (
LIMIT 1` LIMIT 1`
) )
type rctIpldResult struct {
LeafCID string `db:"leaf_cid"`
Data []byte `db:"data"`
TxHash string `db:"tx_hash"`
}
type ipldResult struct { type ipldResult struct {
CID string `db:"cid"` CID string `db:"cid"`
Data []byte `db:"data"` Data []byte `db:"data"`
@ -316,9 +325,26 @@ func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, [
return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex()) return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
} }
// DecodeLeafNode decodes the leaf node data
func DecodeLeafNode(node []byte) ([]byte, error) {
var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
return nil, err
}
ty, err := trie.CheckKeyType(nodeElements)
if err != nil {
return nil, err
}
if ty != sdtypes.Leaf {
return nil, fmt.Errorf("expected leaf node but found %s", ty)
}
return nodeElements[1].([]byte), nil
}
// RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) { func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
rctResults := make([]ipldResult, 0) rctResults := make([]rctIpldResult, 0)
hashStrs := make([]string, len(hashes)) hashStrs := make([]string, len(hashes))
for i, hash := range hashes { for i, hash := range hashes {
hashStrs[i] = hash.Hex() hashStrs[i] = hash.Hex()
@ -329,15 +355,20 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
cids := make([]string, len(rctResults)) cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults)) rcts := make([][]byte, len(rctResults))
for i, res := range rctResults { for i, res := range rctResults {
cids[i] = res.CID cids[i] = res.LeafCID
rcts[i] = res.Data nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, err
}
rcts[i] = nodeVal
} }
return cids, rcts, nil return cids, rcts, nil
} }
// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash // RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) { func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
rctResults := make([]ipldResult, 0) rctResults := make([]rctIpldResult, 0)
if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil { if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
@ -346,33 +377,51 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
txs := make([]common.Hash, len(rctResults)) txs := make([]common.Hash, len(rctResults))
for i, res := range rctResults { for i, res := range rctResults {
cids[i] = res.CID cids[i] = res.LeafCID
rcts[i] = res.Data nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, nil, err
}
rcts[i] = nodeVal
txs[i] = common.HexToHash(res.TxHash) txs[i] = common.HexToHash(res.TxHash)
Review

Lets remove this comment

Lets remove this comment
arijitAD commented 2021-09-20 10:09:53 +00:00 (Migrated from github.com)
Review

Done

Done
} }
return cids, rcts, txs, nil return cids, rcts, txs, nil
} }
// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash // RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) { func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
rctResults := make([]ipldResult, 0) rctResults := make([]rctIpldResult, 0)
if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil { if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
return nil, nil, err return nil, nil, err
} }
cids := make([]string, len(rctResults)) cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults)) rcts := make([][]byte, len(rctResults))
for i, res := range rctResults { for i, res := range rctResults {
Review

Ditto

Ditto
Review

We may want to note that the CIDs we are returning is not for the Receipts themselves but rather the leaf nodes they are contained within

We may want to note that the CIDs we are returning is not for the Receipts themselves but rather the leaf nodes they are contained within
Review

We may want to note that the CID we are returning is not for the Receipt itself but rather the leaf node it was contained within

We may want to note that the CID we are returning is not for the Receipt itself but rather the leaf node it was contained within
arijitAD commented 2021-09-20 10:09:58 +00:00 (Migrated from github.com)
Review

Done

Done
arijitAD commented 2021-09-20 10:15:23 +00:00 (Migrated from github.com)
Review

Done

Done
arijitAD commented 2021-09-20 10:21:04 +00:00 (Migrated from github.com)
Review

Done

Done
cids[i] = res.CID cids[i] = res.LeafCID
rcts[i] = res.Data nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, err
}
rcts[i] = nodeVal
} }
return cids, rcts, nil return cids, rcts, nil
} }
// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash // RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) { func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
rctResult := new(ipldResult) rctResult := new(rctIpldResult)
return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()) if err := r.db.Select(&rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()); err != nil {
return "", nil, err
}
nodeVal, err := DecodeLeafNode(rctResult.Data)
if err != nil {
return "", nil, err
}
return rctResult.LeafCID, nodeVal, nil
} }
type nodeInfo struct { type nodeInfo struct {

View File

@ -56,10 +56,10 @@ func TxModelsContainsCID(txs []models.TxModel, cid string) bool {
return false return false
} }
Review

Not introduced here but I notice this comment is outdated, let's align it with the function

Not introduced here but I notice this comment is outdated, let's align it with the function
arijitAD commented 2021-09-20 10:11:07 +00:00 (Migrated from github.com)
Review

Done

Done
// ListContainsBytes used to check if a list of byte arrays contains a particular byte array // ReceiptModelsContainsCID used to check if a list of ReceiptModel contains a specific cid string
func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool { func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
for _, rct := range rcts { for _, rct := range rcts {
if rct.CID == cid { if rct.LeafCID == cid {
return true return true
} }
} }

View File

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/statediff/testhelpers" "github.com/ethereum/go-ethereum/statediff/testhelpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
@ -47,6 +48,7 @@ import (
// Test variables // Test variables
var ( var (
// block data // block data
db *postgres.DB
BlockNumber = big.NewInt(1) BlockNumber = big.NewInt(1)
MockHeader = types.Header{ MockHeader = types.Header{
Time: 0, Time: 0,
@ -165,36 +167,36 @@ var (
Tx3 = GetTxnRlp(2, MockTransactions) Tx3 = GetTxnRlp(2, MockTransactions)
Tx4 = GetTxnRlp(3, MockTransactions) Tx4 = GetTxnRlp(3, MockTransactions)
Rct1 = GetRctRlp(0, MockReceipts) rctCIDs, rctIPLDData, _ = eth.GetRctLeafNodeData(MockReceipts)
Rct2 = GetRctRlp(1, MockReceipts) HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
Rct3 = GetRctRlp(2, MockReceipts) HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID)
Rct4 = GetRctRlp(3, MockReceipts) Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID)
HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID) Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID)
Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256) Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID) Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID)
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256) Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID) Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID)
Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256) Rct1CID = rctCIDs[0]
Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID) Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID)
Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256) Rct2CID = rctCIDs[1]
Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID) Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID)
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct1, multihash.KECCAK_256) Rct3CID = rctCIDs[2]
Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID) Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID)
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct2, multihash.KECCAK_256) Rct4CID = rctCIDs[3]
Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID) Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID)
Rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct3, multihash.KECCAK_256) State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID) State1MhKey = shared.MultihashKeyFromCID(State1CID)
Rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct4, multihash.KECCAK_256) State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID) State2MhKey = shared.MultihashKeyFromCID(State2CID)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
State1MhKey = shared.MultihashKeyFromCID(State1CID) StorageMhKey = shared.MultihashKeyFromCID(StorageCID)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) Rct1IPLD = rctIPLDData[0]
State2MhKey = shared.MultihashKeyFromCID(State2CID) Rct2IPLD = rctIPLDData[1]
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) Rct3IPLD = rctIPLDData[2]
StorageMhKey = shared.MultihashKeyFromCID(StorageCID) Rct4IPLD = rctIPLDData[3]
MockTrxMeta = []models.TxModel{ MockTrxMeta = []models.TxModel{
{ {
CID: "", // This is empty until we go to publish to ipfs CID: "", // This is empty until we go to publish to ipfs
MhKey: "", MhKey: "",
@ -264,60 +266,61 @@ var (
CID: Trx4CID.String(), CID: Trx4CID.String(),
MhKey: Trx4MhKey, MhKey: Trx4MhKey,
Src: SenderAddr.Hex(), Src: SenderAddr.Hex(),
Dst: "", Dst: AnotherAddress1.String(),
Index: 2, Index: 3,
TxHash: MockTransactions[2].Hash().String(), TxHash: MockTransactions[3].Hash().String(),
Data: MockContractByteCode, Data: []byte{},
}, },
} }
MockRctMeta = []models.ReceiptModel{ MockRctMeta = []models.ReceiptModel{
{ {
CID: "", LeafCID: "",
MhKey: "", LeafMhKey: "",
Contract: "", Contract: "",
ContractHash: "", ContractHash: "",
}, },
{ {
CID: "", LeafCID: "",
MhKey: "", LeafMhKey: "",
Contract: "", Contract: "",
ContractHash: "", ContractHash: "",
}, },
{ {
CID: "", LeafCID: "",
MhKey: "", LeafMhKey: "",
Contract: ContractAddress.String(), Contract: ContractAddress.String(),
ContractHash: ContractHash, ContractHash: ContractHash,
}, },
{ {
CID: "", LeafCID: "",
MhKey: "", LeafMhKey: "",
Contract: "", Contract: "",
ContractHash: "", ContractHash: "",
}, },
} }
MockRctMetaPostPublish = []models.ReceiptModel{ MockRctMetaPostPublish = []models.ReceiptModel{
{ {
CID: Rct1CID.String(), LeafCID: Rct1CID.String(),
MhKey: Rct1MhKey, LeafMhKey: Rct1MhKey,
Contract: "", Contract: "",
ContractHash: "", ContractHash: "",
}, },
{ {
CID: Rct2CID.String(), LeafCID: Rct2CID.String(),
MhKey: Rct2MhKey, LeafMhKey: Rct2MhKey,
Contract: "", Contract: "",
ContractHash: "", ContractHash: "",
}, },
{ {
CID: Rct3CID.String(), LeafCID: Rct3CID.String(),
MhKey: Rct3MhKey, LeafMhKey: Rct3MhKey,
Contract: ContractAddress.String(), Contract: ContractAddress.String(),
ContractHash: ContractHash, ContractHash: ContractHash,
}, },
{ {
CID: Rct4CID.String(), LeafCID: Rct4CID.String(),
MhKey: Rct4MhKey, LeafMhKey: Rct4MhKey,
Contract: "", Contract: "",
ContractHash: "", ContractHash: "",
}, },
@ -478,10 +481,6 @@ var (
Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID) Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID)
Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID) Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID)
Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID) Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(Rct1, Rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(Rct2, Rct2CID)
Rct3IPLD, _ = blocks.NewBlockWithCid(Rct3, Rct3CID)
Rct4IPLD, _ = blocks.NewBlockWithCid(Rct4, Rct4CID)
State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID) State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID) State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID) StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
@ -512,20 +511,20 @@ var (
}, },
Receipts: []ipfs.BlockModel{ Receipts: []ipfs.BlockModel{
{ {
Data: Rct1IPLD.RawData(), Data: Rct1IPLD,
CID: Rct1IPLD.Cid().String(), CID: Rct1CID.String(),
}, },
{ {
Data: Rct2IPLD.RawData(), Data: Rct2IPLD,
CID: Rct2IPLD.Cid().String(), CID: Rct2CID.String(),
}, },
{ {
Data: Rct3IPLD.RawData(), Data: Rct3IPLD,
CID: Rct3IPLD.Cid().String(), CID: Rct3CID.String(),
}, },
{ {
Data: Rct4IPLD.RawData(), Data: Rct4IPLD,
CID: Rct4IPLD.Cid().String(), CID: Rct4CID.String(),
}, },
}, },
StateNodes: []eth.StateNode{ StateNodes: []eth.StateNode{