Change receipt CID and MHKey to point trie leaf cid and mkhey.

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -2,8 +2,8 @@
 CREATE TABLE eth.receipt_cids (
   id                    SERIAL PRIMARY KEY,
   tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  cid                   TEXT NOT NULL,
+  leaf_cid              TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  leaf_mh_key           TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
   contract              VARCHAR(66),
   contract_hash         VARCHAR(66),
   post_state            VARCHAR(66),

db/migrations/00013_create_cid_indexes.sql

@@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
 -- receipt indexes
 CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
 
-CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
+CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
 
-CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
+CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
 
 CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
 

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     restart: unless-stopped
     depends_on:
       - statediff-migrations
-    image: vulcanize/dapptools:v0.29.0-v1.10.8-statediff-0.0.26
+    image: vulcanize/dapptools:latest
     environment:
       DB_USER: vdbm
       DB_NAME: vulcanize_public
@@ -20,7 +20,7 @@ services:
     restart: on-failure
     depends_on:
       - db
-    image: vulcanize/statediff-migrations:v0.6.0
+    image: vulcanize/statediff-migrations:latest
     environment:
       DATABASE_USER: vdbm
       DATABASE_NAME: vulcanize_public

pkg/eth/backend.go

@@ -376,7 +376,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		err = receipt.UnmarshalBinary(rctIPLD.Data)
+		nodeVal, err := DecodeLeafNode(rctIPLD.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		err = receipt.UnmarshalBinary(nodeVal)
 		if err != nil {
 			return nil, err
 		}
@@ -476,7 +481,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		err = receipt.UnmarshalBinary(rctIPLD.Data)
+		nodeVal, err := DecodeLeafNode(rctIPLD.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		err = receipt.UnmarshalBinary(nodeVal)
 		if err != nil {
 			return nil, err
 		}

pkg/eth/cid_retriever.go

@@ -292,7 +292,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte
 func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) {
 	log.Debug("retrieving receipt cids for header id ", headerID)
 	args := make([]interface{}, 0, 4)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
  			receipt_cids.contract, receipt_cids.contract_hash
  			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
 			WHERE receipt_cids.tx_id = transaction_cids.id 
@@ -316,12 +316,12 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF
 	id := 1
 	pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id,  
        			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, 
-       			eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.cid, eth.receipt_cids.post_status
+       			eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid, eth.receipt_cids.post_status
 				FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
 				WHERE eth.log_cids.receipt_id = receipt_cids.id
 				AND receipt_cids.tx_id = transaction_cids.id
  				AND transaction_cids.header_id = header_cids.id
- 				AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1`
+ 				AND receipt_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1`
 
 	args = append(args, blockHash.String())
 	id++
@@ -380,7 +380,7 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilte
 func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) {
 	log.Debug("retrieving receipt cids for block ", blockNumber)
 	args := make([]interface{}, 0, 5)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.cid, receipt_cids.mh_key, receipt_cids.tx_id
+	pgStr := `SELECT receipt_cids.id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id
  			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
 			WHERE receipt_cids.tx_id = transaction_cids.id 
 			AND transaction_cids.header_id = header_cids.id`
@@ -595,7 +595,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) (
 // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
 func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) {
 	log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
  			receipt_cids.contract, receipt_cids.contract_hash
 			FROM eth.receipt_cids, eth.transaction_cids
 			WHERE tx_id = ANY($1::INTEGER[])

pkg/eth/cid_retriever_test.go

@@ -239,6 +239,17 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
+			type rctCIDAndMHKeyResult struct {
+				LeafCID   string `db:"leaf_cid"`
+				LeafMhKey string `db:"leaf_mh_key"`
+			}
+			expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
+			pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.id
+				AND transaction_cids.header_id = header_cids.id
+				AND header_cids.block_number = $1
+				ORDER BY transaction_cids.index`
+			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
 			cids, empty, err := retriever.Retrieve(openFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
@@ -253,10 +264,10 @@ var _ = Describe("Retriever", func() {
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
-			Expect(len(cids[0].Receipts)).To(Equal(4))
+			Expect(len(cids[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
 			Expect(len(cids[0].StateNodes)).To(Equal(2))
 
 			for _, stateNode := range cids[0].StateNodes {
@@ -279,6 +290,17 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
+			type rctCIDAndMHKeyResult struct {
+				LeafCID   string `db:"leaf_cid"`
+				LeafMhKey string `db:"leaf_mh_key"`
+			}
+			expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
+			pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.id
+				AND transaction_cids.header_id = header_cids.id
+				AND header_cids.block_number = $1
+				ORDER BY transaction_cids.index`
+			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
 			cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
@@ -292,6 +314,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids1[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
@@ -307,6 +331,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids2[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
@@ -322,6 +348,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids3[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
@@ -337,6 +365,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
 			expectedReceiptCID.ID = cids4[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
 			Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
@@ -351,10 +381,10 @@ var _ = Describe("Retriever", func() {
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue())
 			Expect(len(cids5[0].StateNodes)).To(Equal(0))
 			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids5[0].Receipts)).To(Equal(4))
+			Expect(len(cids5[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
 
 			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
 			Expect(err).ToNot(HaveOccurred())
@@ -373,6 +403,8 @@ var _ = Describe("Retriever", func() {
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
 			expectedReceiptCID.ID = cids6[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
 			Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids7, empty, err := retriever.Retrieve(stateFilter, 1)

pkg/eth/filterer.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
@@ -164,7 +165,8 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
 func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
 		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
-		for _, receipt := range payload.Receipts {
+		rctLeafCID, rctIPLDData := FetchRctLeafNodeData(payload.Receipts)
+		for idx, receipt := range payload.Receipts {
 			// topics is always length 4
 			topics := make([][]string, 4)
 			contracts := make([]string, len(receipt.Logs))
@@ -177,18 +179,9 @@ func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *
 
 			// TODO: Verify this filter logic.
 			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) {
-				receiptBuffer := new(bytes.Buffer)
-				if err := receipt.EncodeRLP(receiptBuffer); err != nil {
-					return err
-				}
-				data := receiptBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
 				response.Receipts = append(response.Receipts, ipfs.BlockModel{
-					Data: data,
+					Data: rctIPLDData[idx],
-					CID:  cid.String(),
+					CID:  rctLeafCID[idx].String(),
 				})
 			}
 		}
@@ -329,3 +322,44 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool {
 	}
 	return false
 }
+
+// FetchRctLeafNodeData fetches receipt leaf node IPLD data and CIDs
+func FetchRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte) {
+	receiptTrie := ipld.NewRctTrie()
+	for idx, rct := range rcts {
+		ethRct, err := ipld.NewReceipt(rct)
+		if err != nil {
+			return nil, nil
+		}
+		if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
+			return nil, nil
+		}
+	}
+
+	_, err := receiptTrie.GetNodes()
+	if err != nil {
+		return nil, nil
+	}
+
+	rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
+	if err != nil {
+		return nil, nil
+	}
+
+	ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
+	ethRctleafNodeData := make([][]byte, len(rctLeafNodes))
+	for i, rln := range rctLeafNodes {
+		var idx uint
+
+		r := bytes.NewReader(keys[i].TrieKey)
+		err = rlp.Decode(r, &idx)
+		if err != nil {
+			return nil, nil
+		}
+
+		ethRctleafNodeCids[idx] = rln.Cid()
+		ethRctleafNodeData[idx] = rln.RawData()
+	}
+
+	return ethRctleafNodeCids, ethRctleafNodeData
+}

pkg/eth/filterer_test.go

@@ -52,10 +52,10 @@ var _ = Describe("Filterer", func() {
 			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx1)).To(BeTrue())
 			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue())
 			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue())
-			Expect(len(iplds.Receipts)).To(Equal(4))
+			Expect(len(iplds.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
 			Expect(len(iplds.StateNodes)).To(Equal(2))
 			for _, stateNode := range iplds.StateNodes {
 				Expect(stateNode.Type).To(Equal(sdtypes.Leaf))
@@ -87,8 +87,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds1.StateNodes)).To(Equal(0))
 			Expect(len(iplds1.Receipts)).To(Equal(1))
 			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				CID:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
@@ -102,8 +102,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds2.StateNodes)).To(Equal(0))
 			Expect(len(iplds2.Receipts)).To(Equal(1))
 			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				CID:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
@@ -117,8 +117,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds3.StateNodes)).To(Equal(0))
 			Expect(len(iplds3.Receipts)).To(Equal(1))
 			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				CID:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
@@ -132,8 +132,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds4.StateNodes)).To(Equal(0))
 			Expect(len(iplds4.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
+				Data: test_helpers.Rct2IPLD,
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
+				CID:  test_helpers.Rct2CID.String(),
 			}))
 
 			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
@@ -148,10 +148,10 @@ var _ = Describe("Filterer", func() {
 			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx3)).To(BeTrue())
 			Expect(len(iplds5.StorageNodes)).To(Equal(0))
 			Expect(len(iplds5.StateNodes)).To(Equal(0))
-			Expect(len(iplds5.Receipts)).To(Equal(4))
+			Expect(len(iplds5.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3)).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
 
 			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
@@ -165,8 +165,8 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds6.StateNodes)).To(Equal(0))
 			Expect(len(iplds6.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
+				Data: test_helpers.Rct2IPLD,
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
+				CID:  test_helpers.Rct2CID.String(),
 			}))
 
 			iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)

pkg/eth/ipld_fetcher.go

@@ -153,13 +153,14 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs
 	log.Debug("fetching receipt iplds")
 	rctIPLDs := make([]ipfs.BlockModel, len(cids))
 	for i, c := range cids {
-		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
+		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey)
 		if err != nil {
 			return nil, err
 		}
+		//nodeVal, err := DecodeLeafNode(rctBytes)
 		rctIPLDs[i] = ipfs.BlockModel{
 			Data: rctBytes,
-			CID:  c.CID,
+			CID:  c.LeafCID,
 		}
 	}
 	return rctIPLDs, nil

pkg/eth/ipld_retriever.go

@@ -19,6 +19,9 @@ package eth
 import (
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/statediff/trie"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -81,29 +84,29 @@ const (
 									FROM eth.transaction_cids
 										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
 									WHERE tx_hash = $1`
-	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data
+	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data
 									FROM eth.receipt_cids
 										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+										INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash
+	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
 										FROM eth.receipt_cids
 											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
 											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+											INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 										WHERE block_hash = $1
 										ORDER BY eth.transaction_cids.index ASC`
-	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data
+	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data
 										FROM eth.receipt_cids
 											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
 											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+											INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 										WHERE block_number = $1
 										ORDER BY eth.transaction_cids.index ASC`
-	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data
+	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data
 									FROM eth.receipt_cids
 										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+										INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
 									WHERE tx_hash = $1`
 	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type
 												FROM eth.state_cids
@@ -149,6 +152,12 @@ const (
 																	LIMIT 1`
 )
 
+type rctIpldResult struct {
+	LeafCID string `db:"leaf_cid"`
+	Data    []byte `db:"data"`
+	TxHash  string `db:"tx_hash"`
+}
+
 type ipldResult struct {
 	CID    string `db:"cid"`
 	Data   []byte `db:"data"`
@@ -316,9 +325,26 @@ func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, [
 	return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
 }
 
+// DecodeLeafNode decodes the leaf node data
+func DecodeLeafNode(node []byte) ([]byte, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return nil, err
+	}
+	ty, err := trie.CheckKeyType(nodeElements)
+	if err != nil {
+		return nil, err
+	}
+
+	if ty != sdtypes.Leaf {
+		return nil, fmt.Errorf("expected leaf node but found %s", ty)
+	}
+	return nodeElements[1].([]byte), nil
+}
+
 // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
 func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	hashStrs := make([]string, len(hashes))
 	for i, hash := range hashes {
 		hashStrs[i] = hash.Hex()
@@ -329,15 +355,20 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.CID
+		cids[i] = res.LeafCID
-		rcts[i] = res.Data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, err
+		}
+		rcts[i] = nodeVal
+		//rcts[i] = res.Data
 	}
 	return cids, rcts, nil
 }
 
 // RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash
 func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
 		return nil, nil, nil, err
 	}
@@ -346,8 +377,13 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
 	txs := make([]common.Hash, len(rctResults))
 
 	for i, res := range rctResults {
-		cids[i] = res.CID
+		cids[i] = res.LeafCID
-		rcts[i] = res.Data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		rcts[i] = nodeVal
+		//rcts[i] = res.Data
 		txs[i] = common.HexToHash(res.TxHash)
 	}
 
@@ -356,23 +392,28 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
 
 // RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash
 func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
 		return nil, nil, err
 	}
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.CID
+		cids[i] = res.LeafCID
-		rcts[i] = res.Data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, err
+		}
+		rcts[i] = nodeVal
+		//rcts[i] = res.Data
 	}
 	return cids, rcts, nil
 }
 
 // RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
 func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
-	rctResult := new(ipldResult)
+	rctResult := new(rctIpldResult)
-	return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
+	return rctResult.LeafCID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
 }
 
 type nodeInfo struct {

pkg/eth/test_helpers.go

@@ -59,7 +59,7 @@ func TxModelsContainsCID(txs []models.TxModel, cid string) bool {
 // ListContainsBytes used to check if a list of byte arrays contains a particular byte array
 func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
 	for _, rct := range rcts {
-		if rct.CID == cid {
+		if rct.LeafCID == cid {
 			return true
 		}
 	}

pkg/eth/test_helpers/test_data.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
 	"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
 	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/statediff/testhelpers"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
@@ -47,6 +48,7 @@ import (
 // Test variables
 var (
 	// block data
+	db          *postgres.DB
 	BlockNumber = big.NewInt(1)
 	MockHeader  = types.Header{
 		Time:        0,
@@ -165,36 +167,36 @@ var (
 	Tx3 = GetTxnRlp(2, MockTransactions)
 	Tx4 = GetTxnRlp(3, MockTransactions)
 
-	Rct1 = GetRctRlp(0, MockReceipts)
+	rctCIDs, rctIPLDData = eth.FetchRctLeafNodeData(MockReceipts)
-	Rct2 = GetRctRlp(1, MockReceipts)
+	HeaderCID, _         = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
-	Rct3 = GetRctRlp(2, MockReceipts)
+	HeaderMhKey          = shared.MultihashKeyFromCID(HeaderCID)
-	Rct4 = GetRctRlp(3, MockReceipts)
+	Trx1CID, _           = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
-
+	Trx1MhKey            = shared.MultihashKeyFromCID(Trx1CID)
-	HeaderCID, _  = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
+	Trx2CID, _           = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
-	HeaderMhKey   = shared.MultihashKeyFromCID(HeaderCID)
+	Trx2MhKey            = shared.MultihashKeyFromCID(Trx2CID)
-	Trx1CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
+	Trx3CID, _           = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
-	Trx1MhKey     = shared.MultihashKeyFromCID(Trx1CID)
+	Trx3MhKey            = shared.MultihashKeyFromCID(Trx3CID)
-	Trx2CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
+	Trx4CID, _           = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
-	Trx2MhKey     = shared.MultihashKeyFromCID(Trx2CID)
+	Trx4MhKey            = shared.MultihashKeyFromCID(Trx3CID)
-	Trx3CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
+	Rct1CID              = rctCIDs[0]
-	Trx3MhKey     = shared.MultihashKeyFromCID(Trx3CID)
+	Rct1MhKey            = shared.MultihashKeyFromCID(Rct1CID)
-	Trx4CID, _    = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
+	Rct2CID              = rctCIDs[1]
-	Trx4MhKey     = shared.MultihashKeyFromCID(Trx4CID)
+	Rct2MhKey            = shared.MultihashKeyFromCID(Rct2CID)
-	Rct1CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct1, multihash.KECCAK_256)
+	Rct3CID              = rctCIDs[2]
-	Rct1MhKey     = shared.MultihashKeyFromCID(Rct1CID)
+	Rct3MhKey            = shared.MultihashKeyFromCID(Rct3CID)
-	Rct2CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct2, multihash.KECCAK_256)
+	Rct4CID              = rctCIDs[3]
-	Rct2MhKey     = shared.MultihashKeyFromCID(Rct2CID)
+	Rct4MhKey            = shared.MultihashKeyFromCID(Rct4CID)
-	Rct3CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct3, multihash.KECCAK_256)
+	State1CID, _         = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
-	Rct3MhKey     = shared.MultihashKeyFromCID(Rct3CID)
+	State1MhKey          = shared.MultihashKeyFromCID(State1CID)
-	Rct4CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct4, multihash.KECCAK_256)
+	State2CID, _         = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
-	Rct4MhKey     = shared.MultihashKeyFromCID(Rct4CID)
+	State2MhKey          = shared.MultihashKeyFromCID(State2CID)
-	State1CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
+	StorageCID, _        = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
-	State1MhKey   = shared.MultihashKeyFromCID(State1CID)
+	StorageMhKey         = shared.MultihashKeyFromCID(StorageCID)
-	State2CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
+	Rct1IPLD             = rctIPLDData[0]
-	State2MhKey   = shared.MultihashKeyFromCID(State2CID)
+	Rct2IPLD             = rctIPLDData[1]
-	StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
+	Rct3IPLD             = rctIPLDData[2]
-	StorageMhKey  = shared.MultihashKeyFromCID(StorageCID)
+	Rct4IPLD             = rctIPLDData[3]
-	MockTrxMeta   = []models.TxModel{
+	MockTrxMeta          = []models.TxModel{
 		{
 			CID:    "", // This is empty until we go to publish to ipfs
 			MhKey:  "",
@@ -272,52 +274,53 @@ var (
 	}
 	MockRctMeta = []models.ReceiptModel{
 		{
-			CID:          "",
+			LeafCID:      "",
-			MhKey:        "",
+			LeafMhKey:    "",
 			Contract:     "",
 			ContractHash: "",
 		},
 		{
-			CID:          "",
+			LeafCID:      "",
-			MhKey:        "",
+			LeafMhKey:    "",
 			Contract:     "",
 			ContractHash: "",
 		},
 		{
-			CID:          "",
+			LeafCID:      "",
-			MhKey:        "",
+			LeafMhKey:    "",
 			Contract:     ContractAddress.String(),
 			ContractHash: ContractHash,
 		},
 		{
-			CID:          "",
+			LeafCID:      "",
-			MhKey:        "",
+			LeafMhKey:    "",
 			Contract:     "",
 			ContractHash: "",
 		},
 	}
+
 	MockRctMetaPostPublish = []models.ReceiptModel{
 		{
-			CID:          Rct1CID.String(),
+			LeafCID:      Rct1CID.String(),
-			MhKey:        Rct1MhKey,
+			LeafMhKey:    Rct1MhKey,
 			Contract:     "",
 			ContractHash: "",
 		},
 		{
-			CID:          Rct2CID.String(),
+			LeafCID:      Rct2CID.String(),
-			MhKey:        Rct2MhKey,
+			LeafMhKey:    Rct2MhKey,
 			Contract:     "",
 			ContractHash: "",
 		},
 		{
-			CID:          Rct3CID.String(),
+			LeafCID:      Rct3CID.String(),
-			MhKey:        Rct3MhKey,
+			LeafMhKey:    Rct3MhKey,
 			Contract:     ContractAddress.String(),
 			ContractHash: ContractHash,
 		},
 		{
-			CID:          Rct4CID.String(),
+			LeafCID:      Rct4CID.String(),
-			MhKey:        Rct4MhKey,
+			LeafMhKey:    Rct4MhKey,
 			Contract:     "",
 			ContractHash: "",
 		},
@@ -478,10 +481,6 @@ var (
 	Trx2IPLD, _    = blocks.NewBlockWithCid(Tx2, Trx2CID)
 	Trx3IPLD, _    = blocks.NewBlockWithCid(Tx3, Trx3CID)
 	Trx4IPLD, _    = blocks.NewBlockWithCid(Tx4, Trx4CID)
-	Rct1IPLD, _    = blocks.NewBlockWithCid(Rct1, Rct1CID)
-	Rct2IPLD, _    = blocks.NewBlockWithCid(Rct2, Rct2CID)
-	Rct3IPLD, _    = blocks.NewBlockWithCid(Rct3, Rct3CID)
-	Rct4IPLD, _    = blocks.NewBlockWithCid(Rct4, Rct4CID)
 	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
 	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
 	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
@@ -512,20 +511,20 @@ var (
 		},
 		Receipts: []ipfs.BlockModel{
 			{
-				Data: Rct1IPLD.RawData(),
+				Data: Rct1IPLD,
-				CID:  Rct1IPLD.Cid().String(),
+				CID:  Rct1CID.String(),
 			},
 			{
-				Data: Rct2IPLD.RawData(),
+				Data: Rct2IPLD,
-				CID:  Rct2IPLD.Cid().String(),
+				CID:  Rct2CID.String(),
 			},
 			{
-				Data: Rct3IPLD.RawData(),
+				Data: Rct3IPLD,
-				CID:  Rct3IPLD.Cid().String(),
+				CID:  Rct3CID.String(),
 			},
 			{
-				Data: Rct4IPLD.RawData(),
+				Data: Rct4IPLD,
-				CID:  Rct4IPLD.Cid().String(),
+				CID:  Rct4CID.String(),
 			},
 		},
 		StateNodes: []eth.StateNode{