cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Change receipt CID and MHKey to point trie leaf cid and mkhey.

Browse Source
This commit is contained in:
Arijit Das 2021-09-16 18:19:42 +05:30
parent 2de9c5bd48
commit 5772d52eb1
13 changed files with 248 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
db/migrations/00007_create_eth_receipt_cids_table.sql
View File

@ -2,8 +2,8 @@
CREATE TABLE eth.receipt_cids (
id SERIAL PRIMARY KEY,
tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
cid TEXT NOT NULL,
mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
leaf_cid TEXT NOT NULL,
leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
contract VARCHAR(66),
contract_hash VARCHAR(66),
post_state VARCHAR(66),

4
db/migrations/00013_create_cid_indexes.sql
View File

@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-- receipt indexes
CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);

4
docker-compose.yml
View File

@ -5,7 +5,7 @@ services:
restart: unless-stopped
depends_on:
- statediff-migrations
image: vulcanize/dapptools:v0.29.0-v1.10.8-statediff-0.0.26
image: vulcanize/dapptools:latest
environment:
DB_USER: vdbm
DB_NAME: vulcanize_public
@ -20,7 +20,7 @@ services:
restart: on-failure
depends_on:
- db
image: vulcanize/statediff-migrations:v0.6.0
image: vulcanize/statediff-migrations:latest
environment:
DATABASE_USER: vdbm
DATABASE_NAME: vulcanize_public

BIN
pkg/.DS_Store vendored Normal file
View File

Binary file not shown.

14
pkg/eth/backend.go
View File

@ -376,7 +376,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs {
var receipt types.Receipt
err = receipt.UnmarshalBinary(rctIPLD.Data)
nodeVal, err := DecodeLeafNode(rctIPLD.Data)
if err != nil {
return nil, err
}
err = receipt.UnmarshalBinary(nodeVal)
if err != nil {
return nil, err
}
@ -476,7 +481,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs {
var receipt types.Receipt
err = receipt.UnmarshalBinary(rctIPLD.Data)
nodeVal, err := DecodeLeafNode(rctIPLD.Data)
if err != nil {
return nil, err
}
err = receipt.UnmarshalBinary(nodeVal)
if err != nil {
return nil, err
}

10
pkg/eth/cid_retriever.go
View File

@ -292,7 +292,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte
func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for header id ", headerID)
args := make([]interface{}, 0, 4)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
receipt_cids.contract, receipt_cids.contract_hash
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
@ -316,12 +316,12 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF
id := 1
pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.receipt_id,
eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.cid, eth.receipt_cids.post_status
eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid, eth.receipt_cids.post_status
FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
WHERE eth.log_cids.receipt_id = receipt_cids.id
AND receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1`
AND receipt_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1`
args = append(args, blockHash.String())
id++
@ -380,7 +380,7 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilte
func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for block ", blockNumber)
args := make([]interface{}, 0, 5)
pgStr := `SELECT receipt_cids.id, receipt_cids.cid, receipt_cids.mh_key, receipt_cids.tx_id
pgStr := `SELECT receipt_cids.id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id`
@ -595,7 +595,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) (
// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) {
log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
receipt_cids.contract, receipt_cids.contract_hash
FROM eth.receipt_cids, eth.transaction_cids
WHERE tx_id = ANY($1::INTEGER[])

48
pkg/eth/cid_retriever_test.go
View File

@ -239,6 +239,17 @@ var _ = Describe("Retriever", func() {
Expect(err).ToNot(HaveOccurred())
})
It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
type rctCIDAndMHKeyResult struct {
LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"`
}
expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
cids, empty, err := retriever.Retrieve(openFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
@ -253,10 +264,10 @@ var _ = Describe("Retriever", func() {
Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
Expect(len(cids[0].Receipts)).To(Equal(4))
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
Expect(len(cids[0].Receipts)).To(Equal(3))
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
Expect(len(cids[0].StateNodes)).To(Equal(2))
for _, stateNode := range cids[0].StateNodes {
@ -279,6 +290,17 @@ var _ = Describe("Retriever", func() {
})
It("Applies filters from the provided config.Subscription", func() {
type rctCIDAndMHKeyResult struct {
LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"`
}
expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
@ -292,6 +314,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
expectedReceiptCID.ID = cids1[0].Receipts[0].ID
expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
@ -307,6 +331,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
expectedReceiptCID.ID = cids2[0].Receipts[0].ID
expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
@ -322,6 +348,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
expectedReceiptCID.ID = cids3[0].Receipts[0].ID
expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
@ -337,6 +365,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
expectedReceiptCID.ID = cids4[0].Receipts[0].ID
expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
@ -351,10 +381,10 @@ var _ = Describe("Retriever", func() {
Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue())
Expect(len(cids5[0].StateNodes)).To(Equal(0))
Expect(len(cids5[0].StorageNodes)).To(Equal(0))
Expect(len(cids5[0].Receipts)).To(Equal(4))
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue())
Expect(len(cids5[0].Receipts)).To(Equal(3))
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
Expect(err).ToNot(HaveOccurred())
@ -373,6 +403,8 @@ var _ = Describe("Retriever", func() {
expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
expectedReceiptCID.ID = cids6[0].Receipts[0].ID
expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
cids7, empty, err := retriever.Retrieve(stateFilter, 1)

58
pkg/eth/filterer.go
View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
@ -164,7 +165,8 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
if !receiptFilter.Off {
response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
for _, receipt := range payload.Receipts {
rctLeafCID, rctIPLDData := FetchRctLeafNodeData(payload.Receipts)
for idx, receipt := range payload.Receipts {
// topics is always length 4
topics := make([][]string, 4)
contracts := make([]string, len(receipt.Logs))
@ -177,18 +179,9 @@ func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *
// TODO: Verify this filter logic.
if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) {
receiptBuffer := new(bytes.Buffer)
if err := receipt.EncodeRLP(receiptBuffer); err != nil {
return err
}
data := receiptBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
if err != nil {
return err
}
response.Receipts = append(response.Receipts, ipfs.BlockModel{
Data: data,
CID: cid.String(),
Data: rctIPLDData[idx],
CID: rctLeafCID[idx].String(),
})
}
}
@ -329,3 +322,44 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool {
}
return false
}
// FetchRctLeafNodeData fetches receipt leaf node IPLD data and CIDs
func FetchRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte) {
receiptTrie := ipld.NewRctTrie()
for idx, rct := range rcts {
ethRct, err := ipld.NewReceipt(rct)
if err != nil {
return nil, nil
}
if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
return nil, nil
}
}
_, err := receiptTrie.GetNodes()
if err != nil {
return nil, nil
}
rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
if err != nil {
return nil, nil
}
ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
ethRctleafNodeData := make([][]byte, len(rctLeafNodes))
for i, rln := range rctLeafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
err = rlp.Decode(r, &idx)
if err != nil {
return nil, nil
}
ethRctleafNodeCids[idx] = rln.Cid()
ethRctleafNodeData[idx] = rln.RawData()
}
return ethRctleafNodeCids, ethRctleafNodeData
}

36
pkg/eth/filterer_test.go
View File

@ -52,10 +52,10 @@ var _ = Describe("Filterer", func() {
Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx1)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue())
Expect(len(iplds.Receipts)).To(Equal(4))
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3)).To(BeTrue())
Expect(len(iplds.Receipts)).To(Equal(3))
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
Expect(len(iplds.StateNodes)).To(Equal(2))
for _, stateNode := range iplds.StateNodes {
Expect(stateNode.Type).To(Equal(sdtypes.Leaf))
@ -87,8 +87,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds1.StateNodes)).To(Equal(0))
Expect(len(iplds1.Receipts)).To(Equal(1))
Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct1IPLD.RawData(),
CID: test_helpers.Rct1IPLD.Cid().String(),
Data: test_helpers.Rct1IPLD,
CID: test_helpers.Rct1CID.String(),
}))
iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
@ -102,8 +102,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds2.StateNodes)).To(Equal(0))
Expect(len(iplds2.Receipts)).To(Equal(1))
Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct1IPLD.RawData(),
CID: test_helpers.Rct1IPLD.Cid().String(),
Data: test_helpers.Rct1IPLD,
CID: test_helpers.Rct1CID.String(),
}))
iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
@ -117,8 +117,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds3.StateNodes)).To(Equal(0))
Expect(len(iplds3.Receipts)).To(Equal(1))
Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct1IPLD.RawData(),
CID: test_helpers.Rct1IPLD.Cid().String(),
Data: test_helpers.Rct1IPLD,
CID: test_helpers.Rct1CID.String(),
}))
iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
@ -132,8 +132,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds4.StateNodes)).To(Equal(0))
Expect(len(iplds4.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct2IPLD.RawData(),
CID: test_helpers.Rct2IPLD.Cid().String(),
Data: test_helpers.Rct2IPLD,
CID: test_helpers.Rct2CID.String(),
}))
iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
@ -148,10 +148,10 @@ var _ = Describe("Filterer", func() {
Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx3)).To(BeTrue())
Expect(len(iplds5.StorageNodes)).To(Equal(0))
Expect(len(iplds5.StateNodes)).To(Equal(0))
Expect(len(iplds5.Receipts)).To(Equal(4))
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3)).To(BeTrue())
Expect(len(iplds5.Receipts)).To(Equal(3))
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@ -165,8 +165,8 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds6.StateNodes)).To(Equal(0))
Expect(len(iplds6.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: test_helpers.Rct2IPLD.RawData(),
CID: test_helpers.Rct2IPLD.Cid().String(),
Data: test_helpers.Rct2IPLD,
CID: test_helpers.Rct2CID.String(),
}))
iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)

5
pkg/eth/ipld_fetcher.go
View File

@ -153,13 +153,14 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs
log.Debug("fetching receipt iplds")
rctIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey)
if err != nil {
return nil, err
}
//nodeVal, err := DecodeLeafNode(rctBytes)
rctIPLDs[i] = ipfs.BlockModel{
Data: rctBytes,
CID: c.CID,
CID: c.LeafCID,
}
}
return rctIPLDs, nil

79
pkg/eth/ipld_retriever.go
View File

@ -19,6 +19,9 @@ package eth
import (
"fmt"
"github.com/ethereum/go-ethereum/statediff/trie"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
@ -81,29 +84,29 @@ const (
FROM eth.transaction_cids
INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
WHERE tx_hash = $1`
RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data
RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash
RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE block_hash = $1
ORDER BY eth.transaction_cids.index ASC`
RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data
RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE block_number = $1
ORDER BY eth.transaction_cids.index ASC`
RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data
RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
WHERE tx_hash = $1`
RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type
FROM eth.state_cids
@ -149,6 +152,12 @@ const (
LIMIT 1`
)
type rctIpldResult struct {
LeafCID string `db:"leaf_cid"`
Data []byte `db:"data"`
TxHash string `db:"tx_hash"`
}
type ipldResult struct {
CID string `db:"cid"`
Data []byte `db:"data"`
@ -316,9 +325,26 @@ func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, [
return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
}
// DecodeLeafNode decodes the leaf node data
func DecodeLeafNode(node []byte) ([]byte, error) {
var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
return nil, err
}
ty, err := trie.CheckKeyType(nodeElements)
if err != nil {
return nil, err
}
if ty != sdtypes.Leaf {
return nil, fmt.Errorf("expected leaf node but found %s", ty)
}
return nodeElements[1].([]byte), nil
}
// RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
rctResults := make([]ipldResult, 0)
rctResults := make([]rctIpldResult, 0)
hashStrs := make([]string, len(hashes))
for i, hash := range hashes {
hashStrs[i] = hash.Hex()
@ -329,15 +355,20 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults))
for i, res := range rctResults {
cids[i] = res.CID
rcts[i] = res.Data
cids[i] = res.LeafCID
nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, err
}
rcts[i] = nodeVal
//rcts[i] = res.Data
}
return cids, rcts, nil
}
// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash
func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
rctResults := make([]ipldResult, 0)
rctResults := make([]rctIpldResult, 0)
if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
return nil, nil, nil, err
}
@ -346,8 +377,13 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
txs := make([]common.Hash, len(rctResults))
for i, res := range rctResults {
cids[i] = res.CID
rcts[i] = res.Data
cids[i] = res.LeafCID
nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, nil, err
}
rcts[i] = nodeVal
//rcts[i] = res.Data
txs[i] = common.HexToHash(res.TxHash)
}
@ -356,23 +392,28 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string,
// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash
func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
rctResults := make([]ipldResult, 0)
rctResults := make([]rctIpldResult, 0)
if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
return nil, nil, err
}
cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults))
for i, res := range rctResults {
cids[i] = res.CID
rcts[i] = res.Data
cids[i] = res.LeafCID
nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, err
}
rcts[i] = nodeVal
//rcts[i] = res.Data
}
return cids, rcts, nil
}
// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
rctResult := new(ipldResult)
return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
rctResult := new(rctIpldResult)
return rctResult.LeafCID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
}
type nodeInfo struct {

2
pkg/eth/test_helpers.go
View File

@ -59,7 +59,7 @@ func TxModelsContainsCID(txs []models.TxModel, cid string) bool {
// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
for _, rct := range rcts {
if rct.CID == cid {
if rct.LeafCID == cid {
return true
}
}

115
pkg/eth/test_helpers/test_data.go
View File

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/statediff/testhelpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
@ -47,6 +48,7 @@ import (
// Test variables
var (
// block data
db *postgres.DB
BlockNumber = big.NewInt(1)
MockHeader = types.Header{
Time: 0,
@ -165,36 +167,36 @@ var (
Tx3 = GetTxnRlp(2, MockTransactions)
Tx4 = GetTxnRlp(3, MockTransactions)
Rct1 = GetRctRlp(0, MockReceipts)
Rct2 = GetRctRlp(1, MockReceipts)
Rct3 = GetRctRlp(2, MockReceipts)
Rct4 = GetRctRlp(3, MockReceipts)
HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID)
Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID)
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID)
Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID)
Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID)
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct1, multihash.KECCAK_256)
Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID)
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct2, multihash.KECCAK_256)
Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID)
Rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct3, multihash.KECCAK_256)
Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID)
Rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, Rct4, multihash.KECCAK_256)
Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
State1MhKey = shared.MultihashKeyFromCID(State1CID)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
State2MhKey = shared.MultihashKeyFromCID(State2CID)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
StorageMhKey = shared.MultihashKeyFromCID(StorageCID)
MockTrxMeta = []models.TxModel{
rctCIDs, rctIPLDData = eth.FetchRctLeafNodeData(MockReceipts)
HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID)
Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID)
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID)
Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID)
Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
Trx4MhKey = shared.MultihashKeyFromCID(Trx3CID)
Rct1CID = rctCIDs[0]
Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID)
Rct2CID = rctCIDs[1]
Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID)
Rct3CID = rctCIDs[2]
Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID)
Rct4CID = rctCIDs[3]
Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
State1MhKey = shared.MultihashKeyFromCID(State1CID)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
State2MhKey = shared.MultihashKeyFromCID(State2CID)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
StorageMhKey = shared.MultihashKeyFromCID(StorageCID)
Rct1IPLD = rctIPLDData[0]
Rct2IPLD = rctIPLDData[1]
Rct3IPLD = rctIPLDData[2]
Rct4IPLD = rctIPLDData[3]
MockTrxMeta = []models.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
MhKey: "",
@ -272,52 +274,53 @@ var (
}
MockRctMeta = []models.ReceiptModel{
{
CID: "",
MhKey: "",
LeafCID: "",
LeafMhKey: "",
Contract: "",
ContractHash: "",
},
{
CID: "",
MhKey: "",
LeafCID: "",
LeafMhKey: "",
Contract: "",
ContractHash: "",
},
{
CID: "",
MhKey: "",
LeafCID: "",
LeafMhKey: "",
Contract: ContractAddress.String(),
ContractHash: ContractHash,
},
{
CID: "",
MhKey: "",
LeafCID: "",
LeafMhKey: "",
Contract: "",
ContractHash: "",
},
}
MockRctMetaPostPublish = []models.ReceiptModel{
{
CID: Rct1CID.String(),
MhKey: Rct1MhKey,
LeafCID: Rct1CID.String(),
LeafMhKey: Rct1MhKey,
Contract: "",
ContractHash: "",
},
{
CID: Rct2CID.String(),
MhKey: Rct2MhKey,
LeafCID: Rct2CID.String(),
LeafMhKey: Rct2MhKey,
Contract: "",
ContractHash: "",
},
{
CID: Rct3CID.String(),
MhKey: Rct3MhKey,
LeafCID: Rct3CID.String(),
LeafMhKey: Rct3MhKey,
Contract: ContractAddress.String(),
ContractHash: ContractHash,
},
{
CID: Rct4CID.String(),
MhKey: Rct4MhKey,
LeafCID: Rct4CID.String(),
LeafMhKey: Rct4MhKey,
Contract: "",
ContractHash: "",
},
@ -478,10 +481,6 @@ var (
Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID)
Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID)
Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(Rct1, Rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(Rct2, Rct2CID)
Rct3IPLD, _ = blocks.NewBlockWithCid(Rct3, Rct3CID)
Rct4IPLD, _ = blocks.NewBlockWithCid(Rct4, Rct4CID)
State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
@ -512,20 +511,20 @@ var (
},
Receipts: []ipfs.BlockModel{
{
Data: Rct1IPLD.RawData(),
CID: Rct1IPLD.Cid().String(),
Data: Rct1IPLD,
CID: Rct1CID.String(),
},
{
Data: Rct2IPLD.RawData(),
CID: Rct2IPLD.Cid().String(),
Data: Rct2IPLD,
CID: Rct2CID.String(),
},
{
Data: Rct3IPLD.RawData(),
CID: Rct3IPLD.Cid().String(),
Data: Rct3IPLD,
CID: Rct3CID.String(),
},
{
Data: Rct4IPLD.RawData(),
CID: Rct4IPLD.Cid().String(),
Data: Rct4IPLD,
CID: Rct4CID.String(),
},
},
StateNodes: []eth.StateNode{