cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

work on wasm watchers

Browse Source
This commit is contained in:
Ian Norden
2020-02-24 12:54:10 -06:00
parent e3e8700d34
commit 25aa4634e9
32 changed files with 683 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/super_node/eth/backend.go
+6 -6
View File
@@ -86,7 +86,7 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
return nil, fmt.Errorf("header at block %d is not available", number)
}
// Fetch the header IPLDs for those CIDs
headerIPLDs, err := b.Fetcher.FetchHeaders([]HeaderModel{headerCids[0]})
headerIPLD, err := b.Fetcher.FetchHeader(headerCids[0])
if err != nil {
return nil, err
}
@@ -94,7 +94,7 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
// We throw an error in FetchHeaders() if the number of headers does not match the number of CIDs and we already
// confirmed the number of CIDs is greater than 0 so there is no need to bound check the slice before accessing
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
if err := rlp.DecodeBytes(headerIPLD.Data, &header); err != nil {
return nil, err
}
return &header, nil
@@ -172,12 +172,12 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
// Fetch and decode the header IPLD
headerIPLDs, err := b.Fetcher.FetchHeaders([]HeaderModel{headerCID})
headerIPLD, err := b.Fetcher.FetchHeader(headerCID)
if err != nil {
return nil, err
}
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
if err := rlp.DecodeBytes(headerIPLD.Data, &header); err != nil {
return nil, err
}
// Fetch and decode the uncle IPLDs
@@ -232,12 +232,12 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
return nil, err
}
// Fetch and decode the header IPLD
headerIPLDs, err := b.Fetcher.FetchHeaders([]HeaderModel{headerCID})
headerIPLD, err := b.Fetcher.FetchHeader(headerCID)
if err != nil {
return nil, err
}
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
if err := rlp.DecodeBytes(headerIPLD.Data, &header); err != nil {
return nil, err
}
// Fetch and decode the uncle IPLDs
pkg/super_node/eth/filterer.go
+3 -2
View File
@@ -87,10 +87,10 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
if err != nil {
return err
}
response.Headers = append(response.Headers, ipfs.BlockModel{
response.Header = ipfs.BlockModel{
Data: headerRLP,
CID: cid.String(),
})
}
if headerFilter.Uncles {
response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles))
for i, uncle := range payload.Block.Body().Uncles {
@@ -126,6 +126,7 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLD
trxHashes = make([]common.Hash, 0, trxLen)
response.Transactions = make([]ipfs.BlockModel, 0, trxLen)
for i, trx := range payload.Block.Body().Transactions {
// TODO: check if want corresponding receipt and if we do we must include this transaction
if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) {
trxBuffer := new(bytes.Buffer)
if err := trx.EncodeRLP(trxBuffer); err != nil {
pkg/super_node/eth/filterer_test.go
+9 -9
View File
@@ -43,7 +43,7 @@ var _ = Describe("Filterer", func() {
iplds, ok := payload.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(iplds.Headers).To(Equal(mocks.MockIPLDs.Headers))
Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
var expectedEmptyUncles []ipfs.BlockModel
Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
Expect(len(iplds.Transactions)).To(Equal(2))
@@ -77,7 +77,7 @@ var _ = Describe("Filterer", func() {
iplds1, ok := payload1.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds1.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds1.Headers)).To(Equal(0))
Expect(iplds1.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds1.Uncles)).To(Equal(0))
Expect(len(iplds1.Transactions)).To(Equal(0))
Expect(len(iplds1.StorageNodes)).To(Equal(0))
@@ -93,7 +93,7 @@ var _ = Describe("Filterer", func() {
iplds2, ok := payload2.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds2.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds2.Headers)).To(Equal(0))
Expect(iplds2.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds2.Uncles)).To(Equal(0))
Expect(len(iplds2.Transactions)).To(Equal(0))
Expect(len(iplds2.StorageNodes)).To(Equal(0))
@@ -109,7 +109,7 @@ var _ = Describe("Filterer", func() {
iplds3, ok := payload3.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds3.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds3.Headers)).To(Equal(0))
Expect(iplds3.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds3.Uncles)).To(Equal(0))
Expect(len(iplds3.Transactions)).To(Equal(0))
Expect(len(iplds3.StorageNodes)).To(Equal(0))
@@ -125,7 +125,7 @@ var _ = Describe("Filterer", func() {
iplds4, ok := payload4.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds4.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds4.Headers)).To(Equal(0))
Expect(iplds4.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds4.Uncles)).To(Equal(0))
Expect(len(iplds4.Transactions)).To(Equal(0))
Expect(len(iplds4.StorageNodes)).To(Equal(0))
@@ -141,7 +141,7 @@ var _ = Describe("Filterer", func() {
iplds5, ok := payload5.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds5.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds5.Headers)).To(Equal(0))
Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds5.Uncles)).To(Equal(0))
Expect(len(iplds5.Transactions)).To(Equal(2))
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
@@ -157,7 +157,7 @@ var _ = Describe("Filterer", func() {
iplds6, ok := payload6.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds6.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds6.Headers)).To(Equal(0))
Expect(iplds6.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds6.Uncles)).To(Equal(0))
Expect(len(iplds6.Transactions)).To(Equal(1))
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
@@ -174,7 +174,7 @@ var _ = Describe("Filterer", func() {
iplds7, ok := payload7.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds7.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds7.Headers)).To(Equal(0))
Expect(iplds7.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds7.Uncles)).To(Equal(0))
Expect(len(iplds7.Transactions)).To(Equal(0))
Expect(len(iplds7.StorageNodes)).To(Equal(0))
@@ -191,7 +191,7 @@ var _ = Describe("Filterer", func() {
iplds8, ok := payload8.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds8.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(len(iplds8.Headers)).To(Equal(0))
Expect(iplds8.Header).To(Equal(ipfs.BlockModel{}))
Expect(len(iplds8.Uncles)).To(Equal(0))
Expect(len(iplds8.Transactions)).To(Equal(0))
Expect(len(iplds8.StorageNodes)).To(Equal(0))
pkg/super_node/eth/indexer.go
+5 -1
View File
@@ -55,6 +55,7 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("eth indexer error when indexing header")
return err
}
for _, uncle := range cidPayload.UncleCIDs {
@@ -62,6 +63,7 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("eth indexer error when indexing uncle")
return err
}
}
@@ -69,12 +71,14 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("eth indexer error when indexing transactions and receipts")
return err
}
if err := in.indexStateAndStorageCIDs(tx, cidPayload, headerID); err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("eth indexer error when indexing state and storage nodes")
return err
}
return tx.Commit()
@@ -92,7 +96,7 @@ func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int
func (in *CIDIndexer) indexUncleCID(tx *sqlx.Tx, uncle UncleModel, headerID int64) error {
_, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward) = ($3, $4, $5)`,
uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID)
uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID, uncle.Reward)
return err
}
pkg/super_node/eth/ipld_fetcher.go
+14 -23
View File
@@ -61,7 +61,7 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
iplds := IPLDs{}
iplds.BlockNumber = cidWrapper.BlockNumber
var err error
iplds.Headers, err = f.FetchHeaders(cidWrapper.Headers)
iplds.Header, err = f.FetchHeader(cidWrapper.Header)
if err != nil {
return nil, err
}
@@ -89,30 +89,21 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
}
// FetchHeaders fetches headers
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching header iplds")
headerCids := make([]cid.Cid, len(cids))
for i, c := range cids {
dc, err := cid.Decode(c.CID)
if err != nil {
return nil, err
}
headerCids[i] = dc
// It uses the f.fetch method
func (f *IPLDFetcher) FetchHeader(c HeaderModel) (ipfs.BlockModel, error) {
log.Debug("fetching header ipld")
dc, err := cid.Decode(c.CID)
if err != nil {
return ipfs.BlockModel{}, err
}
headers := f.fetchBatch(headerCids)
headerIPLDs := make([]ipfs.BlockModel, len(headers))
for i, header := range headers {
headerIPLDs[i] = ipfs.BlockModel{
Data: header.RawData(),
CID: header.Cid().String(),
}
header, err := f.fetch(dc)
if err != nil {
return ipfs.BlockModel{}, err
}
if len(headerIPLDs) != len(headerCids) {
log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
return headerIPLDs, errUnexpectedNumberOfIPLDs
}
return headerIPLDs, nil
return ipfs.BlockModel{
Data: header.RawData(),
CID: header.Cid().String(),
}, nil
}
// FetchUncles fetches uncles
pkg/super_node/eth/ipld_fetcher_test.go
+3 -6
View File
@@ -50,10 +50,8 @@ var (
mockBlockService *mocks.MockIPFSBlockService
mockCIDWrapper = &eth.CIDWrapper{
BlockNumber: big.NewInt(9000),
Headers: []eth.HeaderModel{
{
CID: mockHeaderBlock.Cid().String(),
},
Header: eth.HeaderModel{
CID: mockHeaderBlock.Cid().String(),
},
Uncles: []eth.UncleModel{
{
@@ -107,8 +105,7 @@ var _ = Describe("Fetcher", func() {
iplds, ok := i.(eth.IPLDs)
Expect(ok).To(BeTrue())
Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
Expect(len(iplds.Headers)).To(Equal(1))
Expect(iplds.Headers[0]).To(Equal(ipfs.BlockModel{
Expect(iplds.Header).To(Equal(ipfs.BlockModel{
Data: mockHeaderBlock.RawData(),
CID: mockHeaderBlock.Cid().String(),
}))
pkg/super_node/eth/mocks/test_data.go
+10 -14
View File
@@ -296,15 +296,13 @@ var (
MockCIDWrapper = &eth.CIDWrapper{
BlockNumber: big.NewInt(1),
Headers: []eth2.HeaderModel{
{
BlockNumber: "1",
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
CID: HeaderCID.String(),
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: "5000000000000000000",
},
Header: eth2.HeaderModel{
BlockNumber: "1",
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
CID: HeaderCID.String(),
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: "5000000000000000000",
},
Transactions: MockTrxMetaPostPublsh,
Receipts: MockRctMetaPostPublish,
@@ -331,11 +329,9 @@ var (
MockIPLDs = eth.IPLDs{
BlockNumber: big.NewInt(1),
Headers: []ipfs.BlockModel{
{
Data: HeaderIPLD.RawData(),
CID: HeaderIPLD.Cid().String(),
},
Header: ipfs.BlockModel{
Data: HeaderIPLD.RawData(),
CID: HeaderIPLD.Cid().String(),
},
Transactions: []ipfs.BlockModel{
{
pkg/super_node/eth/retriever.go
+177 -86
View File
@@ -57,7 +57,7 @@ func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
}
// Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumber int64) (shared.CIDsForFetching, bool, error) {
func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumber int64) ([]shared.CIDsForFetching, bool, error) {
streamFilter, ok := filter.(*SubscriptionSettings)
if !ok {
return nil, true, fmt.Errorf("eth retriever expected filter type %T got %T", &SubscriptionSettings{}, filter)
@@ -68,21 +68,26 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
return nil, true, err
}
cw := new(CIDWrapper)
cw.BlockNumber = big.NewInt(blockNumber)
// Retrieve cached header CIDs
if !streamFilter.HeaderFilter.Off {
cw.Headers, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("header cid retrieval error")
return nil, true, err
// Retrieve cached header CIDs at this block height
headers, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
if streamFilter.HeaderFilter.Uncles {
for _, headerCID := range cw.Headers {
uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
log.Error("header cid retrieval error")
return nil, true, err
}
cws := make([]shared.CIDsForFetching, len(headers))
empty := true
for i, header := range headers {
cw := new(CIDWrapper)
cw.BlockNumber = big.NewInt(blockNumber)
if !streamFilter.HeaderFilter.Off {
cw.Header = header
empty = false
if streamFilter.HeaderFilter.Uncles {
// Retrieve uncle cids for this header id
uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
@@ -90,66 +95,73 @@ func (ecr *CIDRetriever) Retrieve(filter shared.SubscriptionSettings, blockNumbe
log.Error("uncle cid retrieval error")
return nil, true, err
}
cw.Uncles = append(cw.Uncles, uncleCIDs...)
cw.Uncles = uncleCIDs
}
}
}
// Retrieve cached trx CIDs
if !streamFilter.TxFilter.Off {
cw.Transactions, err = ecr.RetrieveTxCIDs(tx, streamFilter.TxFilter, blockNumber)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
// Retrieve cached trx CIDs
if !streamFilter.TxFilter.Off {
cw.Transactions, err = ecr.RetrieveTxCIDs(tx, streamFilter.TxFilter, header.ID)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("transaction cid retrieval error")
return nil, true, err
}
log.Error("transaction cid retrieval error")
return nil, true, err
}
}
trxIds := make([]int64, 0, len(cw.Transactions))
for _, tx := range cw.Transactions {
trxIds = append(trxIds, tx.ID)
}
// Retrieve cached receipt CIDs
if !streamFilter.ReceiptFilter.Off {
cw.Receipts, err = ecr.RetrieveRctCIDs(tx, streamFilter.ReceiptFilter, blockNumber, nil, trxIds)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
if len(cw.Transactions) > 0 {
empty = false
}
log.Error("receipt cid retrieval error")
return nil, true, err
}
}
// Retrieve cached state CIDs
if !streamFilter.StateFilter.Off {
cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, streamFilter.StateFilter, blockNumber)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
trxIds := make([]int64, len(cw.Transactions))
for j, tx := range cw.Transactions {
trxIds[j] = tx.ID
}
// Retrieve cached receipt CIDs
if !streamFilter.ReceiptFilter.Off {
cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, streamFilter.ReceiptFilter, header.ID, trxIds)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("receipt cid retrieval error")
return nil, true, err
}
log.Error("state cid retrieval error")
return nil, true, err
}
}
// Retrieve cached storage CIDs
if !streamFilter.StorageFilter.Off {
cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, streamFilter.StorageFilter, blockNumber)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
if len(cw.Receipts) > 0 {
empty = false
}
log.Error("storage cid retrieval error")
return nil, true, err
}
// Retrieve cached state CIDs
if !streamFilter.StateFilter.Off {
cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, streamFilter.StateFilter, header.ID)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("state cid retrieval error")
return nil, true, err
}
if len(cw.StateNodes) > 0 {
empty = false
}
}
// Retrieve cached storage CIDs
if !streamFilter.StorageFilter.Off {
cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, streamFilter.StorageFilter, header.ID)
if err != nil {
if err := tx.Rollback(); err != nil {
log.Error(err)
}
log.Error("storage cid retrieval error")
return nil, true, err
}
if len(cw.StorageNodes) > 0 {
empty = false
}
}
cws[i] = cw
}
return cw, empty(cw), tx.Commit()
}
func empty(cidWrapper *CIDWrapper) bool {
if len(cidWrapper.Transactions) > 0 || len(cidWrapper.Headers) > 0 || len(cidWrapper.Uncles) > 0 || len(cidWrapper.Receipts) > 0 || len(cidWrapper.StateNodes) > 0 || len(cidWrapper.StorageNodes) > 0 {
return false
}
return true
return cws, empty, tx.Commit()
}
// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
@@ -172,8 +184,8 @@ func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64
// RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
// also returns the ids for the returned transaction cids
func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNumber int64) ([]TxModel, error) {
log.Debug("retrieving transaction cids for block ", blockNumber)
func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]TxModel, error) {
log.Debug("retrieving transaction cids for header id ", headerID)
args := make([]interface{}, 0, 3)
results := make([]TxModel, 0)
id := 1
@@ -181,8 +193,8 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNum
transaction_cids.tx_hash, transaction_cids.cid,
transaction_cids.dst, transaction_cids.src, transaction_cids.index
FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $%d`, id)
args = append(args, blockNumber)
WHERE header_cids.id = $%d`, id)
args = append(args, headerID)
id++
if len(txFilter.Dst) > 0 {
pgStr += fmt.Sprintf(` AND transaction_cids.dst = ANY($%d::VARCHAR(66)[])`, id)
@@ -197,18 +209,98 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNum
return results, tx.Select(&results, pgStr, args...)
}
// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight that conform to the provided
// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
// filter parameters and correspond to the provided tx ids
func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]ReceiptModel, error) {
log.Debug("retrieving receipt cids for block ", blockNumber)
id := 1
func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]ReceiptModel, error) {
log.Debug("retrieving receipt cids for header id ", headerID)
args := make([]interface{}, 0, 4)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid,
receipt_cids.contract, receipt_cids.topic0s, receipt_cids.topic1s,
receipt_cids.topic2s, receipt_cids.topic3s
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id
AND header_cids.id = $1`
id := 2
args = append(args, headerID)
if len(rctFilter.Contracts) > 0 {
// Filter on contract addresses if there are any
pgStr += fmt.Sprintf(` AND ((receipt_cids.contract = ANY($%d::VARCHAR(66)[])`, id)
args = append(args, pq.Array(rctFilter.Contracts))
id++
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr += " AND ("
first := true
for i, topicSet := range rctFilter.Topics {
if i < 4 && len(topicSet) > 0 {
if first {
pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
first = false
} else {
pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
}
args = append(args, pq.Array(topicSet))
id++
}
}
pgStr += ")"
}
pgStr += ")"
// Filter on txIDs if there are any and we are matching txs
if rctFilter.MatchTxs && len(trxIds) > 0 {
pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
args = append(args, pq.Array(trxIds))
}
pgStr += ")"
} else { // If there are no contract addresses to filter on
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr += " AND (("
first := true
for i, topicSet := range rctFilter.Topics {
if i < 4 && len(topicSet) > 0 {
if first {
pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
first = false
} else {
pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
}
args = append(args, pq.Array(topicSet))
id++
}
}
pgStr += ")"
// Filter on txIDs if there are any and we are matching txs
if rctFilter.MatchTxs && len(trxIds) > 0 {
pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
args = append(args, pq.Array(trxIds))
}
pgStr += ")"
} else if rctFilter.MatchTxs && len(trxIds) > 0 {
// If there are no contract addresses or topics to filter on,
// Filter on txIDs if there are any and we are matching txs
pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
args = append(args, pq.Array(trxIds))
}
}
pgStr += ` ORDER BY transaction_cids.index`
receiptCids := make([]ReceiptModel, 0)
return receiptCids, tx.Select(&receiptCids, pgStr, args...)
}
// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
// filter parameters and correspond to the provided tx ids
func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]ReceiptModel, error) {
log.Debug("retrieving receipt cids for block ", blockNumber)
args := make([]interface{}, 0, 5)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid,
receipt_cids.contract, receipt_cids.topic0s, receipt_cids.topic1s,
receipt_cids.topic2s, receipt_cids.topic3s
FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.id
AND transaction_cids.header_id = header_cids.id`
id := 1
if blockNumber > 0 {
pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
args = append(args, blockNumber)
@@ -294,15 +386,15 @@ func hasTopics(topics [][]string) bool {
return false
}
// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided blockheight that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, blockNumber int64) ([]StateNodeModel, error) {
log.Debug("retrieving state cids for block ", blockNumber)
// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]StateNodeModel, error) {
log.Debug("retrieving state cids for header id ", headerID)
args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.id, state_cids.header_id,
state_cids.state_key, state_cids.leaf, state_cids.cid
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
WHERE header_cids.block_number = $1`
args = append(args, blockNumber)
WHERE header_cids.id = $1`
args = append(args, headerID)
addrLen := len(stateFilter.Addresses)
if addrLen > 0 {
keys := make([]string, addrLen)
@@ -319,18 +411,17 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
}
// RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided blockheight that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, blockNumber int64) ([]StorageNodeWithStateKeyModel, error) {
log.Debug("retrieving storage cids for block ", blockNumber)
// RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]StorageNodeWithStateKeyModel, error) {
log.Debug("retrieving storage cids for header id ", headerID)
args := make([]interface{}, 0, 3)
id := 1
pgStr := fmt.Sprintf(`SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE storage_cids.state_id = state_cids.id
AND state_cids.header_id = header_cids.id
AND header_cids.block_number = $%d`, id)
args = append(args, blockNumber)
id++
AND header_cids.id = $1`
args = append(args, headerID)
id := 2
addrLen := len(storageFilter.Addresses)
if addrLen > 0 {
keys := make([]string, addrLen)
pkg/super_node/eth/retriever_test.go
+27 -20
View File
@@ -230,14 +230,14 @@ var _ = Describe("Retriever", func() {
cids, empty, err := retriever.Retrieve(openFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper, ok := cids.(*eth.CIDWrapper)
Expect(len(cids)).To(Equal(1))
cidWrapper, ok := cids[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper.Headers)).To(Equal(1))
expectedHeaderCIDs := mocks.MockCIDWrapper.Headers
expectedHeaderCIDs[0].ID = cidWrapper.Headers[0].ID
expectedHeaderCIDs[0].NodeID = cidWrapper.Headers[0].NodeID
Expect(cidWrapper.Headers).To(Equal(expectedHeaderCIDs))
expectedHeaderCID := mocks.MockCIDWrapper.Header
expectedHeaderCID.ID = cidWrapper.Header.ID
expectedHeaderCID.NodeID = cidWrapper.Header.NodeID
Expect(cidWrapper.Header).To(Equal(expectedHeaderCID))
Expect(len(cidWrapper.Transactions)).To(Equal(2))
Expect(eth.TxModelsContainsCID(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
@@ -266,10 +266,11 @@ var _ = Describe("Retriever", func() {
cids1, empty, err := retriever.Retrieve(rctContractFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper1, ok := cids1.(*eth.CIDWrapper)
Expect(len(cids1)).To(Equal(1))
cidWrapper1, ok := cids1[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper1.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper1.Headers)).To(Equal(0))
Expect(cidWrapper1.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper1.Transactions)).To(Equal(0))
Expect(len(cidWrapper1.StateNodes)).To(Equal(0))
Expect(len(cidWrapper1.StorageNodes)).To(Equal(0))
@@ -282,10 +283,11 @@ var _ = Describe("Retriever", func() {
cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper2, ok := cids2.(*eth.CIDWrapper)
Expect(len(cids2)).To(Equal(1))
cidWrapper2, ok := cids2[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper2.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper2.Headers)).To(Equal(0))
Expect(cidWrapper2.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper2.Transactions)).To(Equal(0))
Expect(len(cidWrapper2.StateNodes)).To(Equal(0))
Expect(len(cidWrapper2.StorageNodes)).To(Equal(0))
@@ -298,10 +300,11 @@ var _ = Describe("Retriever", func() {
cids3, empty, err := retriever.Retrieve(rctTopicsAndContractFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper3, ok := cids3.(*eth.CIDWrapper)
Expect(len(cids3)).To(Equal(1))
cidWrapper3, ok := cids3[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper3.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper3.Headers)).To(Equal(0))
Expect(cidWrapper3.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper3.Transactions)).To(Equal(0))
Expect(len(cidWrapper3.StateNodes)).To(Equal(0))
Expect(len(cidWrapper3.StorageNodes)).To(Equal(0))
@@ -314,10 +317,11 @@ var _ = Describe("Retriever", func() {
cids4, empty, err := retriever.Retrieve(rctContractsAndTopicFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper4, ok := cids4.(*eth.CIDWrapper)
Expect(len(cids4)).To(Equal(1))
cidWrapper4, ok := cids4[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper4.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper4.Headers)).To(Equal(0))
Expect(cidWrapper4.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper4.Transactions)).To(Equal(0))
Expect(len(cidWrapper4.StateNodes)).To(Equal(0))
Expect(len(cidWrapper4.StorageNodes)).To(Equal(0))
@@ -330,10 +334,11 @@ var _ = Describe("Retriever", func() {
cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper5, ok := cids5.(*eth.CIDWrapper)
Expect(len(cids5)).To(Equal(1))
cidWrapper5, ok := cids5[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper5.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper5.Headers)).To(Equal(0))
Expect(cidWrapper5.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper5.Transactions)).To(Equal(2))
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, mocks.Trx1CID.String())).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, mocks.Trx2CID.String())).To(BeTrue())
@@ -346,10 +351,11 @@ var _ = Describe("Retriever", func() {
cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper6, ok := cids6.(*eth.CIDWrapper)
Expect(len(cids6)).To(Equal(1))
cidWrapper6, ok := cids6[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper6.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper6.Headers)).To(Equal(0))
Expect(cidWrapper6.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper6.Transactions)).To(Equal(1))
expectedTxCID := mocks.MockCIDWrapper.Transactions[1]
expectedTxCID.ID = cidWrapper6.Transactions[0].ID
@@ -366,10 +372,11 @@ var _ = Describe("Retriever", func() {
cids7, empty, err := retriever.Retrieve(stateFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())
cidWrapper7, ok := cids7.(*eth.CIDWrapper)
Expect(len(cids7)).To(Equal(1))
cidWrapper7, ok := cids7[0].(*eth.CIDWrapper)
Expect(ok).To(BeTrue())
Expect(cidWrapper7.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper7.Headers)).To(Equal(0))
Expect(cidWrapper7.Header).To(Equal(eth.HeaderModel{}))
Expect(len(cidWrapper7.Transactions)).To(Equal(0))
Expect(len(cidWrapper7.Receipts)).To(Equal(0))
Expect(len(cidWrapper7.StorageNodes)).To(Equal(0))
pkg/super_node/eth/subscription_config.go
+9 -10
View File
@@ -17,7 +17,6 @@
package eth
import (
"errors"
"math/big"
"github.com/spf13/viper"
@@ -53,7 +52,8 @@ type TxFilter struct {
// ReceiptFilter contains filter settings for receipts
type ReceiptFilter struct {
Off bool
Off bool
// TODO: change this so that we filter for receipts first and we always return the corresponding transaction
MatchTxs bool // turn on to retrieve receipts that pair with retrieved transactions
Contracts []string
Topics [][]string
@@ -70,7 +70,7 @@ type StateFilter struct {
type StorageFilter struct {
Off bool
Addresses []string
StorageKeys []string
StorageKeys []string // need to be the hashs key themselves not slot position
IntermediateNodes bool
}
@@ -96,13 +96,12 @@ func NewEthSubscriptionConfig() (*SubscriptionSettings, error) {
Src: viper.GetStringSlice("superNode.ethSubscription.txFilter.src"),
Dst: viper.GetStringSlice("superNode.ethSubscription.txFilter.dst"),
}
// Below defaults to false and one slice of length 0
// Which means we get all receipts by default
t := viper.Get("superNode.ethSubscription.receiptFilter.topics")
topics, ok := t.([][]string)
if !ok {
return nil, errors.New("superNode.ethSubscription.receiptFilter.topics needs to be a slice of string slices")
}
// By default all of the topic slices will be empty => match on any/all topics
topics := make([][]string, 4)
topics[0] = viper.GetStringSlice("superNode.ethSubscription.receiptFilter.topic0s")
topics[1] = viper.GetStringSlice("superNode.ethSubscription.receiptFilter.topic1s")
topics[2] = viper.GetStringSlice("superNode.ethSubscription.receiptFilter.topic2s")
topics[3] = viper.GetStringSlice("superNode.ethSubscription.receiptFilter.topic3s")
sc.ReceiptFilter = ReceiptFilter{
Off: viper.GetBool("superNode.ethSubscription.receiptFilter.off"),
MatchTxs: viper.GetBool("superNode.ethSubscription.receiptFilter.matchTxs"),
pkg/super_node/eth/types.go
+2 -2
View File
@@ -67,7 +67,7 @@ type CIDPayload struct {
// Passed to IPLDFetcher
type CIDWrapper struct {
BlockNumber *big.Int
Headers []HeaderModel
Header HeaderModel
Uncles []UncleModel
Transactions []TxModel
Receipts []ReceiptModel
@@ -79,7 +79,7 @@ type CIDWrapper struct {
// Returned by IPLDFetcher and ResponseFilterer
type IPLDs struct {
BlockNumber *big.Int
Headers []ipfs.BlockModel
Header ipfs.BlockModel
Uncles []ipfs.BlockModel
Transactions []ipfs.BlockModel
Receipts []ipfs.BlockModel