From 6f1bfc7fcec0479d41f33f26e7625a4e7b4d387f Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 2 Mar 2023 18:27:30 -0600 Subject: [PATCH] combine ipld and cid retriever into one struct. the original reason for the separation of concerns was because we used to fetch cids directly from the database but then use those to fetch iplds using an ipfs.BlockService abstraction ontop of our database (and/or a ipfs.BlockExchange). But now they are both accessed directly in the local DB. Additionally, some of the queries should be further refined/refactored/combined as we no longer need to take two trips but can retrieve cid and ipld in the same query. --- cmd/serve.go | 1 - pkg/eth/api.go | 4 +- pkg/eth/backend.go | 35 +- pkg/eth/cid_retriever.go | 376 ------------ pkg/eth/retriever.go | 555 ++++++++++++++++++ ...id_retriever_test.go => retriever_test.go} | 4 +- pkg/eth/test_helpers/test_data.go | 310 +--------- pkg/graphql/graphql.go | 2 +- test_config/test_config.go | 1 + 9 files changed, 584 insertions(+), 704 deletions(-) delete mode 100644 pkg/eth/cid_retriever.go create mode 100644 pkg/eth/retriever.go rename pkg/eth/{cid_retriever_test.go => retriever_test.go} (98%) diff --git a/cmd/serve.go b/cmd/serve.go index ec662cfc..0bd3e29b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -25,7 +25,6 @@ import ( "sync" "time" - "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/rpc" "github.com/mailgun/groupcache/v2" diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 0e923cca..96196df9 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -720,7 +720,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log // If we have a blockHash to filter on, fire off single retrieval query if crit.BlockHash != nil { - filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, 0, crit.BlockHash) + filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, 0, crit.BlockHash) if err != nil { return nil, err } @@ -748,7 +748,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log end := endingBlock.Int64() var logs []*types.Log for i := start; i <= end; i++ { - filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, i, nil) + filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, i, nil) if err != nil { return nil, err } diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index cecd29a0..03688d0f 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -28,6 +28,7 @@ import ( validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" + "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" @@ -48,8 +49,6 @@ import ( sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" ) var ( @@ -105,8 +104,7 @@ type Backend struct { DB *sqlx.DB // postgres db interfaces - Retriever *CIDRetriever - IPLDRetriever *IPLDRetriever + Retriever *Retriever // ethereum interfaces EthDB ethdb.Database @@ -131,7 +129,7 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { groupName = StateDBGroupCacheName } - r := NewCIDRetriever(db) + r := NewRetriever(db) ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{ Name: groupName, Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024, @@ -143,7 +141,6 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { return &Backend{ DB: db, Retriever: r, - IPLDRetriever: NewIPLDRetriever(db), EthDB: ethDB, StateDatabase: state.NewDatabase(ethDB), Config: c, @@ -204,7 +201,7 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He } }() - _, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash) + _, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash) if err != nil { return nil, err } @@ -407,7 +404,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo // GetHeaderByBlockHash retrieves header for a provided block hash func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.Header, error) { - _, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash) + _, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash) if err != nil { return nil, err } @@ -418,7 +415,7 @@ func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.He // GetUnclesByBlockHash retrieves uncles for a provided block hash func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.Header, error) { - _, uncleBytes, err := b.IPLDRetriever.RetrieveUnclesByBlockHash(tx, hash) + _, uncleBytes, err := b.Retriever.RetrieveUnclesByBlockHash(tx, hash) if err != nil { return nil, err } @@ -439,7 +436,7 @@ func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types. // GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) { - _, uncleBytes, err := b.IPLDRetriever.RetrieveUncles(tx, hash, number) + _, uncleBytes, err := b.Retriever.RetrieveUncles(tx, hash, number) if err != nil { return nil, err } @@ -460,7 +457,7 @@ func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, n // GetTransactionsByBlockHash retrieves transactions for a provided block hash func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) { - _, transactionBytes, err := b.IPLDRetriever.RetrieveTransactionsByBlockHash(tx, hash) + _, transactionBytes, err := b.Retriever.RetrieveTransactionsByBlockHash(tx, hash) if err != nil { return nil, err } @@ -480,7 +477,7 @@ func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (typ // GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) { - _, transactionBytes, err := b.IPLDRetriever.RetrieveTransactions(tx, hash, number) + _, transactionBytes, err := b.Retriever.RetrieveTransactions(tx, hash, number) if err != nil { return nil, err } @@ -500,7 +497,7 @@ func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.H // GetReceiptsByBlockHash retrieves receipts for a provided block hash func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) { - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash) + _, receiptBytes, txs, err := b.Retriever.RetrieveReceiptsByBlockHash(tx, hash) if err != nil { return nil, err } @@ -518,7 +515,7 @@ func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.R // GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) { - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) + _, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number) if err != nil { return nil, err } @@ -607,7 +604,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64) } }() - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) + _, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number) if err != nil { return nil, err } @@ -695,8 +692,8 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) { func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { vmError := func() error { return nil } txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(header, b, nil) - return vm.NewEVM(context, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil + blockContext := core.NewEVMBlockContext(header, b, nil) + return vm.NewEVM(blockContext, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil } // GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash @@ -748,7 +745,7 @@ func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, return nil, err } - _, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash) + _, accountRlp, err := b.Retriever.RetrieveAccountByAddressAndBlockHash(address, hash) if err != nil { return nil, err } @@ -879,7 +876,7 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, return nil, err } - _, _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash) + _, _, storageRlp, err := b.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash) return storageRlp, err } diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go deleted file mode 100644 index 7548f4c8..00000000 --- a/pkg/eth/cid_retriever.go +++ /dev/null @@ -1,376 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package eth - -import ( - "fmt" - "math/big" - "strconv" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/log" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/jmoiron/sqlx" - "github.com/lib/pq" - "gorm.io/driver/postgres" - "gorm.io/gorm" -) - -// CIDRetriever satisfies the CIDRetriever interface for ethereum -type CIDRetriever struct { - db *sqlx.DB - gormDB *gorm.DB -} - -type IPLDModelRecord struct { - models.IPLDModel -} - -// TableName overrides the table name used by IPLD -func (IPLDModelRecord) TableName() string { - return "ipld.blocks" -} - -type HeaderCIDRecord struct { - CID string `gorm:"column:cid"` - BlockHash string `gorm:"primaryKey"` - BlockNumber string `gorm:"primaryKey"` - ParentHash string - Timestamp uint64 - StateRoot string - TotalDifficulty string `gorm:"column:td"` - TxRoot string - RctRoot string `gorm:"column:receipt_root"` - UncleRoot string - Bloom []byte - MhKey string - - // gorm doesn't check if foreign key exists in database. - // It is required to eager load relations using preload. - TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"` - IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` -} - -// TableName overrides the table name used by HeaderCIDRecord -func (HeaderCIDRecord) TableName() string { - return "eth.header_cids" -} - -type TransactionCIDRecord struct { - CID string `gorm:"column:cid"` - TxHash string `gorm:"primaryKey"` - BlockNumber string `gorm:"primaryKey"` - HeaderID string `gorm:"column:header_id"` - Index int64 - Src string - Dst string - MhKey string - IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` -} - -// TableName overrides the table name used by TransactionCIDRecord -func (TransactionCIDRecord) TableName() string { - return "eth.transaction_cids" -} - -// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface -func NewCIDRetriever(db *sqlx.DB) *CIDRetriever { - gormDB, err := gorm.Open(postgres.New(postgres.Config{ - Conn: db, - }), &gorm.Config{}) - - if err != nil { - log.Error(err) - return nil - } - - return &CIDRetriever{ - db: db, - gormDB: gormDB, - } -} - -// RetrieveFirstBlockNumber is used to retrieve the first block number in the db -func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) { - var blockNumber int64 - err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1") - return blockNumber, err -} - -// RetrieveLastBlockNumber is used to retrieve the latest block number in the db -func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) { - var blockNumber int64 - err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1") - return blockNumber, err -} - -func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) { - for i, topicSet := range topics { - if len(topicSet) == 0 { - continue - } - - if !first { - pgStr += " AND" - } else { - first = false - } - pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id) - args = append(args, pq.Array(topicSet)) - *id++ - } - return pgStr, args -} - -func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) { - if len(rctFilter.LogAddresses) > 0 { - pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id) - args = append(args, pq.Array(rctFilter.LogAddresses)) - *id++ - } - - // Filter on topics if there are any - if hasTopics(rctFilter.Topics) { - pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) - } - - return pgStr, args -} - -func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) { - rctCond := " AND (receipt_cids.tx_id = ANY ( " - logQuery := "SELECT rct_id FROM eth.log_cids WHERE" - if len(rctFilter.LogAddresses) > 0 { - // Filter on log contract addresses if there are any - pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id) - args = append(args, pq.Array(rctFilter.LogAddresses)) - *id++ - - // Filter on topics if there are any - if hasTopics(rctFilter.Topics) { - pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) - } - - pgStr += ")" - - // Filter on txHashes if there are any, and we are matching txs - if rctFilter.MatchTxs && len(txHashes) > 0 { - pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) - args = append(args, pq.Array(txHashes)) - } - pgStr += ")" - } else { // If there are no contract addresses to filter on - // Filter on topics if there are any - if hasTopics(rctFilter.Topics) { - pgStr += rctCond + logQuery - pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true) - pgStr += ")" - // Filter on txHashes if there are any, and we are matching txs - if rctFilter.MatchTxs && len(txHashes) > 0 { - pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) - args = append(args, pq.Array(txHashes)) - } - pgStr += ")" - } else if rctFilter.MatchTxs && len(txHashes) > 0 { - // If there are no contract addresses or topics to filter on, - // Filter on txHashes if there are any, and we are matching txs - pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id) - args = append(args, pq.Array(txHashes)) - } - } - - return pgStr, args -} - -// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided -// filter parameters. -func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) { - log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String()) - args := make([]interface{}, 0, 4) - id := 1 - pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash, - eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, - eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, - data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash - FROM eth.log_cids, eth.receipt_cids, ipld.blocks - WHERE eth.log_cids.rct_id = receipt_cids.tx_id - AND eth.log_cids.header_id = receipt_cids.header_id - AND eth.log_cids.block_number = receipt_cids.block_number - AND log_cids.cid = blocks.key - AND log_cids.block_number = blocks.block_number - AND receipt_cids.header_id = $1` - - args = append(args, blockHash.String()) - id++ - - if blockNumber != nil { - pgStr += ` AND receipt_cids.block_number = $2` - id++ - args = append(args, blockNumber.Int64()) - } - - pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) - pgStr += ` ORDER BY log_cids.index` - - logCIDs := make([]LogResult, 0) - err := tx.Select(&logCIDs, pgStr, args...) - if err != nil { - return nil, err - } - - return logCIDs, nil -} - -// RetrieveFilteredLog retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided -// filter parameters. -func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { - log.Debug("retrieving log cids for receipt ids") - args := make([]interface{}, 0, 4) - pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, - eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, - eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, - eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash - FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE eth.log_cids.rct_id = receipt_cids.tx_id - AND eth.log_cids.header_id = eth.receipt_cids.header_id - AND eth.log_cids.block_number = eth.receipt_cids.block_number - AND receipt_cids.tx_id = transaction_cids.tx_hash - AND receipt_cids.header_id = transaction_cids.header_id - AND receipt_cids.block_number = transaction_cids.block_number - AND transaction_cids.header_id = header_cids.block_hash - AND transaction_cids.block_number = header_cids.block_number` - id := 1 - if blockNumber > 0 { - pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) - args = append(args, blockNumber) - id++ - } - if blockHash != nil { - pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id) - args = append(args, blockHash.String()) - id++ - } - - pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) - pgStr += ` ORDER BY log_cids.index` - - logCIDs := make([]LogResult, 0) - err := tx.Select(&logCIDs, pgStr, args...) - if err != nil { - return nil, err - } - - return logCIDs, nil -} - -func hasTopics(topics [][]string) bool { - for _, topicSet := range topics { - if len(topicSet) > 0 { - return true - } - } - return false -} - -// RetrieveBlockNumberByHash returns the block number for the given block hash -func (ecr *CIDRetriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) { - log.Debug("retrieving block number for block hash ", blockHash.String()) - pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1` - var blockNumberStr string - if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil { - return 0, err - } - return strconv.ParseUint(blockNumberStr, 10, 64) -} - -// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) { - log.Debug("retrieving header cids and tx cids for block number ", blockNumber) - - var headerCIDs []HeaderCIDRecord - - // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 - // Will use join for TransactionCIDs once preload for 1:N is supported. - err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { - return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") - }).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error - - if err != nil { - log.Error("header cid retrieval error") - return nil, err - } - - return headerCIDs, nil -} - -// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number) -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) { - log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String()) - - var headerCIDs []HeaderCIDRecord - - conditions := map[string]interface{}{"block_hash": blockHash.String()} - if blockNumber != nil { - conditions["header_cids.block_number"] = blockNumber.Int64() - } - - // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 - // Will use join for TransactionCIDs once preload for 1:N is supported. - err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { - return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") - }).Joins("IPLD").Find(&headerCIDs, conditions).Error - - if err != nil { - log.Error("header cid retrieval error") - return HeaderCIDRecord{}, err - } - - if len(headerCIDs) == 0 { - return HeaderCIDRecord{}, errHeaderHashNotFound - } else if len(headerCIDs) > 1 { - return HeaderCIDRecord{}, errMultipleHeadersForHash - } - - return headerCIDs[0], nil -} - -// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number) -func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) { - log.Debug("retrieving tx cid for tx hash ", txHash) - - var txCIDs []TransactionCIDRecord - - var err error - if blockNumber != nil { - err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error - } else { - err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error - } - if err != nil { - log.Error("tx retrieval error") - return TransactionCIDRecord{}, err - } - - if len(txCIDs) == 0 { - return TransactionCIDRecord{}, errTxHashNotFound - } else if len(txCIDs) > 1 { - // a transaction can be part of a only one canonical block - return TransactionCIDRecord{}, errTxHashInMultipleBlocks - } - - return txCIDs[0], nil -} diff --git a/pkg/eth/retriever.go b/pkg/eth/retriever.go new file mode 100644 index 00000000..e9a04622 --- /dev/null +++ b/pkg/eth/retriever.go @@ -0,0 +1,555 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package eth + +import ( + "fmt" + "math/big" + "strconv" + + "github.com/cerc-io/ipld-eth-server/v4/pkg/log" + "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/jmoiron/sqlx" + "github.com/lib/pq" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +// Retriever is used for fetching +type Retriever struct { + db *sqlx.DB + gormDB *gorm.DB +} + +type IPLDModelRecord struct { + models.IPLDModel +} + +// TableName overrides the table name used by IPLD +func (IPLDModelRecord) TableName() string { + return "ipld.blocks" +} + +type HeaderCIDRecord struct { + CID string `gorm:"column:cid"` + BlockHash string `gorm:"primaryKey"` + BlockNumber string `gorm:"primaryKey"` + ParentHash string + Timestamp uint64 + StateRoot string + TotalDifficulty string `gorm:"column:td"` + TxRoot string + RctRoot string `gorm:"column:receipt_root"` + UncleRoot string + Bloom []byte + MhKey string + + // gorm doesn't check if foreign key exists in database. + // It is required to eager load relations using preload. + TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"` + IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` +} + +// TableName overrides the table name used by HeaderCIDRecord +func (HeaderCIDRecord) TableName() string { + return "eth.header_cids" +} + +type TransactionCIDRecord struct { + CID string `gorm:"column:cid"` + TxHash string `gorm:"primaryKey"` + BlockNumber string `gorm:"primaryKey"` + HeaderID string `gorm:"column:header_id"` + Index int64 + Src string + Dst string + MhKey string + IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` +} + +// TableName overrides the table name used by TransactionCIDRecord +func (TransactionCIDRecord) TableName() string { + return "eth.transaction_cids" +} + +// NewRetriever returns a pointer to a new Retriever which supports the Retriever interface +func NewRetriever(db *sqlx.DB) *Retriever { + gormDB, err := gorm.Open(postgres.New(postgres.Config{ + Conn: db, + }), &gorm.Config{}) + + if err != nil { + log.Error(err) + return nil + } + + return &Retriever{ + db: db, + gormDB: gormDB, + } +} + +// RetrieveFirstBlockNumber is used to retrieve the first block number in the db +func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) { + var blockNumber int64 + err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1") + return blockNumber, err +} + +// RetrieveLastBlockNumber is used to retrieve the latest block number in the db +func (r *Retriever) RetrieveLastBlockNumber() (int64, error) { + var blockNumber int64 + err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1") + return blockNumber, err +} + +func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) { + for i, topicSet := range topics { + if len(topicSet) == 0 { + continue + } + + if !first { + pgStr += " AND" + } else { + first = false + } + pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id) + args = append(args, pq.Array(topicSet)) + *id++ + } + return pgStr, args +} + +func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) { + if len(rctFilter.LogAddresses) > 0 { + pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id) + args = append(args, pq.Array(rctFilter.LogAddresses)) + *id++ + } + + // Filter on topics if there are any + if hasTopics(rctFilter.Topics) { + pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) + } + + return pgStr, args +} + +func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) { + rctCond := " AND (receipt_cids.tx_id = ANY ( " + logQuery := "SELECT rct_id FROM eth.log_cids WHERE" + if len(rctFilter.LogAddresses) > 0 { + // Filter on log contract addresses if there are any + pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id) + args = append(args, pq.Array(rctFilter.LogAddresses)) + *id++ + + // Filter on topics if there are any + if hasTopics(rctFilter.Topics) { + pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) + } + + pgStr += ")" + + // Filter on txHashes if there are any, and we are matching txs + if rctFilter.MatchTxs && len(txHashes) > 0 { + pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) + } + pgStr += ")" + } else { // If there are no contract addresses to filter on + // Filter on topics if there are any + if hasTopics(rctFilter.Topics) { + pgStr += rctCond + logQuery + pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true) + pgStr += ")" + // Filter on txHashes if there are any, and we are matching txs + if rctFilter.MatchTxs && len(txHashes) > 0 { + pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) + } + pgStr += ")" + } else if rctFilter.MatchTxs && len(txHashes) > 0 { + // If there are no contract addresses or topics to filter on, + // Filter on txHashes if there are any, and we are matching txs + pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) + } + } + + return pgStr, args +} + +// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided +// filter parameters. +func (r *Retriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) { + log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String()) + args := make([]interface{}, 0, 4) + id := 1 + pgStr := RetrieveFilteredGQLLogs + args = append(args, blockHash.String()) + id++ + + if blockNumber != nil { + pgStr += ` AND receipt_cids.block_number = $2` + id++ + args = append(args, blockNumber.Int64()) + } + + pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) + pgStr += ` ORDER BY log_cids.index` + + logCIDs := make([]LogResult, 0) + err := tx.Select(&logCIDs, pgStr, args...) + if err != nil { + return nil, err + } + + return logCIDs, nil +} + +// RetrieveFilteredLogs retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided +// filter parameters. +func (r *Retriever) RetrieveFilteredLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { + log.Debug("retrieving log cids for receipt ids") + args := make([]interface{}, 0, 4) + pgStr := RetrieveFilteredLogs + id := 1 + if blockNumber > 0 { + pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) + args = append(args, blockNumber) + id++ + } + if blockHash != nil { + pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id) + args = append(args, blockHash.String()) + id++ + } + + pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) + pgStr += ` ORDER BY log_cids.index` + + logCIDs := make([]LogResult, 0) + err := tx.Select(&logCIDs, pgStr, args...) + if err != nil { + return nil, err + } + + return logCIDs, nil +} + +func hasTopics(topics [][]string) bool { + for _, topicSet := range topics { + if len(topicSet) > 0 { + return true + } + } + return false +} + +// RetrieveBlockNumberByHash returns the block number for the given block hash +func (r *Retriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) { + log.Debug("retrieving block number for block hash ", blockHash.String()) + pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1` + var blockNumberStr string + if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil { + return 0, err + } + return strconv.ParseUint(blockNumberStr, 10, 64) +} + +// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number +func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) { + log.Debug("retrieving header cids and tx cids for block number ", blockNumber) + + var headerCIDs []HeaderCIDRecord + + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 + // Will use join for TransactionCIDs once preload for 1:N is supported. + err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { + return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") + }).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error + + if err != nil { + log.Error("header cid retrieval error") + return nil, err + } + + return headerCIDs, nil +} + +// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number) +func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) { + log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String()) + + var headerCIDs []HeaderCIDRecord + + conditions := map[string]interface{}{"block_hash": blockHash.String()} + if blockNumber != nil { + conditions["header_cids.block_number"] = blockNumber.Int64() + } + + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 + // Will use join for TransactionCIDs once preload for 1:N is supported. + err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { + return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") + }).Joins("IPLD").Find(&headerCIDs, conditions).Error + + if err != nil { + log.Error("header cid retrieval error") + return HeaderCIDRecord{}, err + } + + if len(headerCIDs) == 0 { + return HeaderCIDRecord{}, errHeaderHashNotFound + } else if len(headerCIDs) > 1 { + return HeaderCIDRecord{}, errMultipleHeadersForHash + } + + return headerCIDs[0], nil +} + +// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number) +func (r *Retriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) { + log.Debug("retrieving tx cid for tx hash ", txHash) + + var txCIDs []TransactionCIDRecord + + var err error + if blockNumber != nil { + err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error + } else { + err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error + } + if err != nil { + log.Error("tx retrieval error") + return TransactionCIDRecord{}, err + } + + if len(txCIDs) == 0 { + return TransactionCIDRecord{}, errTxHashNotFound + } else if len(txCIDs) > 1 { + // a transaction can be part of a only one canonical block + return TransactionCIDRecord{}, errTxHashInMultipleBlocks + } + + return txCIDs[0], nil +} + +var EmptyNodeValue = make([]byte, common.HashLength) + +// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash +func (r *Retriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) { + headerResult := new(ipldResult) + return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex()) +} + +// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block) +func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + uncleResults := make([]ipldResult, 0) + if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(uncleResults)) + uncles := make([][]byte, len(uncleResults)) + for i, res := range uncleResults { + cids[i] = res.CID + uncles[i] = res.Data + } + return cids, uncles, nil +} + +// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block) +func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { + uncleResults := make([]ipldResult, 0) + if err := tx.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil { + return nil, nil, err + } + cids := make([]string, len(uncleResults)) + uncles := make([][]byte, len(uncleResults)) + for i, res := range uncleResults { + cids[i] = res.CID + uncles[i] = res.Data + } + return cids, uncles, nil +} + +// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number +func (r *Retriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + txResults := make([]ipldResult, 0) + if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(txResults)) + txs := make([][]byte, len(txResults)) + for i, res := range txResults { + cids[i] = res.CID + txs[i] = res.Data + } + return cids, txs, nil +} + +// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash +func (r *Retriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { + txResults := make([]ipldResult, 0) + if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil { + return nil, nil, err + } + cids := make([]string, len(txResults)) + txs := make([][]byte, len(txResults)) + for i, res := range txResults { + cids[i] = res.CID + txs[i] = res.Data + } + return cids, txs, nil +} + +// DecodeLeafNode decodes the leaf node data +func DecodeLeafNode(node []byte) ([]byte, error) { + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return nil, err + } + ty, err := trie_helpers.CheckKeyType(nodeElements) + if err != nil { + return nil, err + } + + if ty != sdtypes.Leaf { + return nil, fmt.Errorf("expected leaf node but found %s", ty) + } + return nodeElements[1].([]byte), nil +} + +// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number. +// cid returned corresponds to the leaf node data which contains the receipt. +func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) { + rctResults := make([]rctIpldResult, 0) + if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil { + return nil, nil, nil, err + } + cids := make([]string, len(rctResults)) + rcts := make([][]byte, len(rctResults)) + txs := make([]common.Hash, len(rctResults)) + + for i, res := range rctResults { + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, nil, err + } + rcts[i] = nodeVal + txs[i] = common.HexToHash(res.TxHash) + } + + return cids, rcts, txs, nil +} + +// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash. +// cid returned corresponds to the leaf node data which contains the receipt. +func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) { + rctResults := make([]rctIpldResult, 0) + if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil { + return nil, nil, nil, err + } + cids := make([]string, len(rctResults)) + rcts := make([][]byte, len(rctResults)) + txs := make([]common.Hash, len(rctResults)) + + for i, res := range rctResults { + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, nil, err + } + rcts[i] = nodeVal + txs[i] = common.HexToHash(res.TxHash) + } + + return cids, rcts, txs, nil +} + +// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash +// TODO: ensure this handles deleted accounts appropriately +func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { + accountResult := new(nodeInfo) + leafKey := crypto.Keccak256Hash(address.Bytes()) + if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { + return "", nil, err + } + + if accountResult.NodeType == sdtypes.Removed.Int() { + return "", EmptyNodeValue, nil + } + + blockNumber, err := strconv.ParseUint(accountResult.BlockNumber, 10, 64) + if err != nil { + return "", nil, err + } + accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.CID, blockNumber) + if err != nil { + return "", nil, err + } + + var i []interface{} + if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { + return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return "", nil, fmt.Errorf("eth Retriever expected state leaf node rlp to decode into two elements") + } + return accountResult.CID, i[1].([]byte), nil +} + +// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash +func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) { + storageResult := new(nodeInfo) + stateLeafKey := crypto.Keccak256Hash(address.Bytes()) + storageHash := crypto.Keccak256Hash(key.Bytes()) + if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { + return "", nil, nil, err + } + if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { + return "", EmptyNodeValue, EmptyNodeValue, nil + } + + blockNumber, err := strconv.ParseUint(storageResult.BlockNumber, 10, 64) + if err != nil { + return "", nil, nil, err + } + storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.CID, blockNumber) + if err != nil { + return "", nil, nil, err + } + + var i []interface{} + if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { + err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) + return "", nil, nil, err + } + if len(i) != 2 { + return "", nil, nil, fmt.Errorf("eth Retriever expected storage leaf node rlp to decode into two elements") + } + return storageResult.CID, storageResult.Data, i[1].([]byte), nil +} diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/retriever_test.go similarity index 98% rename from pkg/eth/cid_retriever_test.go rename to pkg/eth/retriever_test.go index 2941da0d..b2bb27dd 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/retriever_test.go @@ -33,13 +33,13 @@ var _ = Describe("Retriever", func() { var ( db *sqlx.DB diffIndexer interfaces.StateDiffIndexer - retriever *eth.CIDRetriever + retriever *eth.Retriever ) BeforeEach(func() { db = shared.SetupDB() diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash()) - retriever = eth.NewCIDRetriever(db) + retriever = eth.NewRetriever(db) }) AfterEach(func() { shared.TearDownDB(db) diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index 2e5d40b9..37553009 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -17,28 +17,22 @@ package test_helpers import ( - "bytes" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "math/big" + "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/shared" testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" - blocks "github.com/ipfs/go-block-format" - "github.com/multiformats/go-multihash" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" ) // Test variables @@ -75,10 +69,8 @@ var ( Extra: []byte{}, }, } - ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) - MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) - MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) - MockChildHeader = types.Header{ + MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) + MockChildHeader = types.Header{ Time: 0, Number: new(big.Int).Add(BlockNumber, common.Big1), Root: common.HexToHash("0x0"), @@ -89,7 +81,6 @@ var ( ParentHash: MockBlock.Header().Hash(), } MockChild = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) - MockChildRlp, _ = rlp.EncodeToBytes(MockChild.Header()) Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594") @@ -157,41 +148,10 @@ var ( Index: 5, } - Tx1 = GetTxnRlp(0, MockTransactions) - Tx2 = GetTxnRlp(1, MockTransactions) - Tx3 = GetTxnRlp(2, MockTransactions) - Tx4 = GetTxnRlp(3, MockTransactions) - - rctCIDs, rctIPLDData, _ = GetRctLeafNodeData(MockReceipts) - HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) - HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID) - Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256) - Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID) - Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256) - Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID) - Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256) - Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID) - Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256) - Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID) - Rct1CID = rctCIDs[0] - Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID) - Rct2CID = rctCIDs[1] - Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID) - Rct3CID = rctCIDs[2] - Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID) - Rct4CID = rctCIDs[3] - Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID) - State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) - State1MhKey = shared.MultihashKeyFromCID(State1CID) - State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) - State2MhKey = shared.MultihashKeyFromCID(State2CID) - StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) - StorageMhKey = shared.MultihashKeyFromCID(StorageCID) - Rct1IPLD = rctIPLDData[0] - Rct2IPLD = rctIPLDData[1] - Rct3IPLD = rctIPLDData[2] - Rct4IPLD = rctIPLDData[3] - MockTrxMeta = []models.TxModel{ + rctCIDs, _, _ = GetRctLeafNodeData(MockReceipts) + Rct1CID = rctCIDs[0] + Rct4CID = rctCIDs[3] + MockTrxMeta = []models.TxModel{ { CID: "", // This is empty until we go to publish to ipfs MhKey: "", @@ -229,48 +189,6 @@ var ( Data: []byte{}, }, } - MockTrxMetaPostPublsh = []models.TxModel{ - { - BlockNumber: "1", - CID: Trx1CID.String(), // This is empty until we go to publish to ipfs - MhKey: Trx1MhKey, - Src: SenderAddr.Hex(), - Dst: Address.String(), - Index: 0, - TxHash: MockTransactions[0].Hash().String(), - Data: []byte{}, - }, - { - BlockNumber: "1", - CID: Trx2CID.String(), - MhKey: Trx2MhKey, - Src: SenderAddr.Hex(), - Dst: AnotherAddress.String(), - Index: 1, - TxHash: MockTransactions[1].Hash().String(), - Data: []byte{}, - }, - { - BlockNumber: "1", - CID: Trx3CID.String(), - MhKey: Trx3MhKey, - Src: SenderAddr.Hex(), - Dst: "", - Index: 2, - TxHash: MockTransactions[2].Hash().String(), - Data: MockContractByteCode, - }, - { - BlockNumber: "1", - CID: Trx4CID.String(), - MhKey: Trx4MhKey, - Src: SenderAddr.Hex(), - Dst: AnotherAddress1.String(), - Index: 3, - TxHash: MockTransactions[3].Hash().String(), - Data: []byte{}, - }, - } MockRctMeta = []models.ReceiptModel{ { LeafCID: "", @@ -298,41 +216,6 @@ var ( }, } - MockRctMetaPostPublish = []models.ReceiptModel{ - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct1CID.String(), - LeafMhKey: Rct1MhKey, - Contract: "", - ContractHash: "", - }, - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct2CID.String(), - LeafMhKey: Rct2MhKey, - Contract: "", - ContractHash: "", - }, - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct3CID.String(), - LeafMhKey: Rct3MhKey, - Contract: ContractAddress.String(), - ContractHash: ContractHash, - }, - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct4CID.String(), - LeafMhKey: Rct4MhKey, - Contract: "", - ContractHash: "", - }, - } - // statediff data storageLocation = common.HexToHash("0") StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() @@ -401,24 +284,6 @@ var ( StorageNodes: []sdtypes.StorageNode{}, }, } - MockStateMetaPostPublish = []models.StateNodeModel{ - { - BlockNumber: "1", - CID: State1CID.String(), - MhKey: State1MhKey, - Path: []byte{'\x06'}, - NodeType: 2, - StateKey: common.BytesToHash(ContractLeafKey).Hex(), - }, - { - BlockNumber: "1", - CID: State2CID.String(), - MhKey: State2MhKey, - Path: []byte{'\x0c'}, - NodeType: 2, - StateKey: common.BytesToHash(AccountLeafKey).Hex(), - }, - } MockStorageNodes = map[string][]sdtypes.StorageNode{ contractPath: { { @@ -439,149 +304,6 @@ var ( StorageNodes: MockStorageNodes, StateNodes: MockStateNodes, } - MockConvertedPayloadForChild = eth.ConvertedPayload{ - TotalDifficulty: MockChild.Difficulty(), - Block: MockChild, - Receipts: MockReceipts, - TxMetaData: MockTrxMeta, - ReceiptMetaData: MockRctMeta, - StorageNodes: MockStorageNodes, - StateNodes: MockStateNodes, - } - - Reward = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts) - MockCIDWrapper = ð.CIDWrapper{ - BlockNumber: new(big.Int).Set(BlockNumber), - Header: models.HeaderModel{ - BlockNumber: "1", - BlockHash: MockBlock.Hash().String(), - ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000", - CID: HeaderCID.String(), - MhKey: HeaderMhKey, - TotalDifficulty: MockBlock.Difficulty().String(), - Reward: Reward.String(), - StateRoot: MockBlock.Root().String(), - RctRoot: MockBlock.ReceiptHash().String(), - TxRoot: MockBlock.TxHash().String(), - UncleRoot: MockBlock.UncleHash().String(), - Bloom: MockBlock.Bloom().Bytes(), - Timestamp: MockBlock.Time(), - TimesValidated: 1, - Coinbase: "0x0000000000000000000000000000000000000000", - }, - Transactions: MockTrxMetaPostPublsh, - Receipts: MockRctMetaPostPublish, - Uncles: []models.UncleModel{}, - StateNodes: MockStateMetaPostPublish, - StorageNodes: []models.StorageNodeWithStateKeyModel{ - { - BlockNumber: "1", - Path: []byte{}, - CID: StorageCID.String(), - MhKey: StorageMhKey, - NodeType: 2, - StateKey: common.BytesToHash(ContractLeafKey).Hex(), - StorageKey: common.BytesToHash(StorageLeafKey).Hex(), - }, - }, - } - - HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID) - Trx1IPLD, _ = blocks.NewBlockWithCid(Tx1, Trx1CID) - Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID) - Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID) - Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID) - State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID) - State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID) - StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID) - - MockIPLDs = eth.IPLDs{ - BlockNumber: new(big.Int).Set(BlockNumber), - Header: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: HeaderIPLD.RawData(), - Key: HeaderIPLD.Cid().String(), - }, - Transactions: []models.IPLDModel{ - { - BlockNumber: BlockNumber.String(), - Data: Trx1IPLD.RawData(), - Key: Trx1IPLD.Cid().String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Trx2IPLD.RawData(), - Key: Trx2IPLD.Cid().String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Trx3IPLD.RawData(), - Key: Trx3IPLD.Cid().String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Trx4IPLD.RawData(), - Key: Trx4IPLD.Cid().String(), - }, - }, - Receipts: []models.IPLDModel{ - { - BlockNumber: BlockNumber.String(), - Data: Rct1IPLD, - Key: Rct1CID.String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Rct2IPLD, - Key: Rct2CID.String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Rct3IPLD, - Key: Rct3CID.String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Rct4IPLD, - Key: Rct4CID.String(), - }, - }, - StateNodes: []eth.StateNode{ - { - StateLeafKey: common.BytesToHash(ContractLeafKey), - Type: sdtypes.Leaf, - IPLD: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: State1IPLD.RawData(), - Key: State1IPLD.Cid().String(), - }, - Path: []byte{'\x06'}, - }, - { - StateLeafKey: common.BytesToHash(AccountLeafKey), - Type: sdtypes.Leaf, - IPLD: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: State2IPLD.RawData(), - Key: State2IPLD.Cid().String(), - }, - Path: []byte{'\x0c'}, - }, - }, - StorageNodes: []eth.StorageNode{ - { - StateLeafKey: common.BytesToHash(ContractLeafKey), - StorageLeafKey: common.BytesToHash(StorageLeafKey), - Type: sdtypes.Leaf, - IPLD: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: StorageIPLD.RawData(), - Key: StorageIPLD.Cid().String(), - }, - Path: []byte{}, - }, - }, - } LondonBlockNum = new(big.Int).Add(BlockNumber, big.NewInt(2)) MockLondonHeader = types.Header{ @@ -741,21 +463,3 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts, return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr } - -func GetTxnRlp(num int, txs types.Transactions) []byte { - buf := new(bytes.Buffer) - txs.EncodeIndex(num, buf) - tx := make([]byte, buf.Len()) - copy(tx, buf.Bytes()) - buf.Reset() - return tx -} - -func GetRctRlp(num int, rcts types.Receipts) []byte { - buf := new(bytes.Buffer) - rcts.EncodeIndex(num, buf) - rct := make([]byte, buf.Len()) - copy(rct, buf.Bytes()) - buf.Reset() - return rct -} diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index afcbd0f3..1f73b606 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -1009,7 +1009,7 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct { Contract common.Address Slot common.Hash }) (*StorageResult, error) { - cid, ipldBlock, rlpValue, err := r.backend.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash) + cid, ipldBlock, rlpValue, err := r.backend.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash) if err != nil { if err == sql.ErrNoRows { diff --git a/test_config/test_config.go b/test_config/test_config.go index 982f9b44..de67b704 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -18,6 +18,7 @@ package test_config import ( "errors" + "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/spf13/viper"