cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

combine ipld and cid retriever into one struct. the original reason for the separation of concerns was because we used to fetch cids directly from the database but then use those to fetch iplds using an ipfs.BlockService abstraction ontop of our database (and/or a ipfs.BlockExchange). But now they are both accessed directly in the local DB. Additionally, some of the queries should be further refined/refactored/combined as we no longer need to take two trips but can retrieve cid and ipld in the same query.

Browse Source
This commit is contained in:
i-norden 2023-03-02 18:27:30 -06:00
parent 5bd9783aed
commit 6f1bfc7fce
9 changed files with 584 additions and 704 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/serve.go
View File

@ -25,7 +25,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/mailgun/groupcache/v2" "github.com/mailgun/groupcache/v2"

4
pkg/eth/api.go
View File

@ -720,7 +720,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log
// If we have a blockHash to filter on, fire off single retrieval query // If we have a blockHash to filter on, fire off single retrieval query
if crit.BlockHash != nil { if crit.BlockHash != nil {
filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, 0, crit.BlockHash) filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, 0, crit.BlockHash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -748,7 +748,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log
end := endingBlock.Int64() end := endingBlock.Int64()
var logs []*types.Log var logs []*types.Log
for i := start; i <= end; i++ { for i := start; i <= end; i++ {
filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, i, nil) filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, i, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }

35
pkg/eth/backend.go
View File

@ -28,6 +28,7 @@ import (
validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg"
ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres" ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus"
@ -48,8 +49,6 @@ import (
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
) )
var ( var (
@ -105,8 +104,7 @@ type Backend struct {
DB *sqlx.DB DB *sqlx.DB
// postgres db interfaces // postgres db interfaces
Retriever *CIDRetriever Retriever *Retriever
IPLDRetriever *IPLDRetriever
// ethereum interfaces // ethereum interfaces
EthDB ethdb.Database EthDB ethdb.Database
@ -131,7 +129,7 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) {
groupName = StateDBGroupCacheName groupName = StateDBGroupCacheName
} }
r := NewCIDRetriever(db) r := NewRetriever(db)
ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{ ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
Name: groupName, Name: groupName,
Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024, Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024,
@ -143,7 +141,6 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) {
return &Backend{ return &Backend{
DB: db, DB: db,
Retriever: r, Retriever: r,
IPLDRetriever: NewIPLDRetriever(db),
EthDB: ethDB, EthDB: ethDB,
StateDatabase: state.NewDatabase(ethDB), StateDatabase: state.NewDatabase(ethDB),
Config: c, Config: c,
@ -204,7 +201,7 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He
} }
}() }()
_, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash) _, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -407,7 +404,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
// GetHeaderByBlockHash retrieves header for a provided block hash // GetHeaderByBlockHash retrieves header for a provided block hash
func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.Header, error) { func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.Header, error) {
_, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash) _, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -418,7 +415,7 @@ func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.He
// GetUnclesByBlockHash retrieves uncles for a provided block hash // GetUnclesByBlockHash retrieves uncles for a provided block hash
func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.Header, error) { func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.Header, error) {
_, uncleBytes, err := b.IPLDRetriever.RetrieveUnclesByBlockHash(tx, hash) _, uncleBytes, err := b.Retriever.RetrieveUnclesByBlockHash(tx, hash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -439,7 +436,7 @@ func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.
// GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number // GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number
func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) { func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) {
_, uncleBytes, err := b.IPLDRetriever.RetrieveUncles(tx, hash, number) _, uncleBytes, err := b.Retriever.RetrieveUncles(tx, hash, number)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -460,7 +457,7 @@ func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, n
// GetTransactionsByBlockHash retrieves transactions for a provided block hash // GetTransactionsByBlockHash retrieves transactions for a provided block hash
func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) { func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) {
_, transactionBytes, err := b.IPLDRetriever.RetrieveTransactionsByBlockHash(tx, hash) _, transactionBytes, err := b.Retriever.RetrieveTransactionsByBlockHash(tx, hash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -480,7 +477,7 @@ func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (typ
// GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number // GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number
func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) { func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) {
_, transactionBytes, err := b.IPLDRetriever.RetrieveTransactions(tx, hash, number) _, transactionBytes, err := b.Retriever.RetrieveTransactions(tx, hash, number)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -500,7 +497,7 @@ func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.H
// GetReceiptsByBlockHash retrieves receipts for a provided block hash // GetReceiptsByBlockHash retrieves receipts for a provided block hash
func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) { func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) {
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash) _, receiptBytes, txs, err := b.Retriever.RetrieveReceiptsByBlockHash(tx, hash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -518,7 +515,7 @@ func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.R
// GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number // GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number
func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) { func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) {
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) _, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -607,7 +604,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64)
} }
}() }()
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) _, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -695,8 +692,8 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) {
func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) {
vmError := func() error { return nil } vmError := func() error { return nil }
txContext := core.NewEVMTxContext(msg) txContext := core.NewEVMTxContext(msg)
context := core.NewEVMBlockContext(header, b, nil) blockContext := core.NewEVMBlockContext(header, b, nil)
return vm.NewEVM(context, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil return vm.NewEVM(blockContext, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil
} }
// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash // GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
@ -748,7 +745,7 @@ func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address,
return nil, err return nil, err
} }
_, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash) _, accountRlp, err := b.Retriever.RetrieveAccountByAddressAndBlockHash(address, hash)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -879,7 +876,7 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address,
return nil, err return nil, err
} }
_, _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash) _, _, storageRlp, err := b.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash)
return storageRlp, err return storageRlp, err
} }

376
pkg/eth/cid_retriever.go
View File

@ -1,376 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"fmt"
"math/big"
"strconv"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
// CIDRetriever satisfies the CIDRetriever interface for ethereum
type CIDRetriever struct {
db *sqlx.DB
gormDB *gorm.DB
}
type IPLDModelRecord struct {
models.IPLDModel
}
// TableName overrides the table name used by IPLD
func (IPLDModelRecord) TableName() string {
return "ipld.blocks"
}
type HeaderCIDRecord struct {
CID string `gorm:"column:cid"`
BlockHash string `gorm:"primaryKey"`
BlockNumber string `gorm:"primaryKey"`
ParentHash string
Timestamp uint64
StateRoot string
TotalDifficulty string `gorm:"column:td"`
TxRoot string
RctRoot string `gorm:"column:receipt_root"`
UncleRoot string
Bloom []byte
MhKey string
// gorm doesn't check if foreign key exists in database.
// It is required to eager load relations using preload.
TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"`
IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
}
// TableName overrides the table name used by HeaderCIDRecord
func (HeaderCIDRecord) TableName() string {
return "eth.header_cids"
}
type TransactionCIDRecord struct {
CID string `gorm:"column:cid"`
TxHash string `gorm:"primaryKey"`
BlockNumber string `gorm:"primaryKey"`
HeaderID string `gorm:"column:header_id"`
Index int64
Src string
Dst string
MhKey string
IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
}
// TableName overrides the table name used by TransactionCIDRecord
func (TransactionCIDRecord) TableName() string {
return "eth.transaction_cids"
}
// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
func NewCIDRetriever(db *sqlx.DB) *CIDRetriever {
gormDB, err := gorm.Open(postgres.New(postgres.Config{
Conn: db,
}), &gorm.Config{})
if err != nil {
log.Error(err)
return nil
}
return &CIDRetriever{
db: db,
gormDB: gormDB,
}
}
// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
var blockNumber int64
err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1")
return blockNumber, err
}
// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
var blockNumber int64
err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1")
return blockNumber, err
}
func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
for i, topicSet := range topics {
if len(topicSet) == 0 {
continue
}
if !first {
pgStr += " AND"
} else {
first = false
}
pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
args = append(args, pq.Array(topicSet))
*id++
}
return pgStr, args
}
func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
if len(rctFilter.LogAddresses) > 0 {
pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
args = append(args, pq.Array(rctFilter.LogAddresses))
*id++
}
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
}
return pgStr, args
}
func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
rctCond := " AND (receipt_cids.tx_id = ANY ( "
logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
if len(rctFilter.LogAddresses) > 0 {
// Filter on log contract addresses if there are any
pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
args = append(args, pq.Array(rctFilter.LogAddresses))
*id++
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
}
pgStr += ")"
// Filter on txHashes if there are any, and we are matching txs
if rctFilter.MatchTxs && len(txHashes) > 0 {
pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
args = append(args, pq.Array(txHashes))
}
pgStr += ")"
} else { // If there are no contract addresses to filter on
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr += rctCond + logQuery
pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
pgStr += ")"
// Filter on txHashes if there are any, and we are matching txs
if rctFilter.MatchTxs && len(txHashes) > 0 {
pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
args = append(args, pq.Array(txHashes))
}
pgStr += ")"
} else if rctFilter.MatchTxs && len(txHashes) > 0 {
// If there are no contract addresses or topics to filter on,
// Filter on txHashes if there are any, and we are matching txs
pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
args = append(args, pq.Array(txHashes))
}
}
return pgStr, args
}
// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
// filter parameters.
func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
args := make([]interface{}, 0, 4)
id := 1
pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash,
eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address,
eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data,
data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash
FROM eth.log_cids, eth.receipt_cids, ipld.blocks
WHERE eth.log_cids.rct_id = receipt_cids.tx_id
AND eth.log_cids.header_id = receipt_cids.header_id
AND eth.log_cids.block_number = receipt_cids.block_number
AND log_cids.cid = blocks.key
AND log_cids.block_number = blocks.block_number
AND receipt_cids.header_id = $1`
args = append(args, blockHash.String())
id++
if blockNumber != nil {
pgStr += ` AND receipt_cids.block_number = $2`
id++
args = append(args, blockNumber.Int64())
}
pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
pgStr += ` ORDER BY log_cids.index`
logCIDs := make([]LogResult, 0)
err := tx.Select(&logCIDs, pgStr, args...)
if err != nil {
return nil, err
}
return logCIDs, nil
}
// RetrieveFilteredLog retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided
// filter parameters.
func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
log.Debug("retrieving log cids for receipt ids")
args := make([]interface{}, 0, 4)
pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id,
eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash
FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE eth.log_cids.rct_id = receipt_cids.tx_id
AND eth.log_cids.header_id = eth.receipt_cids.header_id
AND eth.log_cids.block_number = eth.receipt_cids.block_number
AND receipt_cids.tx_id = transaction_cids.tx_hash
AND receipt_cids.header_id = transaction_cids.header_id
AND receipt_cids.block_number = transaction_cids.block_number
AND transaction_cids.header_id = header_cids.block_hash
AND transaction_cids.block_number = header_cids.block_number`
id := 1
if blockNumber > 0 {
pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
args = append(args, blockNumber)
id++
}
if blockHash != nil {
pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
args = append(args, blockHash.String())
id++
}
pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
pgStr += ` ORDER BY log_cids.index`
logCIDs := make([]LogResult, 0)
err := tx.Select(&logCIDs, pgStr, args...)
if err != nil {
return nil, err
}
return logCIDs, nil
}
func hasTopics(topics [][]string) bool {
for _, topicSet := range topics {
if len(topicSet) > 0 {
return true
}
}
return false
}
// RetrieveBlockNumberByHash returns the block number for the given block hash
func (ecr *CIDRetriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) {
log.Debug("retrieving block number for block hash ", blockHash.String())
pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1`
var blockNumberStr string
if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil {
return 0, err
}
return strconv.ParseUint(blockNumberStr, 10, 64)
}
// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
var headerCIDs []HeaderCIDRecord
// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
// Will use join for TransactionCIDs once preload for 1:N is supported.
err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
}).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error
if err != nil {
log.Error("header cid retrieval error")
return nil, err
}
return headerCIDs, nil
}
// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
var headerCIDs []HeaderCIDRecord
conditions := map[string]interface{}{"block_hash": blockHash.String()}
if blockNumber != nil {
conditions["header_cids.block_number"] = blockNumber.Int64()
}
// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
// Will use join for TransactionCIDs once preload for 1:N is supported.
err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
}).Joins("IPLD").Find(&headerCIDs, conditions).Error
if err != nil {
log.Error("header cid retrieval error")
return HeaderCIDRecord{}, err
}
if len(headerCIDs) == 0 {
return HeaderCIDRecord{}, errHeaderHashNotFound
} else if len(headerCIDs) > 1 {
return HeaderCIDRecord{}, errMultipleHeadersForHash
}
return headerCIDs[0], nil
}
// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
log.Debug("retrieving tx cid for tx hash ", txHash)
var txCIDs []TransactionCIDRecord
var err error
if blockNumber != nil {
err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
} else {
err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
}
if err != nil {
log.Error("tx retrieval error")
return TransactionCIDRecord{}, err
}
if len(txCIDs) == 0 {
return TransactionCIDRecord{}, errTxHashNotFound
} else if len(txCIDs) > 1 {
// a transaction can be part of a only one canonical block
return TransactionCIDRecord{}, errTxHashInMultipleBlocks
}
return txCIDs[0], nil
}

555
pkg/eth/retriever.go Normal file
View File

@ -0,0 +1,555 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"fmt"
"math/big"
"strconv"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/trie_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
// Retriever is used for fetching
type Retriever struct {
db *sqlx.DB
gormDB *gorm.DB
}
type IPLDModelRecord struct {
models.IPLDModel
}
// TableName overrides the table name used by IPLD
func (IPLDModelRecord) TableName() string {
return "ipld.blocks"
}
type HeaderCIDRecord struct {
CID string `gorm:"column:cid"`
BlockHash string `gorm:"primaryKey"`
BlockNumber string `gorm:"primaryKey"`
ParentHash string
Timestamp uint64
StateRoot string
TotalDifficulty string `gorm:"column:td"`
TxRoot string
RctRoot string `gorm:"column:receipt_root"`
UncleRoot string
Bloom []byte
MhKey string
// gorm doesn't check if foreign key exists in database.
// It is required to eager load relations using preload.
TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"`
IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
}
// TableName overrides the table name used by HeaderCIDRecord
func (HeaderCIDRecord) TableName() string {
return "eth.header_cids"
}
type TransactionCIDRecord struct {
CID string `gorm:"column:cid"`
TxHash string `gorm:"primaryKey"`
BlockNumber string `gorm:"primaryKey"`
HeaderID string `gorm:"column:header_id"`
Index int64
Src string
Dst string
MhKey string
IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"`
}
// TableName overrides the table name used by TransactionCIDRecord
func (TransactionCIDRecord) TableName() string {
return "eth.transaction_cids"
}
// NewRetriever returns a pointer to a new Retriever which supports the Retriever interface
func NewRetriever(db *sqlx.DB) *Retriever {
gormDB, err := gorm.Open(postgres.New(postgres.Config{
Conn: db,
}), &gorm.Config{})
if err != nil {
log.Error(err)
return nil
}
return &Retriever{
db: db,
gormDB: gormDB,
}
}
// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) {
var blockNumber int64
err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1")
return blockNumber, err
}
// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
func (r *Retriever) RetrieveLastBlockNumber() (int64, error) {
var blockNumber int64
err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1")
return blockNumber, err
}
func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
for i, topicSet := range topics {
if len(topicSet) == 0 {
continue
}
if !first {
pgStr += " AND"
} else {
first = false
}
pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
args = append(args, pq.Array(topicSet))
*id++
}
return pgStr, args
}
func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
if len(rctFilter.LogAddresses) > 0 {
pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
args = append(args, pq.Array(rctFilter.LogAddresses))
*id++
}
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
}
return pgStr, args
}
func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
rctCond := " AND (receipt_cids.tx_id = ANY ( "
logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
if len(rctFilter.LogAddresses) > 0 {
// Filter on log contract addresses if there are any
pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
args = append(args, pq.Array(rctFilter.LogAddresses))
*id++
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
}
pgStr += ")"
// Filter on txHashes if there are any, and we are matching txs
if rctFilter.MatchTxs && len(txHashes) > 0 {
pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
args = append(args, pq.Array(txHashes))
}
pgStr += ")"
} else { // If there are no contract addresses to filter on
// Filter on topics if there are any
if hasTopics(rctFilter.Topics) {
pgStr += rctCond + logQuery
pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
pgStr += ")"
// Filter on txHashes if there are any, and we are matching txs
if rctFilter.MatchTxs && len(txHashes) > 0 {
pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
args = append(args, pq.Array(txHashes))
}
pgStr += ")"
} else if rctFilter.MatchTxs && len(txHashes) > 0 {
// If there are no contract addresses or topics to filter on,
// Filter on txHashes if there are any, and we are matching txs
pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
args = append(args, pq.Array(txHashes))
}
}
return pgStr, args
}
// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
// filter parameters.
func (r *Retriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
args := make([]interface{}, 0, 4)
id := 1
pgStr := RetrieveFilteredGQLLogs
args = append(args, blockHash.String())
id++
if blockNumber != nil {
pgStr += ` AND receipt_cids.block_number = $2`
id++
args = append(args, blockNumber.Int64())
}
pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
pgStr += ` ORDER BY log_cids.index`
logCIDs := make([]LogResult, 0)
err := tx.Select(&logCIDs, pgStr, args...)
if err != nil {
return nil, err
}
return logCIDs, nil
}
// RetrieveFilteredLogs retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided
// filter parameters.
func (r *Retriever) RetrieveFilteredLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
log.Debug("retrieving log cids for receipt ids")
args := make([]interface{}, 0, 4)
pgStr := RetrieveFilteredLogs
id := 1
if blockNumber > 0 {
pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
args = append(args, blockNumber)
id++
}
if blockHash != nil {
pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
args = append(args, blockHash.String())
id++
}
pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
pgStr += ` ORDER BY log_cids.index`
logCIDs := make([]LogResult, 0)
err := tx.Select(&logCIDs, pgStr, args...)
if err != nil {
return nil, err
}
return logCIDs, nil
}
func hasTopics(topics [][]string) bool {
for _, topicSet := range topics {
if len(topicSet) > 0 {
return true
}
}
return false
}
// RetrieveBlockNumberByHash returns the block number for the given block hash
func (r *Retriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) {
log.Debug("retrieving block number for block hash ", blockHash.String())
pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1`
var blockNumberStr string
if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil {
return 0, err
}
return strconv.ParseUint(blockNumberStr, 10, 64)
}
// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
var headerCIDs []HeaderCIDRecord
// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
// Will use join for TransactionCIDs once preload for 1:N is supported.
err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
}).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error
if err != nil {
log.Error("header cid retrieval error")
return nil, err
}
return headerCIDs, nil
}
// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
var headerCIDs []HeaderCIDRecord
conditions := map[string]interface{}{"block_hash": blockHash.String()}
if blockNumber != nil {
conditions["header_cids.block_number"] = blockNumber.Int64()
}
// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
// Will use join for TransactionCIDs once preload for 1:N is supported.
err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
}).Joins("IPLD").Find(&headerCIDs, conditions).Error
if err != nil {
log.Error("header cid retrieval error")
return HeaderCIDRecord{}, err
}
if len(headerCIDs) == 0 {
return HeaderCIDRecord{}, errHeaderHashNotFound
} else if len(headerCIDs) > 1 {
return HeaderCIDRecord{}, errMultipleHeadersForHash
}
return headerCIDs[0], nil
}
// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
func (r *Retriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
log.Debug("retrieving tx cid for tx hash ", txHash)
var txCIDs []TransactionCIDRecord
var err error
if blockNumber != nil {
err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
} else {
err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
}
if err != nil {
log.Error("tx retrieval error")
return TransactionCIDRecord{}, err
}
if len(txCIDs) == 0 {
return TransactionCIDRecord{}, errTxHashNotFound
} else if len(txCIDs) > 1 {
// a transaction can be part of a only one canonical block
return TransactionCIDRecord{}, errTxHashInMultipleBlocks
}
return txCIDs[0], nil
}
var EmptyNodeValue = make([]byte, common.HashLength)
// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
func (r *Retriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
headerResult := new(ipldResult)
return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
}
// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block)
func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
uncleResults := make([]ipldResult, 0)
if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil {
return nil, nil, err
}
cids := make([]string, len(uncleResults))
uncles := make([][]byte, len(uncleResults))
for i, res := range uncleResults {
cids[i] = res.CID
uncles[i] = res.Data
}
return cids, uncles, nil
}
// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block)
func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
uncleResults := make([]ipldResult, 0)
if err := tx.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
return nil, nil, err
}
cids := make([]string, len(uncleResults))
uncles := make([][]byte, len(uncleResults))
for i, res := range uncleResults {
cids[i] = res.CID
uncles[i] = res.Data
}
return cids, uncles, nil
}
// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number
func (r *Retriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
txResults := make([]ipldResult, 0)
if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil {
return nil, nil, err
}
cids := make([]string, len(txResults))
txs := make([][]byte, len(txResults))
for i, res := range txResults {
cids[i] = res.CID
txs[i] = res.Data
}
return cids, txs, nil
}
// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
func (r *Retriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
txResults := make([]ipldResult, 0)
if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
return nil, nil, err
}
cids := make([]string, len(txResults))
txs := make([][]byte, len(txResults))
for i, res := range txResults {
cids[i] = res.CID
txs[i] = res.Data
}
return cids, txs, nil
}
// DecodeLeafNode decodes the leaf node data
func DecodeLeafNode(node []byte) ([]byte, error) {
var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
return nil, err
}
ty, err := trie_helpers.CheckKeyType(nodeElements)
if err != nil {
return nil, err
}
if ty != sdtypes.Leaf {
return nil, fmt.Errorf("expected leaf node but found %s", ty)
}
return nodeElements[1].([]byte), nil
}
// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) {
rctResults := make([]rctIpldResult, 0)
if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil {
return nil, nil, nil, err
}
cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults))
txs := make([]common.Hash, len(rctResults))
for i, res := range rctResults {
cids[i] = res.LeafCID
nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, nil, err
}
rcts[i] = nodeVal
txs[i] = common.HexToHash(res.TxHash)
}
return cids, rcts, txs, nil
}
// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
rctResults := make([]rctIpldResult, 0)
if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
return nil, nil, nil, err
}
cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults))
txs := make([]common.Hash, len(rctResults))
for i, res := range rctResults {
cids[i] = res.LeafCID
nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, nil, err
}
rcts[i] = nodeVal
txs[i] = common.HexToHash(res.TxHash)
}
return cids, rcts, txs, nil
}
// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
// TODO: ensure this handles deleted accounts appropriately
func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
accountResult := new(nodeInfo)
leafKey := crypto.Keccak256Hash(address.Bytes())
if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
return "", nil, err
}
if accountResult.NodeType == sdtypes.Removed.Int() {
return "", EmptyNodeValue, nil
}
blockNumber, err := strconv.ParseUint(accountResult.BlockNumber, 10, 64)
if err != nil {
return "", nil, err
}
accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.CID, blockNumber)
if err != nil {
return "", nil, err
}
var i []interface{}
if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
}
if len(i) != 2 {
return "", nil, fmt.Errorf("eth Retriever expected state leaf node rlp to decode into two elements")
}
return accountResult.CID, i[1].([]byte), nil
}
// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash
func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) {
storageResult := new(nodeInfo)
stateLeafKey := crypto.Keccak256Hash(address.Bytes())
storageHash := crypto.Keccak256Hash(key.Bytes())
if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
return "", nil, nil, err
}
if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() {
return "", EmptyNodeValue, EmptyNodeValue, nil
}
blockNumber, err := strconv.ParseUint(storageResult.BlockNumber, 10, 64)
if err != nil {
return "", nil, nil, err
}
storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.CID, blockNumber)
if err != nil {
return "", nil, nil, err
}
var i []interface{}
if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
return "", nil, nil, err
}
if len(i) != 2 {
return "", nil, nil, fmt.Errorf("eth Retriever expected storage leaf node rlp to decode into two elements")
}
return storageResult.CID, storageResult.Data, i[1].([]byte), nil
}

4
pkg/eth/cid_retriever_test.go → pkg/eth/retriever_test.go
View File

@ -33,13 +33,13 @@ var _ = Describe("Retriever", func() {
var ( var (
db *sqlx.DB db *sqlx.DB
diffIndexer interfaces.StateDiffIndexer diffIndexer interfaces.StateDiffIndexer
retriever *eth.CIDRetriever retriever *eth.Retriever
) )
BeforeEach(func() { BeforeEach(func() {
db = shared.SetupDB() db = shared.SetupDB()
diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash()) diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
retriever = eth.NewCIDRetriever(db) retriever = eth.NewRetriever(db)
}) })
AfterEach(func() { AfterEach(func() {
shared.TearDownDB(db) shared.TearDownDB(db)

300
pkg/eth/test_helpers/test_data.go
View File

@ -17,28 +17,22 @@
package test_helpers package test_helpers
import ( import (
"bytes"
"crypto/ecdsa" "crypto/ecdsa"
"crypto/elliptic" "crypto/elliptic"
"crypto/rand" "crypto/rand"
"math/big" "math/big"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers" testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types" sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
blocks "github.com/ipfs/go-block-format"
"github.com/multiformats/go-multihash"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
) )
// Test variables // Test variables
@ -75,9 +69,7 @@ var (
Extra: []byte{}, Extra: []byte{},
}, },
} }
ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts)
MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
MockChildHeader = types.Header{ MockChildHeader = types.Header{
Time: 0, Time: 0,
Number: new(big.Int).Add(BlockNumber, common.Big1), Number: new(big.Int).Add(BlockNumber, common.Big1),
@ -89,7 +81,6 @@ var (
ParentHash: MockBlock.Header().Hash(), ParentHash: MockBlock.Header().Hash(),
} }
MockChild = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) MockChild = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
MockChildRlp, _ = rlp.EncodeToBytes(MockChild.Header())
Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594") AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
@ -157,40 +148,9 @@ var (
Index: 5, Index: 5,
} }
Tx1 = GetTxnRlp(0, MockTransactions) rctCIDs, _, _ = GetRctLeafNodeData(MockReceipts)
Tx2 = GetTxnRlp(1, MockTransactions)
Tx3 = GetTxnRlp(2, MockTransactions)
Tx4 = GetTxnRlp(3, MockTransactions)
rctCIDs, rctIPLDData, _ = GetRctLeafNodeData(MockReceipts)
HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID)
Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID)
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID)
Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID)
Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID)
Rct1CID = rctCIDs[0] Rct1CID = rctCIDs[0]
Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID)
Rct2CID = rctCIDs[1]
Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID)
Rct3CID = rctCIDs[2]
Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID)
Rct4CID = rctCIDs[3] Rct4CID = rctCIDs[3]
Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
State1MhKey = shared.MultihashKeyFromCID(State1CID)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
State2MhKey = shared.MultihashKeyFromCID(State2CID)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
StorageMhKey = shared.MultihashKeyFromCID(StorageCID)
Rct1IPLD = rctIPLDData[0]
Rct2IPLD = rctIPLDData[1]
Rct3IPLD = rctIPLDData[2]
Rct4IPLD = rctIPLDData[3]
MockTrxMeta = []models.TxModel{ MockTrxMeta = []models.TxModel{
{ {
CID: "", // This is empty until we go to publish to ipfs CID: "", // This is empty until we go to publish to ipfs
@ -229,48 +189,6 @@ var (
Data: []byte{}, Data: []byte{},
}, },
} }
MockTrxMetaPostPublsh = []models.TxModel{
{
BlockNumber: "1",
CID: Trx1CID.String(), // This is empty until we go to publish to ipfs
MhKey: Trx1MhKey,
Src: SenderAddr.Hex(),
Dst: Address.String(),
Index: 0,
TxHash: MockTransactions[0].Hash().String(),
Data: []byte{},
},
{
BlockNumber: "1",
CID: Trx2CID.String(),
MhKey: Trx2MhKey,
Src: SenderAddr.Hex(),
Dst: AnotherAddress.String(),
Index: 1,
TxHash: MockTransactions[1].Hash().String(),
Data: []byte{},
},
{
BlockNumber: "1",
CID: Trx3CID.String(),
MhKey: Trx3MhKey,
Src: SenderAddr.Hex(),
Dst: "",
Index: 2,
TxHash: MockTransactions[2].Hash().String(),
Data: MockContractByteCode,
},
{
BlockNumber: "1",
CID: Trx4CID.String(),
MhKey: Trx4MhKey,
Src: SenderAddr.Hex(),
Dst: AnotherAddress1.String(),
Index: 3,
TxHash: MockTransactions[3].Hash().String(),
Data: []byte{},
},
}
MockRctMeta = []models.ReceiptModel{ MockRctMeta = []models.ReceiptModel{
{ {
LeafCID: "", LeafCID: "",
@ -298,41 +216,6 @@ var (
}, },
} }
MockRctMetaPostPublish = []models.ReceiptModel{
{
BlockNumber: "1",
HeaderID: MockBlock.Hash().String(),
LeafCID: Rct1CID.String(),
LeafMhKey: Rct1MhKey,
Contract: "",
ContractHash: "",
},
{
BlockNumber: "1",
HeaderID: MockBlock.Hash().String(),
LeafCID: Rct2CID.String(),
LeafMhKey: Rct2MhKey,
Contract: "",
ContractHash: "",
},
{
BlockNumber: "1",
HeaderID: MockBlock.Hash().String(),
LeafCID: Rct3CID.String(),
LeafMhKey: Rct3MhKey,
Contract: ContractAddress.String(),
ContractHash: ContractHash,
},
{
BlockNumber: "1",
HeaderID: MockBlock.Hash().String(),
LeafCID: Rct4CID.String(),
LeafMhKey: Rct4MhKey,
Contract: "",
ContractHash: "",
},
}
// statediff data // statediff data
storageLocation = common.HexToHash("0") storageLocation = common.HexToHash("0")
StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes()
@ -401,24 +284,6 @@ var (
StorageNodes: []sdtypes.StorageNode{}, StorageNodes: []sdtypes.StorageNode{},
}, },
} }
MockStateMetaPostPublish = []models.StateNodeModel{
{
BlockNumber: "1",
CID: State1CID.String(),
MhKey: State1MhKey,
Path: []byte{'\x06'},
NodeType: 2,
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
},
{
BlockNumber: "1",
CID: State2CID.String(),
MhKey: State2MhKey,
Path: []byte{'\x0c'},
NodeType: 2,
StateKey: common.BytesToHash(AccountLeafKey).Hex(),
},
}
MockStorageNodes = map[string][]sdtypes.StorageNode{ MockStorageNodes = map[string][]sdtypes.StorageNode{
contractPath: { contractPath: {
{ {
@ -439,149 +304,6 @@ var (
StorageNodes: MockStorageNodes, StorageNodes: MockStorageNodes,
StateNodes: MockStateNodes, StateNodes: MockStateNodes,
} }
MockConvertedPayloadForChild = eth.ConvertedPayload{
TotalDifficulty: MockChild.Difficulty(),
Block: MockChild,
Receipts: MockReceipts,
TxMetaData: MockTrxMeta,
ReceiptMetaData: MockRctMeta,
StorageNodes: MockStorageNodes,
StateNodes: MockStateNodes,
}
Reward = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
MockCIDWrapper = &eth.CIDWrapper{
BlockNumber: new(big.Int).Set(BlockNumber),
Header: models.HeaderModel{
BlockNumber: "1",
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
CID: HeaderCID.String(),
MhKey: HeaderMhKey,
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: Reward.String(),
StateRoot: MockBlock.Root().String(),
RctRoot: MockBlock.ReceiptHash().String(),
TxRoot: MockBlock.TxHash().String(),
UncleRoot: MockBlock.UncleHash().String(),
Bloom: MockBlock.Bloom().Bytes(),
Timestamp: MockBlock.Time(),
TimesValidated: 1,
Coinbase: "0x0000000000000000000000000000000000000000",
},
Transactions: MockTrxMetaPostPublsh,
Receipts: MockRctMetaPostPublish,
Uncles: []models.UncleModel{},
StateNodes: MockStateMetaPostPublish,
StorageNodes: []models.StorageNodeWithStateKeyModel{
{
BlockNumber: "1",
Path: []byte{},
CID: StorageCID.String(),
MhKey: StorageMhKey,
NodeType: 2,
StateKey: common.BytesToHash(ContractLeafKey).Hex(),
StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
},
},
}
HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
Trx1IPLD, _ = blocks.NewBlockWithCid(Tx1, Trx1CID)
Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID)
Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID)
Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID)
State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
MockIPLDs = eth.IPLDs{
BlockNumber: new(big.Int).Set(BlockNumber),
Header: models.IPLDModel{
BlockNumber: BlockNumber.String(),
Data: HeaderIPLD.RawData(),
Key: HeaderIPLD.Cid().String(),
},
Transactions: []models.IPLDModel{
{
BlockNumber: BlockNumber.String(),
Data: Trx1IPLD.RawData(),
Key: Trx1IPLD.Cid().String(),
},
{
BlockNumber: BlockNumber.String(),
Data: Trx2IPLD.RawData(),
Key: Trx2IPLD.Cid().String(),
},
{
BlockNumber: BlockNumber.String(),
Data: Trx3IPLD.RawData(),
Key: Trx3IPLD.Cid().String(),
},
{
BlockNumber: BlockNumber.String(),
Data: Trx4IPLD.RawData(),
Key: Trx4IPLD.Cid().String(),
},
},
Receipts: []models.IPLDModel{
{
BlockNumber: BlockNumber.String(),
Data: Rct1IPLD,
Key: Rct1CID.String(),
},
{
BlockNumber: BlockNumber.String(),
Data: Rct2IPLD,
Key: Rct2CID.String(),
},
{
BlockNumber: BlockNumber.String(),
Data: Rct3IPLD,
Key: Rct3CID.String(),
},
{
BlockNumber: BlockNumber.String(),
Data: Rct4IPLD,
Key: Rct4CID.String(),
},
},
StateNodes: []eth.StateNode{
{
StateLeafKey: common.BytesToHash(ContractLeafKey),
Type: sdtypes.Leaf,
IPLD: models.IPLDModel{
BlockNumber: BlockNumber.String(),
Data: State1IPLD.RawData(),
Key: State1IPLD.Cid().String(),
},
Path: []byte{'\x06'},
},
{
StateLeafKey: common.BytesToHash(AccountLeafKey),
Type: sdtypes.Leaf,
IPLD: models.IPLDModel{
BlockNumber: BlockNumber.String(),
Data: State2IPLD.RawData(),
Key: State2IPLD.Cid().String(),
},
Path: []byte{'\x0c'},
},
},
StorageNodes: []eth.StorageNode{
{
StateLeafKey: common.BytesToHash(ContractLeafKey),
StorageLeafKey: common.BytesToHash(StorageLeafKey),
Type: sdtypes.Leaf,
IPLD: models.IPLDModel{
BlockNumber: BlockNumber.String(),
Data: StorageIPLD.RawData(),
Key: StorageIPLD.Cid().String(),
},
Path: []byte{},
},
},
}
LondonBlockNum = new(big.Int).Add(BlockNumber, big.NewInt(2)) LondonBlockNum = new(big.Int).Add(BlockNumber, big.NewInt(2))
MockLondonHeader = types.Header{ MockLondonHeader = types.Header{
@ -741,21 +463,3 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts,
return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
} }
func GetTxnRlp(num int, txs types.Transactions) []byte {
buf := new(bytes.Buffer)
txs.EncodeIndex(num, buf)
tx := make([]byte, buf.Len())
copy(tx, buf.Bytes())
buf.Reset()
return tx
}
func GetRctRlp(num int, rcts types.Receipts) []byte {
buf := new(bytes.Buffer)
rcts.EncodeIndex(num, buf)
rct := make([]byte, buf.Len())
copy(rct, buf.Bytes())
buf.Reset()
return rct
}

2
pkg/graphql/graphql.go
View File

@ -1009,7 +1009,7 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct {
Contract common.Address Contract common.Address
Slot common.Hash Slot common.Hash
}) (*StorageResult, error) { }) (*StorageResult, error) {
cid, ipldBlock, rlpValue, err := r.backend.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash) cid, ipldBlock, rlpValue, err := r.backend.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash)
if err != nil { if err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {

1
test_config/test_config.go
View File

@ -18,6 +18,7 @@ package test_config
import ( import (
"errors" "errors"
"github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/cerc-io/ipld-eth-server/v4/pkg/log"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/spf13/viper" "github.com/spf13/viper"