cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Optimize GQL and getBlock APIs #201

Merged
prathamesh0 merged 8 commits from pm-optimize-quries into v4 2022-11-04 04:32:09 +00:00
Conversation 0 Commits 8 Files Changed 7 +208 -35
7 changed files with 208 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/run_unit_test.sh vendored
View File

@ -6,7 +6,7 @@ set -e
start_dir=$(pwd)
temp_dir=$(mktemp -d)
cd $temp_dir
echo "git clone -b $(cat /tmp/git_head_ref) https://github.com/$(cat /tmp/git_repository).git"
git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git"
cd ipld-eth-server
@ -17,7 +17,7 @@ rm -f /tmp/git_head_ref /tmp/git_repository
echo 'docker-compose up -d migrations ipld-eth-db'
docker-compose up -d migrations ipld-eth-db
trap "docker-compose down -v --remove-orphans; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR
sleep 30
sleep 60
# Remove old logs so there's no confusion, then run test
rm -f /tmp/test.log /tmp/return_test.txt

85
pkg/eth/backend.go
View File

@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"math/big"
"strconv"
"time"
validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg"
@ -364,8 +365,10 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
return nil, err
}
blockNumber := header.Number.Uint64()
// Fetch uncles
uncles, err := b.GetUnclesByBlockHash(tx, hash)
uncles, err := b.GetUnclesByBlockHashAndNumber(tx, hash, blockNumber)
if err != nil && err != sql.ErrNoRows {
log.Error("error fetching uncles: ", err)
return nil, err
@ -389,14 +392,14 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
// Fetch transactions
transactions, err := b.GetTransactionsByBlockHash(tx, hash)
transactions, err := b.GetTransactionsByBlockHashAndNumber(tx, hash, blockNumber)
if err != nil && err != sql.ErrNoRows {
log.Error("error fetching transactions: ", err)
return nil, err
}
// Fetch receipts
receipts, err := b.GetReceiptsByBlockHash(tx, hash)
receipts, err := b.GetReceiptsByBlockHashAndNumber(tx, hash, blockNumber)
if err != nil && err != sql.ErrNoRows {
log.Error("error fetching receipts: ", err)
return nil, err
@ -438,6 +441,27 @@ func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.
return uncles, nil
}
// GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number
func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) {
_, uncleBytes, err := b.IPLDRetriever.RetrieveUncles(tx, hash, number)
if err != nil {
return nil, err
}
uncles := make([]*types.Header, len(uncleBytes))
for i, bytes := range uncleBytes {
var uncle types.Header
err = rlp.DecodeBytes(bytes, &uncle)
if err != nil {
return nil, err
}
uncles[i] = &uncle
}
return uncles, nil
}
// GetTransactionsByBlockHash retrieves transactions for a provided block hash
func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) {
_, transactionBytes, err := b.IPLDRetriever.RetrieveTransactionsByBlockHash(tx, hash)
@ -458,6 +482,26 @@ func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (typ
return txs, nil
}
// GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number
func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) {
_, transactionBytes, err := b.IPLDRetriever.RetrieveTransactions(tx, hash, number)
if err != nil {
return nil, err
}
txs := make(types.Transactions, len(transactionBytes))
for i, txBytes := range transactionBytes {
var tx types.Transaction
if err := tx.UnmarshalBinary(txBytes); err != nil {
return nil, err
}
txs[i] = &tx
}
return txs, nil
}
// GetReceiptsByBlockHash retrieves receipts for a provided block hash
func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) {
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash)
@ -476,6 +520,24 @@ func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.R
return rcts, nil
}
// GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number
func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) {
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number)
if err != nil {
return nil, err
}
rcts := make(types.Receipts, len(receiptBytes))
for i, rctBytes := range receiptBytes {
rct := new(types.Receipt)
if err := rct.UnmarshalBinary(rctBytes); err != nil {
return nil, err
}
rct.TxHash = txs[i]
rcts[i] = rct
}
return rcts, nil
}
// GetTransaction retrieves a tx by hash
// It also returns the blockhash, blocknumber, and tx index associated with the transaction
func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) {
@ -523,20 +585,13 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece
}
}()
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash)
headerCID, err := b.Retriever.RetrieveHeaderCIDByHash(tx, hash)
if err != nil {
return nil, err
}
rcts := make(types.Receipts, len(receiptBytes))
for i, rctBytes := range receiptBytes {
rct := new(types.Receipt)
if err := rct.UnmarshalBinary(rctBytes); err != nil {
return nil, err
}
rct.TxHash = txs[i]
rcts[i] = rct
}
return rcts, nil
blockNumber, _ := strconv.ParseUint(string(headerCID.BlockNumber), 10, 64)
return b.GetReceiptsByBlockHashAndNumber(tx, hash, blockNumber)
}
// GetLogs returns all the logs for the given block hash
@ -557,7 +612,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64)
}
}()
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash)
_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number)
if err != nil {
return nil, err
}

32
pkg/eth/cid_retriever.go
View File

@ -360,7 +360,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte
// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
// filter parameters.
func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) {
func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
args := make([]interface{}, 0, 4)
id := 1
@ -379,6 +379,12 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF
args = append(args, blockHash.String())
id++
if blockNumber != nil {
pgStr += ` AND receipt_cids.block_number = $2`
id++
args = append(args, blockNumber.Int64())
}
pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
pgStr += ` ORDER BY log_cids.index`
@ -706,17 +712,22 @@ func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64)
return headerCIDs, nil
}
// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash
func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash) (HeaderCIDRecord, error) {
// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
var headerCIDs []HeaderCIDRecord
conditions := map[string]interface{}{"block_hash": blockHash.String()}
if blockNumber != nil {
conditions["header_cids.block_number"] = blockNumber.Int64()
}
// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
// Will use join for TransactionCIDs once preload for 1:N is supported.
err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
}).Joins("IPLD").Find(&headerCIDs, "block_hash = ?", blockHash.String()).Error
}).Joins("IPLD").Find(&headerCIDs, conditions).Error
if err != nil {
log.Error("header cid retrieval error")
@ -732,15 +743,20 @@ func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Has
return headerCIDs[0], nil
}
// RetrieveTxCIDByHash returns the tx for the given tx hash
func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string) (TransactionCIDRecord, error) {
// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
log.Debug("retrieving tx cid for tx hash ", txHash)
var txCIDs []TransactionCIDRecord
err := ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
var err error
if blockNumber != nil {
err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
} else {
err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
}
if err != nil {
log.Error("header cid retrieval error")
log.Error("tx retrieval error")
return TransactionCIDRecord{}, err
}

98
pkg/eth/ipld_retriever.go
View File

@ -64,6 +64,19 @@ const (
AND uncle_cids.block_number = blocks.block_number
)
WHERE block_hash = ANY($1::VARCHAR(66)[])`
RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data
FROM eth.uncle_cids
INNER JOIN eth.header_cids ON (
uncle_cids.header_id = header_cids.block_hash
AND uncle_cids.block_number = header_cids.block_number
)
INNER JOIN public.blocks ON (
uncle_cids.mh_key = blocks.key
AND uncle_cids.block_number = blocks.block_number
)
WHERE header_cids.block_hash = $1
AND header_cids.block_number = $2
ORDER BY uncle_cids.parent_hash`
RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
FROM eth.uncle_cids
INNER JOIN eth.header_cids ON (
@ -101,6 +114,19 @@ const (
AND transaction_cids.block_number = blocks.block_number
)
WHERE tx_hash = ANY($1::VARCHAR(66)[])`
RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data
FROM eth.transaction_cids
INNER JOIN eth.header_cids ON (
transaction_cids.header_id = header_cids.block_hash
AND transaction_cids.block_number = header_cids.block_number
)
INNER JOIN public.blocks ON (
transaction_cids.mh_key = blocks.key
AND transaction_cids.block_number = blocks.block_number
)
WHERE block_hash = $1
AND header_cids.block_number = $2
ORDER BY eth.transaction_cids.index ASC`
RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
FROM eth.transaction_cids
INNER JOIN eth.header_cids ON (
@ -146,6 +172,24 @@ const (
)
WHERE tx_hash = ANY($1::VARCHAR(66)[])
AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))`
RetrieveReceiptsPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (
receipt_cids.tx_id = transaction_cids.tx_hash
AND receipt_cids.header_id = transaction_cids.header_id
AND receipt_cids.block_number = transaction_cids.block_number
)
INNER JOIN eth.header_cids ON (
transaction_cids.header_id = header_cids.block_hash
AND transaction_cids.block_number = header_cids.block_number
)
INNER JOIN public.blocks ON (
receipt_cids.leaf_mh_key = blocks.key
AND receipt_cids.block_number = blocks.block_number
)
WHERE block_hash = $1
AND header_cids.block_number = $2
ORDER BY eth.transaction_cids.index ASC`
RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
FROM eth.receipt_cids
INNER JOIN eth.transaction_cids ON (
@ -338,6 +382,21 @@ func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string,
return cids, uncles, nil
}
// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block)
func (r *IPLDRetriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
uncleResults := make([]ipldResult, 0)
if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil {
return nil, nil, err
}
cids := make([]string, len(uncleResults))
uncles := make([][]byte, len(uncleResults))
for i, res := range uncleResults {
cids[i] = res.CID
uncles[i] = res.Data
}
return cids, uncles, nil
}
// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block)
func (r *IPLDRetriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
uncleResults := make([]ipldResult, 0)
@ -393,6 +452,21 @@ func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]st
return cids, txs, nil
}
// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number
func (r *IPLDRetriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
txResults := make([]ipldResult, 0)
if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil {
return nil, nil, err
}
cids := make([]string, len(txResults))
txs := make([][]byte, len(txResults))
for i, res := range txResults {
cids[i] = res.CID
txs[i] = res.Data
}
return cids, txs, nil
}
// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
txResults := make([]ipldResult, 0)
@ -469,6 +543,30 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
return cids, rcts, nil
}
// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *IPLDRetriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) {
rctResults := make([]rctIpldResult, 0)
if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil {
return nil, nil, nil, err
}
cids := make([]string, len(rctResults))
rcts := make([][]byte, len(rctResults))
txs := make([]common.Hash, len(rctResults))
for i, res := range rctResults {
cids[i] = res.LeafCID
nodeVal, err := DecodeLeafNode(res.Data)
if err != nil {
return nil, nil, nil, err
}
rcts[i] = nodeVal
txs[i] = common.HexToHash(res.TxHash)
}
return cids, rcts, txs, nil
}
// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
// cid returned corresponds to the leaf node data which contains the receipt.
func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) {

10
pkg/graphql/graphql.go
View File

@ -1037,6 +1037,7 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct {
func (r *Resolver) GetLogs(ctx context.Context, args struct {
BlockHash common.Hash
BlockNumber *BigInt
Addresses *[]common.Address
}) (*[]*Log, error) {
var filter eth.ReceiptFilter
@ -1054,7 +1055,7 @@ func (r *Resolver) GetLogs(ctx context.Context, args struct {
return nil, err
}
filteredLogs, err := r.backend.Retriever.RetrieveFilteredGQLLogs(tx, filter, &args.BlockHash)
filteredLogs, err := r.backend.Retriever.RetrieveFilteredGQLLogs(tx, filter, &args.BlockHash, args.BlockNumber.ToInt())
if err != nil {
return nil, err
}
@ -1271,7 +1272,7 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct {
var headerCIDs []eth.HeaderCIDRecord
var err error
if args.Condition.BlockHash != nil {
headerCID, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash))
headerCID, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash), args.Condition.BlockNumber.ToInt())
if err != nil {
if !strings.Contains(err.Error(), "not found") {
return nil, err
@ -1353,8 +1354,11 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct {
func (r *Resolver) EthTransactionCidByTxHash(ctx context.Context, args struct {
TxHash string
BlockNumber *BigInt
}) (*EthTransactionCID, error) {
txCID, err := r.backend.Retriever.RetrieveTxCIDByHash(args.TxHash)
// Need not check args.BlockNumber for nil as .ToInt() uses a pointer receiver and returns nil if BlockNumber is nil
// https://stackoverflow.com/questions/42238624/calling-a-method-on-a-nil-struct-pointer-doesnt-panic-why-not
txCID, err := r.backend.Retriever.RetrieveTxCIDByHash(args.TxHash, args.BlockNumber.ToInt())
if err != nil {
return nil, err

4
pkg/graphql/graphql_test.go
View File

@ -296,7 +296,7 @@ var _ = Describe("GraphQL", func() {
allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockHash: &blockHash})
Expect(err).ToNot(HaveOccurred())
headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash())
headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash(), nil)
Expect(err).ToNot(HaveOccurred())
Expect(len(allEthHeaderCIDsResp.Nodes)).To(Equal(1))
@ -311,7 +311,7 @@ var _ = Describe("GraphQL", func() {
ethTransactionCIDResp, err := client.EthTransactionCIDByTxHash(ctx, txHash)
Expect(err).ToNot(HaveOccurred())
txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash)
txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash, nil)
Expect(err).ToNot(HaveOccurred())
compareEthTxCID(*ethTransactionCIDResp, txCID)

4
pkg/graphql/schema.go
View File

@ -343,12 +343,12 @@ const schema string = `
getStorageAt(blockHash: Bytes32!, contract: Address!, slot: Bytes32!): StorageResult
# Get contract logs by block hash and contract address.
getLogs(blockHash: Bytes32!, addresses: [Address!]): [Log!]
getLogs(blockHash: Bytes32!, blockNumber: BigInt, addresses: [Address!]): [Log!]
# PostGraphile alternative to get headers with transactions using block number or block hash.
allEthHeaderCids(condition: EthHeaderCidCondition): EthHeaderCidsConnection
# PostGraphile alternative to get transactions using transaction hash.
ethTransactionCidByTxHash(txHash: String!): EthTransactionCid
ethTransactionCidByTxHash(txHash: String!, blockNumber: BigInt): EthTransactionCid
}
`