diff --git a/.github/workflows/run_unit_test.sh b/.github/workflows/run_unit_test.sh index 63fbb99f..c1c5e182 100755 --- a/.github/workflows/run_unit_test.sh +++ b/.github/workflows/run_unit_test.sh @@ -6,7 +6,7 @@ set -e start_dir=$(pwd) temp_dir=$(mktemp -d) cd $temp_dir -echo "git clone -b $(cat /tmp/git_head_ref) https://github.com/$(cat /tmp/git_repository).git" + git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git" cd ipld-eth-server @@ -17,7 +17,7 @@ rm -f /tmp/git_head_ref /tmp/git_repository echo 'docker-compose up -d migrations ipld-eth-db' docker-compose up -d migrations ipld-eth-db trap "docker-compose down -v --remove-orphans; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR -sleep 30 +sleep 60 # Remove old logs so there's no confusion, then run test rm -f /tmp/test.log /tmp/return_test.txt diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 7e55877c..24d86e54 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "strconv" "time" validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" @@ -364,8 +365,10 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo return nil, err } + blockNumber := header.Number.Uint64() + // Fetch uncles - uncles, err := b.GetUnclesByBlockHash(tx, hash) + uncles, err := b.GetUnclesByBlockHashAndNumber(tx, hash, blockNumber) if err != nil && err != sql.ErrNoRows { log.Error("error fetching uncles: ", err) return nil, err @@ -389,14 +392,14 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo } // Fetch transactions - transactions, err := b.GetTransactionsByBlockHash(tx, hash) + transactions, err := b.GetTransactionsByBlockHashAndNumber(tx, hash, blockNumber) if err != nil && err != sql.ErrNoRows { log.Error("error fetching transactions: ", err) return nil, err } // Fetch receipts - receipts, err := b.GetReceiptsByBlockHash(tx, hash) + receipts, err := b.GetReceiptsByBlockHashAndNumber(tx, hash, blockNumber) if err != nil && err != sql.ErrNoRows { log.Error("error fetching receipts: ", err) return nil, err @@ -438,6 +441,27 @@ func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types. return uncles, nil } +// GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number +func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) { + _, uncleBytes, err := b.IPLDRetriever.RetrieveUncles(tx, hash, number) + if err != nil { + return nil, err + } + + uncles := make([]*types.Header, len(uncleBytes)) + for i, bytes := range uncleBytes { + var uncle types.Header + err = rlp.DecodeBytes(bytes, &uncle) + if err != nil { + return nil, err + } + + uncles[i] = &uncle + } + + return uncles, nil +} + // GetTransactionsByBlockHash retrieves transactions for a provided block hash func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) { _, transactionBytes, err := b.IPLDRetriever.RetrieveTransactionsByBlockHash(tx, hash) @@ -458,6 +482,26 @@ func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (typ return txs, nil } +// GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number +func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) { + _, transactionBytes, err := b.IPLDRetriever.RetrieveTransactions(tx, hash, number) + if err != nil { + return nil, err + } + + txs := make(types.Transactions, len(transactionBytes)) + for i, txBytes := range transactionBytes { + var tx types.Transaction + if err := tx.UnmarshalBinary(txBytes); err != nil { + return nil, err + } + + txs[i] = &tx + } + + return txs, nil +} + // GetReceiptsByBlockHash retrieves receipts for a provided block hash func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) { _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash) @@ -476,6 +520,24 @@ func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.R return rcts, nil } +// GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number +func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) { + _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) + if err != nil { + return nil, err + } + rcts := make(types.Receipts, len(receiptBytes)) + for i, rctBytes := range receiptBytes { + rct := new(types.Receipt) + if err := rct.UnmarshalBinary(rctBytes); err != nil { + return nil, err + } + rct.TxHash = txs[i] + rcts[i] = rct + } + return rcts, nil +} + // GetTransaction retrieves a tx by hash // It also returns the blockhash, blocknumber, and tx index associated with the transaction func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { @@ -523,20 +585,13 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece } }() - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash) + headerCID, err := b.Retriever.RetrieveHeaderCIDByHash(tx, hash) if err != nil { return nil, err } - rcts := make(types.Receipts, len(receiptBytes)) - for i, rctBytes := range receiptBytes { - rct := new(types.Receipt) - if err := rct.UnmarshalBinary(rctBytes); err != nil { - return nil, err - } - rct.TxHash = txs[i] - rcts[i] = rct - } - return rcts, nil + blockNumber, _ := strconv.ParseUint(string(headerCID.BlockNumber), 10, 64) + + return b.GetReceiptsByBlockHashAndNumber(tx, hash, blockNumber) } // GetLogs returns all the logs for the given block hash @@ -557,7 +612,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64) } }() - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash) + _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) if err != nil { return nil, err } diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index d2f12ecf..1ef56aae 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -360,7 +360,7 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte // RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided // filter parameters. -func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) { +func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) { log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String()) args := make([]interface{}, 0, 4) id := 1 @@ -379,6 +379,12 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF args = append(args, blockHash.String()) id++ + if blockNumber != nil { + pgStr += ` AND receipt_cids.block_number = $2` + id++ + args = append(args, blockNumber.Int64()) + } + pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) pgStr += ` ORDER BY log_cids.index` @@ -706,17 +712,22 @@ func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) return headerCIDs, nil } -// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash) (HeaderCIDRecord, error) { +// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number) +func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) { log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String()) var headerCIDs []HeaderCIDRecord + conditions := map[string]interface{}{"block_hash": blockHash.String()} + if blockNumber != nil { + conditions["header_cids.block_number"] = blockNumber.Int64() + } + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 // Will use join for TransactionCIDs once preload for 1:N is supported. err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") - }).Joins("IPLD").Find(&headerCIDs, "block_hash = ?", blockHash.String()).Error + }).Joins("IPLD").Find(&headerCIDs, conditions).Error if err != nil { log.Error("header cid retrieval error") @@ -732,15 +743,20 @@ func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Has return headerCIDs[0], nil } -// RetrieveTxCIDByHash returns the tx for the given tx hash -func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string) (TransactionCIDRecord, error) { +// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number) +func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) { log.Debug("retrieving tx cid for tx hash ", txHash) var txCIDs []TransactionCIDRecord - err := ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error + var err error + if blockNumber != nil { + err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error + } else { + err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error + } if err != nil { - log.Error("header cid retrieval error") + log.Error("tx retrieval error") return TransactionCIDRecord{}, err } diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 217ab563..dc18fa68 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -64,6 +64,19 @@ const ( AND uncle_cids.block_number = blocks.block_number ) WHERE block_hash = ANY($1::VARCHAR(66)[])` + RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON ( + uncle_cids.header_id = header_cids.block_hash + AND uncle_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + uncle_cids.mh_key = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) + WHERE header_cids.block_hash = $1 + AND header_cids.block_number = $2 + ORDER BY uncle_cids.parent_hash` RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids INNER JOIN eth.header_cids ON ( @@ -101,6 +114,19 @@ const ( AND transaction_cids.block_number = blocks.block_number ) WHERE tx_hash = ANY($1::VARCHAR(66)[])` + RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + transaction_cids.mh_key = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1 + AND header_cids.block_number = $2 + ORDER BY eth.transaction_cids.index ASC` RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids INNER JOIN eth.header_cids ON ( @@ -146,6 +172,24 @@ const ( ) WHERE tx_hash = ANY($1::VARCHAR(66)[]) AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` + RetrieveReceiptsPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.header_id = transaction_cids.header_id + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + receipt_cids.leaf_mh_key = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1 + AND header_cids.block_number = $2 + ORDER BY eth.transaction_cids.index ASC` RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON ( @@ -338,6 +382,21 @@ func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string, return cids, uncles, nil } +// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block) +func (r *IPLDRetriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + uncleResults := make([]ipldResult, 0) + if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(uncleResults)) + uncles := make([][]byte, len(uncleResults)) + for i, res := range uncleResults { + cids[i] = res.CID + uncles[i] = res.Data + } + return cids, uncles, nil +} + // RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block) func (r *IPLDRetriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { uncleResults := make([]ipldResult, 0) @@ -393,6 +452,21 @@ func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]st return cids, txs, nil } +// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number +func (r *IPLDRetriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + txResults := make([]ipldResult, 0) + if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(txResults)) + txs := make([][]byte, len(txResults)) + for i, res := range txResults { + cids[i] = res.CID + txs[i] = res.Data + } + return cids, txs, nil +} + // RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { txResults := make([]ipldResult, 0) @@ -469,6 +543,30 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri return cids, rcts, nil } +// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number. +// cid returned corresponds to the leaf node data which contains the receipt. +func (r *IPLDRetriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) { + rctResults := make([]rctIpldResult, 0) + if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil { + return nil, nil, nil, err + } + cids := make([]string, len(rctResults)) + rcts := make([][]byte, len(rctResults)) + txs := make([]common.Hash, len(rctResults)) + + for i, res := range rctResults { + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, nil, err + } + rcts[i] = nodeVal + txs[i] = common.HexToHash(res.TxHash) + } + + return cids, rcts, txs, nil +} + // RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash. // cid returned corresponds to the leaf node data which contains the receipt. func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) { diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index 97282037..afcbd0f3 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -1036,8 +1036,9 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct { } func (r *Resolver) GetLogs(ctx context.Context, args struct { - BlockHash common.Hash - Addresses *[]common.Address + BlockHash common.Hash + BlockNumber *BigInt + Addresses *[]common.Address }) (*[]*Log, error) { var filter eth.ReceiptFilter @@ -1054,7 +1055,7 @@ func (r *Resolver) GetLogs(ctx context.Context, args struct { return nil, err } - filteredLogs, err := r.backend.Retriever.RetrieveFilteredGQLLogs(tx, filter, &args.BlockHash) + filteredLogs, err := r.backend.Retriever.RetrieveFilteredGQLLogs(tx, filter, &args.BlockHash, args.BlockNumber.ToInt()) if err != nil { return nil, err } @@ -1271,7 +1272,7 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct { var headerCIDs []eth.HeaderCIDRecord var err error if args.Condition.BlockHash != nil { - headerCID, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash)) + headerCID, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash), args.Condition.BlockNumber.ToInt()) if err != nil { if !strings.Contains(err.Error(), "not found") { return nil, err @@ -1352,9 +1353,12 @@ func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct { } func (r *Resolver) EthTransactionCidByTxHash(ctx context.Context, args struct { - TxHash string + TxHash string + BlockNumber *BigInt }) (*EthTransactionCID, error) { - txCID, err := r.backend.Retriever.RetrieveTxCIDByHash(args.TxHash) + // Need not check args.BlockNumber for nil as .ToInt() uses a pointer receiver and returns nil if BlockNumber is nil + // https://stackoverflow.com/questions/42238624/calling-a-method-on-a-nil-struct-pointer-doesnt-panic-why-not + txCID, err := r.backend.Retriever.RetrieveTxCIDByHash(args.TxHash, args.BlockNumber.ToInt()) if err != nil { return nil, err diff --git a/pkg/graphql/graphql_test.go b/pkg/graphql/graphql_test.go index 258d6afe..7dfc3185 100644 --- a/pkg/graphql/graphql_test.go +++ b/pkg/graphql/graphql_test.go @@ -296,7 +296,7 @@ var _ = Describe("GraphQL", func() { allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockHash: &blockHash}) Expect(err).ToNot(HaveOccurred()) - headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash()) + headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash(), nil) Expect(err).ToNot(HaveOccurred()) Expect(len(allEthHeaderCIDsResp.Nodes)).To(Equal(1)) @@ -311,7 +311,7 @@ var _ = Describe("GraphQL", func() { ethTransactionCIDResp, err := client.EthTransactionCIDByTxHash(ctx, txHash) Expect(err).ToNot(HaveOccurred()) - txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash) + txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash, nil) Expect(err).ToNot(HaveOccurred()) compareEthTxCID(*ethTransactionCIDResp, txCID) diff --git a/pkg/graphql/schema.go b/pkg/graphql/schema.go index d07d311b..14c5eb0e 100644 --- a/pkg/graphql/schema.go +++ b/pkg/graphql/schema.go @@ -343,12 +343,12 @@ const schema string = ` getStorageAt(blockHash: Bytes32!, contract: Address!, slot: Bytes32!): StorageResult # Get contract logs by block hash and contract address. - getLogs(blockHash: Bytes32!, addresses: [Address!]): [Log!] + getLogs(blockHash: Bytes32!, blockNumber: BigInt, addresses: [Address!]): [Log!] # PostGraphile alternative to get headers with transactions using block number or block hash. allEthHeaderCids(condition: EthHeaderCidCondition): EthHeaderCidsConnection # PostGraphile alternative to get transactions using transaction hash. - ethTransactionCidByTxHash(txHash: String!): EthTransactionCid + ethTransactionCidByTxHash(txHash: String!, blockNumber: BigInt): EthTransactionCid } `