From 87013110f9c37ab44d1dca187aa8ffd4f0f16196 Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 2 Mar 2023 17:41:44 -0600 Subject: [PATCH 01/14] public.blocks => ipld.blocks --- pkg/eth/backend.go | 11 ++++------- pkg/eth/cid_retriever.go | 4 ++-- pkg/eth/ipld_retriever.go | 14 +++++++------- pkg/shared/functions.go | 4 ++-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 6d92f994..c5f554ac 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -61,9 +61,6 @@ var ( errMultipleHeadersForHash = errors.New("more than one headers for the given hash") errTxHashNotFound = errors.New("transaction for hash not found") errTxHashInMultipleBlocks = errors.New("transaction for hash found in more than one canonical block") - - // errMissingSignature is returned if a block's extra-data section doesn't seem - // to contain a 65 byte secp256k1 signature. ) const ( @@ -71,15 +68,15 @@ const ( FROM canonical_header_hash($1) AS block_hash WHERE block_hash IS NOT NULL` RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( header_cids.mh_key = blocks.key AND header_cids.block_number = blocks.block_number ) WHERE block_hash = (SELECT canonical_header_hash($1))` - RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids + RetrieveTD = `SELECT CAST(td as TEXT) FROM eth.header_cids WHERE header_cids.block_hash = $1` RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index - FROM public.blocks, eth.transaction_cids + FROM ipld.blocks, eth.transaction_cids WHERE blocks.key = transaction_cids.mh_key AND blocks.block_number = transaction_cids.block_number AND transaction_cids.tx_hash = $1 @@ -97,7 +94,7 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` + RetrieveCodeByMhKey = `SELECT data FROM ipld.blocks WHERE key = $1` ) const ( diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index b9b943b4..d6ff7ac1 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -41,7 +41,7 @@ type IPLDModelRecord struct { // TableName overrides the table name used by IPLD func (IPLDModelRecord) TableName() string { - return "public.blocks" + return "ipld.blocks" } type HeaderCIDRecord struct { @@ -205,7 +205,7 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash - FROM eth.log_cids, eth.receipt_cids, public.blocks + FROM eth.log_cids, eth.receipt_cids, ipld.blocks WHERE eth.log_cids.rct_id = receipt_cids.tx_id AND eth.log_cids.header_id = receipt_cids.header_id AND eth.log_cids.block_number = receipt_cids.block_number diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 51a12cb4..5d1075d3 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -33,7 +33,7 @@ import ( const ( RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( header_cids.mh_key = blocks.key AND header_cids.block_number = blocks.block_number ) @@ -44,7 +44,7 @@ const ( uncle_cids.header_id = header_cids.block_hash AND uncle_cids.block_number = header_cids.block_number ) - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( uncle_cids.mh_key = blocks.key AND uncle_cids.block_number = blocks.block_number ) @@ -57,7 +57,7 @@ const ( uncle_cids.header_id = header_cids.block_hash AND uncle_cids.block_number = header_cids.block_number ) - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( uncle_cids.mh_key = blocks.key AND uncle_cids.block_number = blocks.block_number ) @@ -69,7 +69,7 @@ const ( transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number ) - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( transaction_cids.mh_key = blocks.key AND transaction_cids.block_number = blocks.block_number ) @@ -82,7 +82,7 @@ const ( transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number ) - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( transaction_cids.mh_key = blocks.key AND transaction_cids.block_number = blocks.block_number ) @@ -99,7 +99,7 @@ const ( transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number ) - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( receipt_cids.leaf_mh_key = blocks.key AND receipt_cids.block_number = blocks.block_number ) @@ -117,7 +117,7 @@ const ( transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number ) - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( receipt_cids.leaf_mh_key = blocks.key AND receipt_cids.block_number = blocks.block_number ) diff --git a/pkg/shared/functions.go b/pkg/shared/functions.go index f5eac06c..d54ddaf1 100644 --- a/pkg/shared/functions.go +++ b/pkg/shared/functions.go @@ -50,14 +50,14 @@ func Rollback(tx *sqlx.Tx) { // FetchIPLDByMhKeyAndBlockNumber is used to retrieve an ipld from Postgres blockstore with the provided tx, mhkey string and blockNumber func FetchIPLDByMhKeyAndBlockNumber(tx *sqlx.Tx, mhKey string, blockNumber uint64) ([]byte, error) { - pgStr := `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` + pgStr := `SELECT data FROM ipld.blocks WHERE key = $1 AND block_number = $2` var block []byte return block, tx.Get(&block, pgStr, mhKey, blockNumber) } // FetchIPLD is used to retrieve an IPLD from Postgres mhkey and blockNumber func FetchIPLD(db *sqlx.DB, mhKey string, blockNumber uint64) ([]byte, error) { - pgStr := `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` + pgStr := `SELECT data FROM ipld.blocks WHERE key = $1 AND block_number = $2` var block []byte return block, db.Get(&block, pgStr, mhKey, blockNumber) } From 907f15f6407b4a52047484ed3239eb828ba9857c Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 2 Mar 2023 17:59:13 -0600 Subject: [PATCH 02/14] mh_key => cid; leaf_cid => cid; refactor GetReceipts --- pkg/eth/backend.go | 12 +++++------- pkg/eth/cid_retriever.go | 28 +++++++++++++++------------- pkg/eth/ipld_retriever.go | 29 ++++++++++++++--------------- pkg/eth/types.go | 2 +- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index c5f554ac..cecd29a0 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "math/big" - "strconv" "time" validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" @@ -68,8 +67,8 @@ const ( FROM canonical_header_hash($1) AS block_hash WHERE block_hash IS NOT NULL` RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids - INNER JOIN ipld.blocks ON ( - header_cids.mh_key = blocks.key + INNER JOIN public.blocks ON ( + header_cids.cid = blocks.key AND header_cids.block_number = blocks.block_number ) WHERE block_hash = (SELECT canonical_header_hash($1))` @@ -77,7 +76,7 @@ const ( WHERE header_cids.block_hash = $1` RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index FROM ipld.blocks, eth.transaction_cids - WHERE blocks.key = transaction_cids.mh_key + WHERE blocks.key = transaction_cids.cid AND blocks.block_number = transaction_cids.block_number AND transaction_cids.tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` @@ -94,7 +93,7 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM ipld.blocks WHERE key = $1` + RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) const ( @@ -582,11 +581,10 @@ func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Rece } }() - headerCID, err := b.Retriever.RetrieveHeaderCIDByHash(tx, hash) + blockNumber, err := b.Retriever.RetrieveBlockNumberByHash(tx, hash) if err != nil { return nil, err } - blockNumber, _ := strconv.ParseUint(string(headerCID.BlockNumber), 10, 64) return b.GetReceiptsByBlockHashAndNumber(tx, hash, blockNumber) } diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index d6ff7ac1..7548f4c8 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -19,6 +19,7 @@ package eth import ( "fmt" "math/big" + "strconv" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/common" @@ -202,14 +203,14 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF args := make([]interface{}, 0, 4) id := 1 pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash, - eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, + eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, - data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash + data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash FROM eth.log_cids, eth.receipt_cids, ipld.blocks WHERE eth.log_cids.rct_id = receipt_cids.tx_id AND eth.log_cids.header_id = receipt_cids.header_id AND eth.log_cids.block_number = receipt_cids.block_number - AND log_cids.leaf_mh_key = blocks.key + AND log_cids.cid = blocks.key AND log_cids.block_number = blocks.block_number AND receipt_cids.header_id = $1` @@ -239,10 +240,10 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { log.Debug("retrieving log cids for receipt ids") args := make([]interface{}, 0, 4) - pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, + pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, - eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status, header_cids.block_hash + eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE eth.log_cids.rct_id = receipt_cids.tx_id AND eth.log_cids.header_id = eth.receipt_cids.header_id @@ -285,14 +286,15 @@ func hasTopics(topics [][]string) bool { return false } -// RetrieveHeaderCIDByHash returns the header for the given block hash -func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) { - log.Debug("retrieving header cids for block hash ", blockHash.String()) - pgStr := `SELECT block_hash, CAST(block_number as Text), parent_hash, cid, mh_key, CAST(td as Text), - state_root, uncle_root, tx_root, receipt_root, bloom, timestamp FROM eth.header_cids - WHERE block_hash = $1` - var headerCID models.HeaderModel - return headerCID, tx.Get(&headerCID, pgStr, blockHash.String()) +// RetrieveBlockNumberByHash returns the block number for the given block hash +func (ecr *CIDRetriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) { + log.Debug("retrieving block number for block hash ", blockHash.String()) + pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1` + var blockNumberStr string + if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil { + return 0, err + } + return strconv.ParseUint(blockNumberStr, 10, 64) } // RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 5d1075d3..eb849ca1 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -34,7 +34,7 @@ const ( RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN ipld.blocks ON ( - header_cids.mh_key = blocks.key + header_cids.cid = blocks.key AND header_cids.block_number = blocks.block_number ) WHERE block_hash = $1` @@ -45,7 +45,7 @@ const ( AND uncle_cids.block_number = header_cids.block_number ) INNER JOIN ipld.blocks ON ( - uncle_cids.mh_key = blocks.key + uncle_cids.cid = blocks.key AND uncle_cids.block_number = blocks.block_number ) WHERE header_cids.block_hash = $1 @@ -58,7 +58,7 @@ const ( AND uncle_cids.block_number = header_cids.block_number ) INNER JOIN ipld.blocks ON ( - uncle_cids.mh_key = blocks.key + uncle_cids.cid = blocks.key AND uncle_cids.block_number = blocks.block_number ) WHERE header_cids.block_hash = $1 @@ -70,7 +70,7 @@ const ( AND transaction_cids.block_number = header_cids.block_number ) INNER JOIN ipld.blocks ON ( - transaction_cids.mh_key = blocks.key + transaction_cids.cid = blocks.key AND transaction_cids.block_number = blocks.block_number ) WHERE block_hash = $1 @@ -83,12 +83,12 @@ const ( AND transaction_cids.block_number = header_cids.block_number ) INNER JOIN ipld.blocks ON ( - transaction_cids.mh_key = blocks.key + transaction_cids.cid = blocks.key AND transaction_cids.block_number = blocks.block_number ) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` - RetrieveReceiptsPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash + RetrieveReceiptsPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON ( receipt_cids.tx_id = transaction_cids.tx_hash @@ -100,13 +100,13 @@ const ( AND transaction_cids.block_number = header_cids.block_number ) INNER JOIN ipld.blocks ON ( - receipt_cids.leaf_mh_key = blocks.key + receipt_cids.cid = blocks.key AND receipt_cids.block_number = blocks.block_number ) WHERE block_hash = $1 AND header_cids.block_number = $2 ORDER BY eth.transaction_cids.index ASC` - RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash FROM eth.receipt_cids INNER JOIN eth.transaction_cids ON ( receipt_cids.tx_id = transaction_cids.tx_hash @@ -118,12 +118,12 @@ const ( AND transaction_cids.block_number = header_cids.block_number ) INNER JOIN ipld.blocks ON ( - receipt_cids.leaf_mh_key = blocks.key + receipt_cids.cid = blocks.key AND receipt_cids.block_number = blocks.block_number ) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON ( state_cids.header_id = header_cids.block_hash @@ -136,13 +136,13 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, mh_key, block_number, node_type, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, block_number, node_type, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` ) var EmptyNodeValue = make([]byte, common.HashLength) type rctIpldResult struct { - LeafCID string `db:"leaf_cid"` + LeafCID string `db:"cid"` Data []byte `db:"data"` TxHash string `db:"tx_hash"` } @@ -296,7 +296,6 @@ func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Has type nodeInfo struct { CID string `db:"cid"` - MhKey string `db:"mh_key"` BlockNumber string `db:"block_number"` Data []byte `db:"data"` NodeType int `db:"node_type"` @@ -320,7 +319,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr if err != nil { return "", nil, err } - accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.MhKey, blockNumber) + accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.CID, blockNumber) if err != nil { return "", nil, err } @@ -351,7 +350,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err != nil { return "", nil, nil, err } - storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.MhKey, blockNumber) + storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.CID, blockNumber) if err != nil { return "", nil, nil, err } diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 1efa9084..d20e21ed 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -249,7 +249,7 @@ type ConvertedPayload struct { // LogResult represent a log. type LogResult struct { - LeafCID string `db:"leaf_cid"` + LeafCID string `db:"cid"` ReceiptID string `db:"rct_id"` Address string `db:"address"` Index int64 `db:"index"` From 5bd9783aed1c870ca72d196e7277baf438056e06 Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 2 Mar 2023 18:14:37 -0600 Subject: [PATCH 03/14] move all sql statements into own file --- pkg/eth/interfaces.go | 47 ----- pkg/eth/ipld_retriever.go | 367 -------------------------------------- pkg/eth/sql.go | 154 ++++++++++++++++ 3 files changed, 154 insertions(+), 414 deletions(-) delete mode 100644 pkg/eth/interfaces.go delete mode 100644 pkg/eth/ipld_retriever.go create mode 100644 pkg/eth/sql.go diff --git a/pkg/eth/interfaces.go b/pkg/eth/interfaces.go deleted file mode 100644 index 9e2dcae3..00000000 --- a/pkg/eth/interfaces.go +++ /dev/null @@ -1,47 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package eth - -import ( - "context" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/rpc" -) - -// FilterBackend is the geth interface we need to satisfy to use their filters -type FilterBackend interface { - ChainDb() ethdb.Database - HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - - SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - - BloomStatus() (uint64, uint64) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) -} diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go deleted file mode 100644 index eb849ca1..00000000 --- a/pkg/eth/ipld_retriever.go +++ /dev/null @@ -1,367 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package eth - -import ( - "fmt" - "strconv" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" - "github.com/ethereum/go-ethereum/statediff/trie_helpers" - sdtypes "github.com/ethereum/go-ethereum/statediff/types" - "github.com/jmoiron/sqlx" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/rlp" -) - -const ( - RetrieveHeaderByHashPgStr = `SELECT cid, data - FROM eth.header_cids - INNER JOIN ipld.blocks ON ( - header_cids.cid = blocks.key - AND header_cids.block_number = blocks.block_number - ) - WHERE block_hash = $1` - RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data - FROM eth.uncle_cids - INNER JOIN eth.header_cids ON ( - uncle_cids.header_id = header_cids.block_hash - AND uncle_cids.block_number = header_cids.block_number - ) - INNER JOIN ipld.blocks ON ( - uncle_cids.cid = blocks.key - AND uncle_cids.block_number = blocks.block_number - ) - WHERE header_cids.block_hash = $1 - AND header_cids.block_number = $2 - ORDER BY uncle_cids.parent_hash` - RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data - FROM eth.uncle_cids - INNER JOIN eth.header_cids ON ( - uncle_cids.header_id = header_cids.block_hash - AND uncle_cids.block_number = header_cids.block_number - ) - INNER JOIN ipld.blocks ON ( - uncle_cids.cid = blocks.key - AND uncle_cids.block_number = blocks.block_number - ) - WHERE header_cids.block_hash = $1 - ORDER BY uncle_cids.parent_hash` - RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data - FROM eth.transaction_cids - INNER JOIN eth.header_cids ON ( - transaction_cids.header_id = header_cids.block_hash - AND transaction_cids.block_number = header_cids.block_number - ) - INNER JOIN ipld.blocks ON ( - transaction_cids.cid = blocks.key - AND transaction_cids.block_number = blocks.block_number - ) - WHERE block_hash = $1 - AND header_cids.block_number = $2 - ORDER BY eth.transaction_cids.index ASC` - RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data - FROM eth.transaction_cids - INNER JOIN eth.header_cids ON ( - transaction_cids.header_id = header_cids.block_hash - AND transaction_cids.block_number = header_cids.block_number - ) - INNER JOIN ipld.blocks ON ( - transaction_cids.cid = blocks.key - AND transaction_cids.block_number = blocks.block_number - ) - WHERE block_hash = $1 - ORDER BY eth.transaction_cids.index ASC` - RetrieveReceiptsPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash - FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON ( - receipt_cids.tx_id = transaction_cids.tx_hash - AND receipt_cids.header_id = transaction_cids.header_id - AND receipt_cids.block_number = transaction_cids.block_number - ) - INNER JOIN eth.header_cids ON ( - transaction_cids.header_id = header_cids.block_hash - AND transaction_cids.block_number = header_cids.block_number - ) - INNER JOIN ipld.blocks ON ( - receipt_cids.cid = blocks.key - AND receipt_cids.block_number = blocks.block_number - ) - WHERE block_hash = $1 - AND header_cids.block_number = $2 - ORDER BY eth.transaction_cids.index ASC` - RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash - FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON ( - receipt_cids.tx_id = transaction_cids.tx_hash - AND receipt_cids.header_id = transaction_cids.header_id - AND receipt_cids.block_number = transaction_cids.block_number - ) - INNER JOIN eth.header_cids ON ( - transaction_cids.header_id = header_cids.block_hash - AND transaction_cids.block_number = header_cids.block_number - ) - INNER JOIN ipld.blocks ON ( - receipt_cids.cid = blocks.key - AND receipt_cids.block_number = blocks.block_number - ) - WHERE block_hash = $1 - ORDER BY eth.transaction_cids.index ASC` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.node_type - FROM eth.state_cids - INNER JOIN eth.header_cids ON ( - state_cids.header_id = header_cids.block_hash - AND state_cids.block_number = header_cids.block_number - ) - WHERE state_leaf_key = $1 - AND header_cids.block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $2) - AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) - ORDER BY header_cids.block_number DESC - LIMIT 1` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, block_number, node_type, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` -) - -var EmptyNodeValue = make([]byte, common.HashLength) - -type rctIpldResult struct { - LeafCID string `db:"cid"` - Data []byte `db:"data"` - TxHash string `db:"tx_hash"` -} - -type ipldResult struct { - CID string `db:"cid"` - Data []byte `db:"data"` - TxHash string `db:"tx_hash"` -} - -type IPLDRetriever struct { - db *sqlx.DB -} - -func NewIPLDRetriever(db *sqlx.DB) *IPLDRetriever { - return &IPLDRetriever{ - db: db, - } -} - -// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash -func (r *IPLDRetriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) { - headerResult := new(ipldResult) - return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex()) -} - -// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block) -func (r *IPLDRetriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { - uncleResults := make([]ipldResult, 0) - if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil { - return nil, nil, err - } - cids := make([]string, len(uncleResults)) - uncles := make([][]byte, len(uncleResults)) - for i, res := range uncleResults { - cids[i] = res.CID - uncles[i] = res.Data - } - return cids, uncles, nil -} - -// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block) -func (r *IPLDRetriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { - uncleResults := make([]ipldResult, 0) - if err := tx.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil { - return nil, nil, err - } - cids := make([]string, len(uncleResults)) - uncles := make([][]byte, len(uncleResults)) - for i, res := range uncleResults { - cids[i] = res.CID - uncles[i] = res.Data - } - return cids, uncles, nil -} - -// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number -func (r *IPLDRetriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { - txResults := make([]ipldResult, 0) - if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil { - return nil, nil, err - } - cids := make([]string, len(txResults)) - txs := make([][]byte, len(txResults)) - for i, res := range txResults { - cids[i] = res.CID - txs[i] = res.Data - } - return cids, txs, nil -} - -// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash -func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { - txResults := make([]ipldResult, 0) - if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil { - return nil, nil, err - } - cids := make([]string, len(txResults)) - txs := make([][]byte, len(txResults)) - for i, res := range txResults { - cids[i] = res.CID - txs[i] = res.Data - } - return cids, txs, nil -} - -// DecodeLeafNode decodes the leaf node data -func DecodeLeafNode(node []byte) ([]byte, error) { - var nodeElements []interface{} - if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return nil, err - } - ty, err := trie_helpers.CheckKeyType(nodeElements) - if err != nil { - return nil, err - } - - if ty != sdtypes.Leaf { - return nil, fmt.Errorf("expected leaf node but found %s", ty) - } - return nodeElements[1].([]byte), nil -} - -// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number. -// cid returned corresponds to the leaf node data which contains the receipt. -func (r *IPLDRetriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) { - rctResults := make([]rctIpldResult, 0) - if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil { - return nil, nil, nil, err - } - cids := make([]string, len(rctResults)) - rcts := make([][]byte, len(rctResults)) - txs := make([]common.Hash, len(rctResults)) - - for i, res := range rctResults { - cids[i] = res.LeafCID - nodeVal, err := DecodeLeafNode(res.Data) - if err != nil { - return nil, nil, nil, err - } - rcts[i] = nodeVal - txs[i] = common.HexToHash(res.TxHash) - } - - return cids, rcts, txs, nil -} - -// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash. -// cid returned corresponds to the leaf node data which contains the receipt. -func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) { - rctResults := make([]rctIpldResult, 0) - if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil { - return nil, nil, nil, err - } - cids := make([]string, len(rctResults)) - rcts := make([][]byte, len(rctResults)) - txs := make([]common.Hash, len(rctResults)) - - for i, res := range rctResults { - cids[i] = res.LeafCID - nodeVal, err := DecodeLeafNode(res.Data) - if err != nil { - return nil, nil, nil, err - } - rcts[i] = nodeVal - txs[i] = common.HexToHash(res.TxHash) - } - - return cids, rcts, txs, nil -} - -type nodeInfo struct { - CID string `db:"cid"` - BlockNumber string `db:"block_number"` - Data []byte `db:"data"` - NodeType int `db:"node_type"` - StateLeafRemoved bool `db:"state_leaf_removed"` -} - -// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash -// TODO: ensure this handles deleted accounts appropriately -func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { - accountResult := new(nodeInfo) - leafKey := crypto.Keccak256Hash(address.Bytes()) - if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { - return "", nil, err - } - - if accountResult.NodeType == sdtypes.Removed.Int() { - return "", EmptyNodeValue, nil - } - - blockNumber, err := strconv.ParseUint(accountResult.BlockNumber, 10, 64) - if err != nil { - return "", nil, err - } - accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.CID, blockNumber) - if err != nil { - return "", nil, err - } - - var i []interface{} - if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { - return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) - } - if len(i) != 2 { - return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements") - } - return accountResult.CID, i[1].([]byte), nil -} - -// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash -func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) { - storageResult := new(nodeInfo) - stateLeafKey := crypto.Keccak256Hash(address.Bytes()) - storageHash := crypto.Keccak256Hash(key.Bytes()) - if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { - return "", nil, nil, err - } - if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { - return "", EmptyNodeValue, EmptyNodeValue, nil - } - - blockNumber, err := strconv.ParseUint(storageResult.BlockNumber, 10, 64) - if err != nil { - return "", nil, nil, err - } - storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.CID, blockNumber) - if err != nil { - return "", nil, nil, err - } - - var i []interface{} - if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { - err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) - return "", nil, nil, err - } - if len(i) != 2 { - return "", nil, nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements") - } - return storageResult.CID, storageResult.Data, i[1].([]byte), nil -} diff --git a/pkg/eth/sql.go b/pkg/eth/sql.go new file mode 100644 index 00000000..20283cdc --- /dev/null +++ b/pkg/eth/sql.go @@ -0,0 +1,154 @@ +package eth + +const ( + RetrieveHeaderByHashPgStr = `SELECT cid, data + FROM eth.header_cids + INNER JOIN ipld.blocks ON ( + header_cids.cid = blocks.key + AND header_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1` + RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON ( + uncle_cids.header_id = header_cids.block_hash + AND uncle_cids.block_number = header_cids.block_number + ) + INNER JOIN ipld.blocks ON ( + uncle_cids.cid = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) + WHERE header_cids.block_hash = $1 + AND header_cids.block_number = $2 + ORDER BY uncle_cids.parent_hash` + RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data + FROM eth.uncle_cids + INNER JOIN eth.header_cids ON ( + uncle_cids.header_id = header_cids.block_hash + AND uncle_cids.block_number = header_cids.block_number + ) + INNER JOIN ipld.blocks ON ( + uncle_cids.cid = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) + WHERE header_cids.block_hash = $1 + ORDER BY uncle_cids.parent_hash` + RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN ipld.blocks ON ( + transaction_cids.cid = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1 + AND header_cids.block_number = $2 + ORDER BY eth.transaction_cids.index ASC` + RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN ipld.blocks ON ( + transaction_cids.cid = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1 + ORDER BY eth.transaction_cids.index ASC` + RetrieveReceiptsPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.header_id = transaction_cids.header_id + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN ipld.blocks ON ( + receipt_cids.cid = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1 + AND header_cids.block_number = $2 + ORDER BY eth.transaction_cids.index ASC` + RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data, eth.transaction_cids.tx_hash + FROM eth.receipt_cids + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.header_id = transaction_cids.header_id + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN ipld.blocks ON ( + receipt_cids.cid = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) + WHERE block_hash = $1 + ORDER BY eth.transaction_cids.index ASC` + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.node_type + FROM eth.state_cids + INNER JOIN eth.header_cids ON ( + state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + ) + WHERE state_leaf_key = $1 + AND header_cids.block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $2) + AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) + ORDER BY header_cids.block_number DESC + LIMIT 1` + RetrieveFilteredGQLLogs = `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash, + eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, + eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, + data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash + FROM eth.log_cids, eth.receipt_cids, ipld.blocks + WHERE eth.log_cids.rct_id = receipt_cids.tx_id + AND eth.log_cids.header_id = receipt_cids.header_id + AND eth.log_cids.block_number = receipt_cids.block_number + AND log_cids.cid = blocks.key + AND log_cids.block_number = blocks.block_number + AND receipt_cids.header_id = $1` + RetrieveFilteredLogs = `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, + eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, + eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, + eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash + FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE eth.log_cids.rct_id = receipt_cids.tx_id + AND eth.log_cids.header_id = eth.receipt_cids.header_id + AND eth.log_cids.block_number = eth.receipt_cids.block_number + AND receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.header_id = transaction_cids.header_id + AND receipt_cids.block_number = transaction_cids.block_number + AND transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, block_number, node_type, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` +) + +type rctIpldResult struct { + LeafCID string `db:"cid"` + Data []byte `db:"data"` + TxHash string `db:"tx_hash"` +} + +type ipldResult struct { + CID string `db:"cid"` + Data []byte `db:"data"` + TxHash string `db:"tx_hash"` +} + +type nodeInfo struct { + CID string `db:"cid"` + BlockNumber string `db:"block_number"` + Data []byte `db:"data"` + NodeType int `db:"node_type"` + StateLeafRemoved bool `db:"state_leaf_removed"` +} From 6f1bfc7fcec0479d41f33f26e7625a4e7b4d387f Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 2 Mar 2023 18:27:30 -0600 Subject: [PATCH 04/14] combine ipld and cid retriever into one struct. the original reason for the separation of concerns was because we used to fetch cids directly from the database but then use those to fetch iplds using an ipfs.BlockService abstraction ontop of our database (and/or a ipfs.BlockExchange). But now they are both accessed directly in the local DB. Additionally, some of the queries should be further refined/refactored/combined as we no longer need to take two trips but can retrieve cid and ipld in the same query. --- cmd/serve.go | 1 - pkg/eth/api.go | 4 +- pkg/eth/backend.go | 35 +- pkg/eth/cid_retriever.go | 376 ------------ pkg/eth/retriever.go | 555 ++++++++++++++++++ ...id_retriever_test.go => retriever_test.go} | 4 +- pkg/eth/test_helpers/test_data.go | 310 +--------- pkg/graphql/graphql.go | 2 +- test_config/test_config.go | 1 + 9 files changed, 584 insertions(+), 704 deletions(-) delete mode 100644 pkg/eth/cid_retriever.go create mode 100644 pkg/eth/retriever.go rename pkg/eth/{cid_retriever_test.go => retriever_test.go} (98%) diff --git a/cmd/serve.go b/cmd/serve.go index ec662cfc..0bd3e29b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -25,7 +25,6 @@ import ( "sync" "time" - "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/rpc" "github.com/mailgun/groupcache/v2" diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 0e923cca..96196df9 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -720,7 +720,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log // If we have a blockHash to filter on, fire off single retrieval query if crit.BlockHash != nil { - filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, 0, crit.BlockHash) + filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, 0, crit.BlockHash) if err != nil { return nil, err } @@ -748,7 +748,7 @@ func (pea *PublicEthAPI) localGetLogs(crit filters.FilterCriteria) ([]*types.Log end := endingBlock.Int64() var logs []*types.Log for i := start; i <= end; i++ { - filteredLogs, err := pea.B.Retriever.RetrieveFilteredLog(tx, filter, i, nil) + filteredLogs, err := pea.B.Retriever.RetrieveFilteredLogs(tx, filter, i, nil) if err != nil { return nil, err } diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index cecd29a0..03688d0f 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -28,6 +28,7 @@ import ( validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" + "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" @@ -48,8 +49,6 @@ import ( sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" ) var ( @@ -105,8 +104,7 @@ type Backend struct { DB *sqlx.DB // postgres db interfaces - Retriever *CIDRetriever - IPLDRetriever *IPLDRetriever + Retriever *Retriever // ethereum interfaces EthDB ethdb.Database @@ -131,7 +129,7 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { groupName = StateDBGroupCacheName } - r := NewCIDRetriever(db) + r := NewRetriever(db) ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{ Name: groupName, Size: gcc.StateDB.CacheSizeInMB * 1024 * 1024, @@ -143,7 +141,6 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { return &Backend{ DB: db, Retriever: r, - IPLDRetriever: NewIPLDRetriever(db), EthDB: ethDB, StateDatabase: state.NewDatabase(ethDB), Config: c, @@ -204,7 +201,7 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He } }() - _, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash) + _, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash) if err != nil { return nil, err } @@ -407,7 +404,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo // GetHeaderByBlockHash retrieves header for a provided block hash func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.Header, error) { - _, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(tx, hash) + _, headerRLP, err := b.Retriever.RetrieveHeaderByHash(tx, hash) if err != nil { return nil, err } @@ -418,7 +415,7 @@ func (b *Backend) GetHeaderByBlockHash(tx *sqlx.Tx, hash common.Hash) (*types.He // GetUnclesByBlockHash retrieves uncles for a provided block hash func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types.Header, error) { - _, uncleBytes, err := b.IPLDRetriever.RetrieveUnclesByBlockHash(tx, hash) + _, uncleBytes, err := b.Retriever.RetrieveUnclesByBlockHash(tx, hash) if err != nil { return nil, err } @@ -439,7 +436,7 @@ func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types. // GetUnclesByBlockHashAndNumber retrieves uncles for a provided block hash and number func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) ([]*types.Header, error) { - _, uncleBytes, err := b.IPLDRetriever.RetrieveUncles(tx, hash, number) + _, uncleBytes, err := b.Retriever.RetrieveUncles(tx, hash, number) if err != nil { return nil, err } @@ -460,7 +457,7 @@ func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, n // GetTransactionsByBlockHash retrieves transactions for a provided block hash func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Transactions, error) { - _, transactionBytes, err := b.IPLDRetriever.RetrieveTransactionsByBlockHash(tx, hash) + _, transactionBytes, err := b.Retriever.RetrieveTransactionsByBlockHash(tx, hash) if err != nil { return nil, err } @@ -480,7 +477,7 @@ func (b *Backend) GetTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) (typ // GetTransactionsByBlockHashAndNumber retrieves transactions for a provided block hash and number func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Transactions, error) { - _, transactionBytes, err := b.IPLDRetriever.RetrieveTransactions(tx, hash, number) + _, transactionBytes, err := b.Retriever.RetrieveTransactions(tx, hash, number) if err != nil { return nil, err } @@ -500,7 +497,7 @@ func (b *Backend) GetTransactionsByBlockHashAndNumber(tx *sqlx.Tx, hash common.H // GetReceiptsByBlockHash retrieves receipts for a provided block hash func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.Receipts, error) { - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(tx, hash) + _, receiptBytes, txs, err := b.Retriever.RetrieveReceiptsByBlockHash(tx, hash) if err != nil { return nil, err } @@ -518,7 +515,7 @@ func (b *Backend) GetReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) (types.R // GetReceiptsByBlockHashAndNumber retrieves receipts for a provided block hash and number func (b *Backend) GetReceiptsByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, number uint64) (types.Receipts, error) { - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) + _, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number) if err != nil { return nil, err } @@ -607,7 +604,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64) } }() - _, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceipts(tx, hash, number) + _, receiptBytes, txs, err := b.Retriever.RetrieveReceipts(tx, hash, number) if err != nil { return nil, err } @@ -695,8 +692,8 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) { func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { vmError := func() error { return nil } txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(header, b, nil) - return vm.NewEVM(context, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil + blockContext := core.NewEVMBlockContext(header, b, nil) + return vm.NewEVM(blockContext, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil } // GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash @@ -748,7 +745,7 @@ func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, return nil, err } - _, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash) + _, accountRlp, err := b.Retriever.RetrieveAccountByAddressAndBlockHash(address, hash) if err != nil { return nil, err } @@ -879,7 +876,7 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, return nil, err } - _, _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash) + _, _, storageRlp, err := b.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash) return storageRlp, err } diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go deleted file mode 100644 index 7548f4c8..00000000 --- a/pkg/eth/cid_retriever.go +++ /dev/null @@ -1,376 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package eth - -import ( - "fmt" - "math/big" - "strconv" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/log" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/jmoiron/sqlx" - "github.com/lib/pq" - "gorm.io/driver/postgres" - "gorm.io/gorm" -) - -// CIDRetriever satisfies the CIDRetriever interface for ethereum -type CIDRetriever struct { - db *sqlx.DB - gormDB *gorm.DB -} - -type IPLDModelRecord struct { - models.IPLDModel -} - -// TableName overrides the table name used by IPLD -func (IPLDModelRecord) TableName() string { - return "ipld.blocks" -} - -type HeaderCIDRecord struct { - CID string `gorm:"column:cid"` - BlockHash string `gorm:"primaryKey"` - BlockNumber string `gorm:"primaryKey"` - ParentHash string - Timestamp uint64 - StateRoot string - TotalDifficulty string `gorm:"column:td"` - TxRoot string - RctRoot string `gorm:"column:receipt_root"` - UncleRoot string - Bloom []byte - MhKey string - - // gorm doesn't check if foreign key exists in database. - // It is required to eager load relations using preload. - TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"` - IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` -} - -// TableName overrides the table name used by HeaderCIDRecord -func (HeaderCIDRecord) TableName() string { - return "eth.header_cids" -} - -type TransactionCIDRecord struct { - CID string `gorm:"column:cid"` - TxHash string `gorm:"primaryKey"` - BlockNumber string `gorm:"primaryKey"` - HeaderID string `gorm:"column:header_id"` - Index int64 - Src string - Dst string - MhKey string - IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` -} - -// TableName overrides the table name used by TransactionCIDRecord -func (TransactionCIDRecord) TableName() string { - return "eth.transaction_cids" -} - -// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface -func NewCIDRetriever(db *sqlx.DB) *CIDRetriever { - gormDB, err := gorm.Open(postgres.New(postgres.Config{ - Conn: db, - }), &gorm.Config{}) - - if err != nil { - log.Error(err) - return nil - } - - return &CIDRetriever{ - db: db, - gormDB: gormDB, - } -} - -// RetrieveFirstBlockNumber is used to retrieve the first block number in the db -func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) { - var blockNumber int64 - err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1") - return blockNumber, err -} - -// RetrieveLastBlockNumber is used to retrieve the latest block number in the db -func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) { - var blockNumber int64 - err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1") - return blockNumber, err -} - -func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) { - for i, topicSet := range topics { - if len(topicSet) == 0 { - continue - } - - if !first { - pgStr += " AND" - } else { - first = false - } - pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id) - args = append(args, pq.Array(topicSet)) - *id++ - } - return pgStr, args -} - -func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) { - if len(rctFilter.LogAddresses) > 0 { - pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id) - args = append(args, pq.Array(rctFilter.LogAddresses)) - *id++ - } - - // Filter on topics if there are any - if hasTopics(rctFilter.Topics) { - pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) - } - - return pgStr, args -} - -func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) { - rctCond := " AND (receipt_cids.tx_id = ANY ( " - logQuery := "SELECT rct_id FROM eth.log_cids WHERE" - if len(rctFilter.LogAddresses) > 0 { - // Filter on log contract addresses if there are any - pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id) - args = append(args, pq.Array(rctFilter.LogAddresses)) - *id++ - - // Filter on topics if there are any - if hasTopics(rctFilter.Topics) { - pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) - } - - pgStr += ")" - - // Filter on txHashes if there are any, and we are matching txs - if rctFilter.MatchTxs && len(txHashes) > 0 { - pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) - args = append(args, pq.Array(txHashes)) - } - pgStr += ")" - } else { // If there are no contract addresses to filter on - // Filter on topics if there are any - if hasTopics(rctFilter.Topics) { - pgStr += rctCond + logQuery - pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true) - pgStr += ")" - // Filter on txHashes if there are any, and we are matching txs - if rctFilter.MatchTxs && len(txHashes) > 0 { - pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) - args = append(args, pq.Array(txHashes)) - } - pgStr += ")" - } else if rctFilter.MatchTxs && len(txHashes) > 0 { - // If there are no contract addresses or topics to filter on, - // Filter on txHashes if there are any, and we are matching txs - pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id) - args = append(args, pq.Array(txHashes)) - } - } - - return pgStr, args -} - -// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided -// filter parameters. -func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) { - log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String()) - args := make([]interface{}, 0, 4) - id := 1 - pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash, - eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, - eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, - data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash - FROM eth.log_cids, eth.receipt_cids, ipld.blocks - WHERE eth.log_cids.rct_id = receipt_cids.tx_id - AND eth.log_cids.header_id = receipt_cids.header_id - AND eth.log_cids.block_number = receipt_cids.block_number - AND log_cids.cid = blocks.key - AND log_cids.block_number = blocks.block_number - AND receipt_cids.header_id = $1` - - args = append(args, blockHash.String()) - id++ - - if blockNumber != nil { - pgStr += ` AND receipt_cids.block_number = $2` - id++ - args = append(args, blockNumber.Int64()) - } - - pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) - pgStr += ` ORDER BY log_cids.index` - - logCIDs := make([]LogResult, 0) - err := tx.Select(&logCIDs, pgStr, args...) - if err != nil { - return nil, err - } - - return logCIDs, nil -} - -// RetrieveFilteredLog retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided -// filter parameters. -func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { - log.Debug("retrieving log cids for receipt ids") - args := make([]interface{}, 0, 4) - pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, - eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, - eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, - eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash - FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids - WHERE eth.log_cids.rct_id = receipt_cids.tx_id - AND eth.log_cids.header_id = eth.receipt_cids.header_id - AND eth.log_cids.block_number = eth.receipt_cids.block_number - AND receipt_cids.tx_id = transaction_cids.tx_hash - AND receipt_cids.header_id = transaction_cids.header_id - AND receipt_cids.block_number = transaction_cids.block_number - AND transaction_cids.header_id = header_cids.block_hash - AND transaction_cids.block_number = header_cids.block_number` - id := 1 - if blockNumber > 0 { - pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) - args = append(args, blockNumber) - id++ - } - if blockHash != nil { - pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id) - args = append(args, blockHash.String()) - id++ - } - - pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) - pgStr += ` ORDER BY log_cids.index` - - logCIDs := make([]LogResult, 0) - err := tx.Select(&logCIDs, pgStr, args...) - if err != nil { - return nil, err - } - - return logCIDs, nil -} - -func hasTopics(topics [][]string) bool { - for _, topicSet := range topics { - if len(topicSet) > 0 { - return true - } - } - return false -} - -// RetrieveBlockNumberByHash returns the block number for the given block hash -func (ecr *CIDRetriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) { - log.Debug("retrieving block number for block hash ", blockHash.String()) - pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1` - var blockNumberStr string - if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil { - return 0, err - } - return strconv.ParseUint(blockNumberStr, 10, 64) -} - -// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) { - log.Debug("retrieving header cids and tx cids for block number ", blockNumber) - - var headerCIDs []HeaderCIDRecord - - // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 - // Will use join for TransactionCIDs once preload for 1:N is supported. - err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { - return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") - }).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error - - if err != nil { - log.Error("header cid retrieval error") - return nil, err - } - - return headerCIDs, nil -} - -// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number) -func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) { - log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String()) - - var headerCIDs []HeaderCIDRecord - - conditions := map[string]interface{}{"block_hash": blockHash.String()} - if blockNumber != nil { - conditions["header_cids.block_number"] = blockNumber.Int64() - } - - // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 - // Will use join for TransactionCIDs once preload for 1:N is supported. - err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { - return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") - }).Joins("IPLD").Find(&headerCIDs, conditions).Error - - if err != nil { - log.Error("header cid retrieval error") - return HeaderCIDRecord{}, err - } - - if len(headerCIDs) == 0 { - return HeaderCIDRecord{}, errHeaderHashNotFound - } else if len(headerCIDs) > 1 { - return HeaderCIDRecord{}, errMultipleHeadersForHash - } - - return headerCIDs[0], nil -} - -// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number) -func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) { - log.Debug("retrieving tx cid for tx hash ", txHash) - - var txCIDs []TransactionCIDRecord - - var err error - if blockNumber != nil { - err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error - } else { - err = ecr.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error - } - if err != nil { - log.Error("tx retrieval error") - return TransactionCIDRecord{}, err - } - - if len(txCIDs) == 0 { - return TransactionCIDRecord{}, errTxHashNotFound - } else if len(txCIDs) > 1 { - // a transaction can be part of a only one canonical block - return TransactionCIDRecord{}, errTxHashInMultipleBlocks - } - - return txCIDs[0], nil -} diff --git a/pkg/eth/retriever.go b/pkg/eth/retriever.go new file mode 100644 index 00000000..e9a04622 --- /dev/null +++ b/pkg/eth/retriever.go @@ -0,0 +1,555 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package eth + +import ( + "fmt" + "math/big" + "strconv" + + "github.com/cerc-io/ipld-eth-server/v4/pkg/log" + "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/jmoiron/sqlx" + "github.com/lib/pq" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +// Retriever is used for fetching +type Retriever struct { + db *sqlx.DB + gormDB *gorm.DB +} + +type IPLDModelRecord struct { + models.IPLDModel +} + +// TableName overrides the table name used by IPLD +func (IPLDModelRecord) TableName() string { + return "ipld.blocks" +} + +type HeaderCIDRecord struct { + CID string `gorm:"column:cid"` + BlockHash string `gorm:"primaryKey"` + BlockNumber string `gorm:"primaryKey"` + ParentHash string + Timestamp uint64 + StateRoot string + TotalDifficulty string `gorm:"column:td"` + TxRoot string + RctRoot string `gorm:"column:receipt_root"` + UncleRoot string + Bloom []byte + MhKey string + + // gorm doesn't check if foreign key exists in database. + // It is required to eager load relations using preload. + TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"` + IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` +} + +// TableName overrides the table name used by HeaderCIDRecord +func (HeaderCIDRecord) TableName() string { + return "eth.header_cids" +} + +type TransactionCIDRecord struct { + CID string `gorm:"column:cid"` + TxHash string `gorm:"primaryKey"` + BlockNumber string `gorm:"primaryKey"` + HeaderID string `gorm:"column:header_id"` + Index int64 + Src string + Dst string + MhKey string + IPLD IPLDModelRecord `gorm:"foreignKey:MhKey,BlockNumber;references:Key,BlockNumber"` +} + +// TableName overrides the table name used by TransactionCIDRecord +func (TransactionCIDRecord) TableName() string { + return "eth.transaction_cids" +} + +// NewRetriever returns a pointer to a new Retriever which supports the Retriever interface +func NewRetriever(db *sqlx.DB) *Retriever { + gormDB, err := gorm.Open(postgres.New(postgres.Config{ + Conn: db, + }), &gorm.Config{}) + + if err != nil { + log.Error(err) + return nil + } + + return &Retriever{ + db: db, + gormDB: gormDB, + } +} + +// RetrieveFirstBlockNumber is used to retrieve the first block number in the db +func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) { + var blockNumber int64 + err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1") + return blockNumber, err +} + +// RetrieveLastBlockNumber is used to retrieve the latest block number in the db +func (r *Retriever) RetrieveLastBlockNumber() (int64, error) { + var blockNumber int64 + err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1") + return blockNumber, err +} + +func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) { + for i, topicSet := range topics { + if len(topicSet) == 0 { + continue + } + + if !first { + pgStr += " AND" + } else { + first = false + } + pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id) + args = append(args, pq.Array(topicSet)) + *id++ + } + return pgStr, args +} + +func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) { + if len(rctFilter.LogAddresses) > 0 { + pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id) + args = append(args, pq.Array(rctFilter.LogAddresses)) + *id++ + } + + // Filter on topics if there are any + if hasTopics(rctFilter.Topics) { + pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) + } + + return pgStr, args +} + +func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) { + rctCond := " AND (receipt_cids.tx_id = ANY ( " + logQuery := "SELECT rct_id FROM eth.log_cids WHERE" + if len(rctFilter.LogAddresses) > 0 { + // Filter on log contract addresses if there are any + pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id) + args = append(args, pq.Array(rctFilter.LogAddresses)) + *id++ + + // Filter on topics if there are any + if hasTopics(rctFilter.Topics) { + pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false) + } + + pgStr += ")" + + // Filter on txHashes if there are any, and we are matching txs + if rctFilter.MatchTxs && len(txHashes) > 0 { + pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) + } + pgStr += ")" + } else { // If there are no contract addresses to filter on + // Filter on topics if there are any + if hasTopics(rctFilter.Topics) { + pgStr += rctCond + logQuery + pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true) + pgStr += ")" + // Filter on txHashes if there are any, and we are matching txs + if rctFilter.MatchTxs && len(txHashes) > 0 { + pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) + } + pgStr += ")" + } else if rctFilter.MatchTxs && len(txHashes) > 0 { + // If there are no contract addresses or topics to filter on, + // Filter on txHashes if there are any, and we are matching txs + pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id) + args = append(args, pq.Array(txHashes)) + } + } + + return pgStr, args +} + +// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided +// filter parameters. +func (r *Retriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) { + log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String()) + args := make([]interface{}, 0, 4) + id := 1 + pgStr := RetrieveFilteredGQLLogs + args = append(args, blockHash.String()) + id++ + + if blockNumber != nil { + pgStr += ` AND receipt_cids.block_number = $2` + id++ + args = append(args, blockNumber.Int64()) + } + + pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) + pgStr += ` ORDER BY log_cids.index` + + logCIDs := make([]LogResult, 0) + err := tx.Select(&logCIDs, pgStr, args...) + if err != nil { + return nil, err + } + + return logCIDs, nil +} + +// RetrieveFilteredLogs retrieves and returns all the log CIDs provided blockHeight or blockHash that conform to the provided +// filter parameters. +func (r *Retriever) RetrieveFilteredLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { + log.Debug("retrieving log cids for receipt ids") + args := make([]interface{}, 0, 4) + pgStr := RetrieveFilteredLogs + id := 1 + if blockNumber > 0 { + pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) + args = append(args, blockNumber) + id++ + } + if blockHash != nil { + pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id) + args = append(args, blockHash.String()) + id++ + } + + pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter) + pgStr += ` ORDER BY log_cids.index` + + logCIDs := make([]LogResult, 0) + err := tx.Select(&logCIDs, pgStr, args...) + if err != nil { + return nil, err + } + + return logCIDs, nil +} + +func hasTopics(topics [][]string) bool { + for _, topicSet := range topics { + if len(topicSet) > 0 { + return true + } + } + return false +} + +// RetrieveBlockNumberByHash returns the block number for the given block hash +func (r *Retriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) { + log.Debug("retrieving block number for block hash ", blockHash.String()) + pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1` + var blockNumberStr string + if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil { + return 0, err + } + return strconv.ParseUint(blockNumberStr, 10, 64) +} + +// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number +func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) { + log.Debug("retrieving header cids and tx cids for block number ", blockNumber) + + var headerCIDs []HeaderCIDRecord + + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 + // Will use join for TransactionCIDs once preload for 1:N is supported. + err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { + return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") + }).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error + + if err != nil { + log.Error("header cid retrieval error") + return nil, err + } + + return headerCIDs, nil +} + +// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number) +func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) { + log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String()) + + var headerCIDs []HeaderCIDRecord + + conditions := map[string]interface{}{"block_hash": blockHash.String()} + if blockNumber != nil { + conditions["header_cids.block_number"] = blockNumber.Int64() + } + + // https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283 + // Will use join for TransactionCIDs once preload for 1:N is supported. + err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB { + return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number") + }).Joins("IPLD").Find(&headerCIDs, conditions).Error + + if err != nil { + log.Error("header cid retrieval error") + return HeaderCIDRecord{}, err + } + + if len(headerCIDs) == 0 { + return HeaderCIDRecord{}, errHeaderHashNotFound + } else if len(headerCIDs) > 1 { + return HeaderCIDRecord{}, errMultipleHeadersForHash + } + + return headerCIDs[0], nil +} + +// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number) +func (r *Retriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) { + log.Debug("retrieving tx cid for tx hash ", txHash) + + var txCIDs []TransactionCIDRecord + + var err error + if blockNumber != nil { + err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error + } else { + err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error + } + if err != nil { + log.Error("tx retrieval error") + return TransactionCIDRecord{}, err + } + + if len(txCIDs) == 0 { + return TransactionCIDRecord{}, errTxHashNotFound + } else if len(txCIDs) > 1 { + // a transaction can be part of a only one canonical block + return TransactionCIDRecord{}, errTxHashInMultipleBlocks + } + + return txCIDs[0], nil +} + +var EmptyNodeValue = make([]byte, common.HashLength) + +// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash +func (r *Retriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) { + headerResult := new(ipldResult) + return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex()) +} + +// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block) +func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + uncleResults := make([]ipldResult, 0) + if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(uncleResults)) + uncles := make([][]byte, len(uncleResults)) + for i, res := range uncleResults { + cids[i] = res.CID + uncles[i] = res.Data + } + return cids, uncles, nil +} + +// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block) +func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { + uncleResults := make([]ipldResult, 0) + if err := tx.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil { + return nil, nil, err + } + cids := make([]string, len(uncleResults)) + uncles := make([][]byte, len(uncleResults)) + for i, res := range uncleResults { + cids[i] = res.CID + uncles[i] = res.Data + } + return cids, uncles, nil +} + +// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number +func (r *Retriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { + txResults := make([]ipldResult, 0) + if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil { + return nil, nil, err + } + cids := make([]string, len(txResults)) + txs := make([][]byte, len(txResults)) + for i, res := range txResults { + cids[i] = res.CID + txs[i] = res.Data + } + return cids, txs, nil +} + +// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash +func (r *Retriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { + txResults := make([]ipldResult, 0) + if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil { + return nil, nil, err + } + cids := make([]string, len(txResults)) + txs := make([][]byte, len(txResults)) + for i, res := range txResults { + cids[i] = res.CID + txs[i] = res.Data + } + return cids, txs, nil +} + +// DecodeLeafNode decodes the leaf node data +func DecodeLeafNode(node []byte) ([]byte, error) { + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return nil, err + } + ty, err := trie_helpers.CheckKeyType(nodeElements) + if err != nil { + return nil, err + } + + if ty != sdtypes.Leaf { + return nil, fmt.Errorf("expected leaf node but found %s", ty) + } + return nodeElements[1].([]byte), nil +} + +// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number. +// cid returned corresponds to the leaf node data which contains the receipt. +func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) { + rctResults := make([]rctIpldResult, 0) + if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil { + return nil, nil, nil, err + } + cids := make([]string, len(rctResults)) + rcts := make([][]byte, len(rctResults)) + txs := make([]common.Hash, len(rctResults)) + + for i, res := range rctResults { + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, nil, err + } + rcts[i] = nodeVal + txs[i] = common.HexToHash(res.TxHash) + } + + return cids, rcts, txs, nil +} + +// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash. +// cid returned corresponds to the leaf node data which contains the receipt. +func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) { + rctResults := make([]rctIpldResult, 0) + if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil { + return nil, nil, nil, err + } + cids := make([]string, len(rctResults)) + rcts := make([][]byte, len(rctResults)) + txs := make([]common.Hash, len(rctResults)) + + for i, res := range rctResults { + cids[i] = res.LeafCID + nodeVal, err := DecodeLeafNode(res.Data) + if err != nil { + return nil, nil, nil, err + } + rcts[i] = nodeVal + txs[i] = common.HexToHash(res.TxHash) + } + + return cids, rcts, txs, nil +} + +// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash +// TODO: ensure this handles deleted accounts appropriately +func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) { + accountResult := new(nodeInfo) + leafKey := crypto.Keccak256Hash(address.Bytes()) + if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil { + return "", nil, err + } + + if accountResult.NodeType == sdtypes.Removed.Int() { + return "", EmptyNodeValue, nil + } + + blockNumber, err := strconv.ParseUint(accountResult.BlockNumber, 10, 64) + if err != nil { + return "", nil, err + } + accountResult.Data, err = shared.FetchIPLD(r.db, accountResult.CID, blockNumber) + if err != nil { + return "", nil, err + } + + var i []interface{} + if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil { + return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return "", nil, fmt.Errorf("eth Retriever expected state leaf node rlp to decode into two elements") + } + return accountResult.CID, i[1].([]byte), nil +} + +// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash +func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) { + storageResult := new(nodeInfo) + stateLeafKey := crypto.Keccak256Hash(address.Bytes()) + storageHash := crypto.Keccak256Hash(key.Bytes()) + if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { + return "", nil, nil, err + } + if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { + return "", EmptyNodeValue, EmptyNodeValue, nil + } + + blockNumber, err := strconv.ParseUint(storageResult.BlockNumber, 10, 64) + if err != nil { + return "", nil, nil, err + } + storageResult.Data, err = shared.FetchIPLD(r.db, storageResult.CID, blockNumber) + if err != nil { + return "", nil, nil, err + } + + var i []interface{} + if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil { + err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error()) + return "", nil, nil, err + } + if len(i) != 2 { + return "", nil, nil, fmt.Errorf("eth Retriever expected storage leaf node rlp to decode into two elements") + } + return storageResult.CID, storageResult.Data, i[1].([]byte), nil +} diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/retriever_test.go similarity index 98% rename from pkg/eth/cid_retriever_test.go rename to pkg/eth/retriever_test.go index 2941da0d..b2bb27dd 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/retriever_test.go @@ -33,13 +33,13 @@ var _ = Describe("Retriever", func() { var ( db *sqlx.DB diffIndexer interfaces.StateDiffIndexer - retriever *eth.CIDRetriever + retriever *eth.Retriever ) BeforeEach(func() { db = shared.SetupDB() diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash()) - retriever = eth.NewCIDRetriever(db) + retriever = eth.NewRetriever(db) }) AfterEach(func() { shared.TearDownDB(db) diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index 2e5d40b9..37553009 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -17,28 +17,22 @@ package test_helpers import ( - "bytes" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "math/big" + "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/models" - "github.com/ethereum/go-ethereum/statediff/indexer/shared" testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" - blocks "github.com/ipfs/go-block-format" - "github.com/multiformats/go-multihash" - - "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" ) // Test variables @@ -75,10 +69,8 @@ var ( Extra: []byte{}, }, } - ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) - MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) - MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) - MockChildHeader = types.Header{ + MockBlock = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) + MockChildHeader = types.Header{ Time: 0, Number: new(big.Int).Add(BlockNumber, common.Big1), Root: common.HexToHash("0x0"), @@ -89,7 +81,6 @@ var ( ParentHash: MockBlock.Header().Hash(), } MockChild = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie)) - MockChildRlp, _ = rlp.EncodeToBytes(MockChild.Header()) Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594") @@ -157,41 +148,10 @@ var ( Index: 5, } - Tx1 = GetTxnRlp(0, MockTransactions) - Tx2 = GetTxnRlp(1, MockTransactions) - Tx3 = GetTxnRlp(2, MockTransactions) - Tx4 = GetTxnRlp(3, MockTransactions) - - rctCIDs, rctIPLDData, _ = GetRctLeafNodeData(MockReceipts) - HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) - HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID) - Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256) - Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID) - Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256) - Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID) - Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256) - Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID) - Trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256) - Trx4MhKey = shared.MultihashKeyFromCID(Trx4CID) - Rct1CID = rctCIDs[0] - Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID) - Rct2CID = rctCIDs[1] - Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID) - Rct3CID = rctCIDs[2] - Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID) - Rct4CID = rctCIDs[3] - Rct4MhKey = shared.MultihashKeyFromCID(Rct4CID) - State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) - State1MhKey = shared.MultihashKeyFromCID(State1CID) - State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) - State2MhKey = shared.MultihashKeyFromCID(State2CID) - StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) - StorageMhKey = shared.MultihashKeyFromCID(StorageCID) - Rct1IPLD = rctIPLDData[0] - Rct2IPLD = rctIPLDData[1] - Rct3IPLD = rctIPLDData[2] - Rct4IPLD = rctIPLDData[3] - MockTrxMeta = []models.TxModel{ + rctCIDs, _, _ = GetRctLeafNodeData(MockReceipts) + Rct1CID = rctCIDs[0] + Rct4CID = rctCIDs[3] + MockTrxMeta = []models.TxModel{ { CID: "", // This is empty until we go to publish to ipfs MhKey: "", @@ -229,48 +189,6 @@ var ( Data: []byte{}, }, } - MockTrxMetaPostPublsh = []models.TxModel{ - { - BlockNumber: "1", - CID: Trx1CID.String(), // This is empty until we go to publish to ipfs - MhKey: Trx1MhKey, - Src: SenderAddr.Hex(), - Dst: Address.String(), - Index: 0, - TxHash: MockTransactions[0].Hash().String(), - Data: []byte{}, - }, - { - BlockNumber: "1", - CID: Trx2CID.String(), - MhKey: Trx2MhKey, - Src: SenderAddr.Hex(), - Dst: AnotherAddress.String(), - Index: 1, - TxHash: MockTransactions[1].Hash().String(), - Data: []byte{}, - }, - { - BlockNumber: "1", - CID: Trx3CID.String(), - MhKey: Trx3MhKey, - Src: SenderAddr.Hex(), - Dst: "", - Index: 2, - TxHash: MockTransactions[2].Hash().String(), - Data: MockContractByteCode, - }, - { - BlockNumber: "1", - CID: Trx4CID.String(), - MhKey: Trx4MhKey, - Src: SenderAddr.Hex(), - Dst: AnotherAddress1.String(), - Index: 3, - TxHash: MockTransactions[3].Hash().String(), - Data: []byte{}, - }, - } MockRctMeta = []models.ReceiptModel{ { LeafCID: "", @@ -298,41 +216,6 @@ var ( }, } - MockRctMetaPostPublish = []models.ReceiptModel{ - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct1CID.String(), - LeafMhKey: Rct1MhKey, - Contract: "", - ContractHash: "", - }, - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct2CID.String(), - LeafMhKey: Rct2MhKey, - Contract: "", - ContractHash: "", - }, - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct3CID.String(), - LeafMhKey: Rct3MhKey, - Contract: ContractAddress.String(), - ContractHash: ContractHash, - }, - { - BlockNumber: "1", - HeaderID: MockBlock.Hash().String(), - LeafCID: Rct4CID.String(), - LeafMhKey: Rct4MhKey, - Contract: "", - ContractHash: "", - }, - } - // statediff data storageLocation = common.HexToHash("0") StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() @@ -401,24 +284,6 @@ var ( StorageNodes: []sdtypes.StorageNode{}, }, } - MockStateMetaPostPublish = []models.StateNodeModel{ - { - BlockNumber: "1", - CID: State1CID.String(), - MhKey: State1MhKey, - Path: []byte{'\x06'}, - NodeType: 2, - StateKey: common.BytesToHash(ContractLeafKey).Hex(), - }, - { - BlockNumber: "1", - CID: State2CID.String(), - MhKey: State2MhKey, - Path: []byte{'\x0c'}, - NodeType: 2, - StateKey: common.BytesToHash(AccountLeafKey).Hex(), - }, - } MockStorageNodes = map[string][]sdtypes.StorageNode{ contractPath: { { @@ -439,149 +304,6 @@ var ( StorageNodes: MockStorageNodes, StateNodes: MockStateNodes, } - MockConvertedPayloadForChild = eth.ConvertedPayload{ - TotalDifficulty: MockChild.Difficulty(), - Block: MockChild, - Receipts: MockReceipts, - TxMetaData: MockTrxMeta, - ReceiptMetaData: MockRctMeta, - StorageNodes: MockStorageNodes, - StateNodes: MockStateNodes, - } - - Reward = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts) - MockCIDWrapper = ð.CIDWrapper{ - BlockNumber: new(big.Int).Set(BlockNumber), - Header: models.HeaderModel{ - BlockNumber: "1", - BlockHash: MockBlock.Hash().String(), - ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000", - CID: HeaderCID.String(), - MhKey: HeaderMhKey, - TotalDifficulty: MockBlock.Difficulty().String(), - Reward: Reward.String(), - StateRoot: MockBlock.Root().String(), - RctRoot: MockBlock.ReceiptHash().String(), - TxRoot: MockBlock.TxHash().String(), - UncleRoot: MockBlock.UncleHash().String(), - Bloom: MockBlock.Bloom().Bytes(), - Timestamp: MockBlock.Time(), - TimesValidated: 1, - Coinbase: "0x0000000000000000000000000000000000000000", - }, - Transactions: MockTrxMetaPostPublsh, - Receipts: MockRctMetaPostPublish, - Uncles: []models.UncleModel{}, - StateNodes: MockStateMetaPostPublish, - StorageNodes: []models.StorageNodeWithStateKeyModel{ - { - BlockNumber: "1", - Path: []byte{}, - CID: StorageCID.String(), - MhKey: StorageMhKey, - NodeType: 2, - StateKey: common.BytesToHash(ContractLeafKey).Hex(), - StorageKey: common.BytesToHash(StorageLeafKey).Hex(), - }, - }, - } - - HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID) - Trx1IPLD, _ = blocks.NewBlockWithCid(Tx1, Trx1CID) - Trx2IPLD, _ = blocks.NewBlockWithCid(Tx2, Trx2CID) - Trx3IPLD, _ = blocks.NewBlockWithCid(Tx3, Trx3CID) - Trx4IPLD, _ = blocks.NewBlockWithCid(Tx4, Trx4CID) - State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID) - State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID) - StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID) - - MockIPLDs = eth.IPLDs{ - BlockNumber: new(big.Int).Set(BlockNumber), - Header: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: HeaderIPLD.RawData(), - Key: HeaderIPLD.Cid().String(), - }, - Transactions: []models.IPLDModel{ - { - BlockNumber: BlockNumber.String(), - Data: Trx1IPLD.RawData(), - Key: Trx1IPLD.Cid().String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Trx2IPLD.RawData(), - Key: Trx2IPLD.Cid().String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Trx3IPLD.RawData(), - Key: Trx3IPLD.Cid().String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Trx4IPLD.RawData(), - Key: Trx4IPLD.Cid().String(), - }, - }, - Receipts: []models.IPLDModel{ - { - BlockNumber: BlockNumber.String(), - Data: Rct1IPLD, - Key: Rct1CID.String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Rct2IPLD, - Key: Rct2CID.String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Rct3IPLD, - Key: Rct3CID.String(), - }, - { - BlockNumber: BlockNumber.String(), - Data: Rct4IPLD, - Key: Rct4CID.String(), - }, - }, - StateNodes: []eth.StateNode{ - { - StateLeafKey: common.BytesToHash(ContractLeafKey), - Type: sdtypes.Leaf, - IPLD: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: State1IPLD.RawData(), - Key: State1IPLD.Cid().String(), - }, - Path: []byte{'\x06'}, - }, - { - StateLeafKey: common.BytesToHash(AccountLeafKey), - Type: sdtypes.Leaf, - IPLD: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: State2IPLD.RawData(), - Key: State2IPLD.Cid().String(), - }, - Path: []byte{'\x0c'}, - }, - }, - StorageNodes: []eth.StorageNode{ - { - StateLeafKey: common.BytesToHash(ContractLeafKey), - StorageLeafKey: common.BytesToHash(StorageLeafKey), - Type: sdtypes.Leaf, - IPLD: models.IPLDModel{ - BlockNumber: BlockNumber.String(), - Data: StorageIPLD.RawData(), - Key: StorageIPLD.Cid().String(), - }, - Path: []byte{}, - }, - }, - } LondonBlockNum = new(big.Int).Add(BlockNumber, big.NewInt(2)) MockLondonHeader = types.Header{ @@ -741,21 +463,3 @@ func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts, return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr } - -func GetTxnRlp(num int, txs types.Transactions) []byte { - buf := new(bytes.Buffer) - txs.EncodeIndex(num, buf) - tx := make([]byte, buf.Len()) - copy(tx, buf.Bytes()) - buf.Reset() - return tx -} - -func GetRctRlp(num int, rcts types.Receipts) []byte { - buf := new(bytes.Buffer) - rcts.EncodeIndex(num, buf) - rct := make([]byte, buf.Len()) - copy(rct, buf.Bytes()) - buf.Reset() - return rct -} diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index afcbd0f3..1f73b606 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -1009,7 +1009,7 @@ func (r *Resolver) GetStorageAt(ctx context.Context, args struct { Contract common.Address Slot common.Hash }) (*StorageResult, error) { - cid, ipldBlock, rlpValue, err := r.backend.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash) + cid, ipldBlock, rlpValue, err := r.backend.Retriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(args.Contract, args.Slot, args.BlockHash) if err != nil { if err == sql.ErrNoRows { diff --git a/test_config/test_config.go b/test_config/test_config.go index 982f9b44..de67b704 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -18,6 +18,7 @@ package test_config import ( "errors" + "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/spf13/viper" From 573a3dc991d02b41106202a0b4fda51516bd6272 Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 7 Mar 2023 21:35:18 -0600 Subject: [PATCH 05/14] node_type => removed --- pkg/eth/retriever.go | 12 ++++++------ pkg/eth/sql.go | 17 ++++++----------- pkg/eth/types.go | 41 ----------------------------------------- 3 files changed, 12 insertions(+), 58 deletions(-) diff --git a/pkg/eth/retriever.go b/pkg/eth/retriever.go index e9a04622..296f14ca 100644 --- a/pkg/eth/retriever.go +++ b/pkg/eth/retriever.go @@ -445,7 +445,7 @@ func DecodeLeafNode(node []byte) ([]byte, error) { // RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number. // cid returned corresponds to the leaf node data which contains the receipt. func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) { - rctResults := make([]rctIpldResult, 0) + rctResults := make([]ipldResult, 0) if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil { return nil, nil, nil, err } @@ -454,7 +454,7 @@ func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint6 txs := make([]common.Hash, len(rctResults)) for i, res := range rctResults { - cids[i] = res.LeafCID + cids[i] = res.CID nodeVal, err := DecodeLeafNode(res.Data) if err != nil { return nil, nil, nil, err @@ -469,7 +469,7 @@ func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint6 // RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash. // cid returned corresponds to the leaf node data which contains the receipt. func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) { - rctResults := make([]rctIpldResult, 0) + rctResults := make([]ipldResult, 0) if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil { return nil, nil, nil, err } @@ -478,7 +478,7 @@ func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ( txs := make([]common.Hash, len(rctResults)) for i, res := range rctResults { - cids[i] = res.LeafCID + cids[i] = res.CID nodeVal, err := DecodeLeafNode(res.Data) if err != nil { return nil, nil, nil, err @@ -499,7 +499,7 @@ func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, return "", nil, err } - if accountResult.NodeType == sdtypes.Removed.Int() { + if accountResult.Removed { return "", EmptyNodeValue, nil } @@ -530,7 +530,7 @@ func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } - if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { + if storageResult.StateLeafRemoved || storageResult.Removed { return "", EmptyNodeValue, EmptyNodeValue, nil } diff --git a/pkg/eth/sql.go b/pkg/eth/sql.go index 20283cdc..3e41bfea 100644 --- a/pkg/eth/sql.go +++ b/pkg/eth/sql.go @@ -93,7 +93,7 @@ const ( ) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` - RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.node_type + RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.removed FROM eth.state_cids INNER JOIN eth.header_cids ON ( state_cids.header_id = header_cids.block_hash @@ -106,7 +106,7 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveFilteredGQLLogs = `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.header_id as block_hash, + RetrieveFilteredGQLLogs = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.header_id as block_hash, eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, data, eth.receipt_cids.cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash @@ -117,7 +117,7 @@ const ( AND log_cids.cid = blocks.key AND log_cids.block_number = blocks.block_number AND receipt_cids.header_id = $1` - RetrieveFilteredLogs = `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, + RetrieveFilteredLogs = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, eth.receipt_cids.cid as cid, eth.receipt_cids.post_status, header_cids.block_hash @@ -130,15 +130,10 @@ const ( AND receipt_cids.block_number = transaction_cids.block_number AND transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number` - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, block_number, node_type, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` + // TODO: replace me with new query now that we have the correct storage index + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, block_number, removed, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` ) -type rctIpldResult struct { - LeafCID string `db:"cid"` - Data []byte `db:"data"` - TxHash string `db:"tx_hash"` -} - type ipldResult struct { CID string `db:"cid"` Data []byte `db:"data"` @@ -149,6 +144,6 @@ type nodeInfo struct { CID string `db:"cid"` BlockNumber string `db:"block_number"` Data []byte `db:"data"` - NodeType int `db:"node_type"` + Removed bool `db:"removed"` StateLeafRemoved bool `db:"state_leaf_removed"` } diff --git a/pkg/eth/types.go b/pkg/eth/types.go index d20e21ed..c74c959c 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -193,47 +193,6 @@ func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Mes return msg, nil } -// IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server -// Returned by IPLDFetcher and ResponseFilterer -type IPLDs struct { - BlockNumber *big.Int - TotalDifficulty *big.Int - Header models.IPLDModel - Uncles []models.IPLDModel - Transactions []models.IPLDModel - Receipts []models.IPLDModel - StateNodes []StateNode - StorageNodes []StorageNode -} - -type StateNode struct { - Type sdtypes.NodeType - StateLeafKey common.Hash - Path []byte - IPLD models.IPLDModel -} - -type StorageNode struct { - Type sdtypes.NodeType - StateLeafKey common.Hash - StorageLeafKey common.Hash - Path []byte - IPLD models.IPLDModel -} - -// CIDWrapper is used to direct fetching of IPLDs from IPFS -// Returned by CIDRetriever -// Passed to IPLDFetcher -type CIDWrapper struct { - BlockNumber *big.Int - Header models.HeaderModel - Uncles []models.UncleModel - Transactions []models.TxModel - Receipts []models.ReceiptModel - StateNodes []models.StateNodeModel - StorageNodes []models.StorageNodeWithStateKeyModel -} - // ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers // Returned by PayloadConverter // Passed to IPLDPublisher and ResponseFilterer From 4c29841c21870049de7a3fa706867dc10af99452 Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 7 Mar 2023 21:48:58 -0600 Subject: [PATCH 06/14] internalize some node type utils from geth as they are no longer used over there in v5 --- pkg/eth/backend.go | 10 ++-- pkg/eth/backend_utils.go | 28 +--------- pkg/eth/helpers.go | 17 ------ pkg/eth/node_types.go | 116 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 48 deletions(-) create mode 100644 pkg/eth/node_types.go diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 03688d0f..00f1216d 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -45,8 +45,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" - sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" - sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" ) @@ -953,7 +951,7 @@ func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSlice if depthReached > metaData.maxDepth { metaData.maxDepth = depthReached } - if node.NodeType == sdtypes.Leaf { + if node.NodeType == Leaf { metaData.leafCount++ } leavesFetchTime += leafFetchTime @@ -995,7 +993,7 @@ func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSlice if depthReached > metaData.maxDepth { metaData.maxDepth = depthReached } - if node.NodeType == sdtypes.Leaf { + if node.NodeType == Leaf { metaData.leafCount++ } @@ -1037,7 +1035,7 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice continue } - node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB()) + node, nodeElements, err := ResolveNodeIt(it, b.StateDatabase.TrieDB()) if err != nil { return err } @@ -1052,7 +1050,7 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice if depthReached > metaData.maxDepth { metaData.maxDepth = depthReached } - if node.NodeType == sdtypes.Leaf { + if node.NodeType == Leaf { metaData.leafCount++ } leavesFetchTime += leafFetchTime diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index 68655604..bc200cb3 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -35,8 +35,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" - sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" ) @@ -334,7 +332,7 @@ func fillSliceNodeData( ethDB ethdb.KeyValueReader, nodesMap map[string]string, leavesMap map[string]GetSliceResponseAccount, - node sdtypes.StateNode, + node StateNode, nodeElements []interface{}, storage bool, ) (int64, error) { @@ -344,7 +342,7 @@ func fillSliceNodeData( // Extract account data if it's a Leaf node leafStartTime := makeTimestamp() - if node.NodeType == sdtypes.Leaf && !storage { + if node.NodeType == Leaf && !storage { stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements) if err != nil { return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error()) @@ -362,7 +360,7 @@ func fillSliceNodeData( return makeTimestamp() - leafStartTime, nil } -func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) { +func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node StateNode, nodeElements []interface{}) (string, string, []byte, error) { var account types.StateAccount if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) @@ -387,23 +385,3 @@ func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNo return stateLeafKeyString, storageRootString, codeBytes, nil } - -func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) { - nodePath := make([]byte, len(path)) - copy(nodePath, path) - - var nodeElements []interface{} - if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return sdtypes.StateNode{}, nil, err - } - - ty, err := sdtrie.CheckKeyType(nodeElements) - if err != nil { - return sdtypes.StateNode{}, nil, err - } - return sdtypes.StateNode{ - NodeType: ty, - Path: nodePath, - NodeValue: node, - }, nodeElements, nil -} diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index cce23579..fcbf3cac 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -18,25 +18,8 @@ package eth import ( "time" - - sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) -func ResolveToNodeType(nodeType int) sdtypes.NodeType { - switch nodeType { - case 0: - return sdtypes.Branch - case 1: - return sdtypes.Extension - case 2: - return sdtypes.Leaf - case 3: - return sdtypes.Removed - default: - return sdtypes.Unknown - } -} - // Timestamp in milliseconds func makeTimestamp() int64 { return time.Now().UnixNano() / int64(time.Millisecond) diff --git a/pkg/eth/node_types.go b/pkg/eth/node_types.go new file mode 100644 index 00000000..f0f1ba15 --- /dev/null +++ b/pkg/eth/node_types.go @@ -0,0 +1,116 @@ +package eth + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// NodeType for explicitly setting type of node +type NodeType string + +const ( + Unknown NodeType = "Unknown" + Branch NodeType = "Branch" + Extension NodeType = "Extension" + Leaf NodeType = "Leaf" + Removed NodeType = "Removed" // used to represent paths which have been emptied +) + +func (n NodeType) Int() int { + switch n { + case Branch: + return 0 + case Extension: + return 1 + case Leaf: + return 2 + case Removed: + return 3 + default: + return -1 + } +} + +// CheckKeyType checks what type of key we have +func CheckKeyType(elements []interface{}) (NodeType, error) { + if len(elements) > 2 { + return Branch, nil + } + if len(elements) < 2 { + return Unknown, fmt.Errorf("node cannot be less than two elements in length") + } + switch elements[0].([]byte)[0] / 16 { + case '\x00': + return Extension, nil + case '\x01': + return Extension, nil + case '\x02': + return Leaf, nil + case '\x03': + return Leaf, nil + default: + return Unknown, fmt.Errorf("unknown hex prefix") + } +} + +// StateNode holds the data for a single state diff node +type StateNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + StorageNodes []StorageNode `json:"storage"` + LeafKey []byte `json:"leafKey"` +} + +// StorageNode holds the data for a single storage diff node +type StorageNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + LeafKey []byte `json:"leafKey"` +} + +func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (StateNode, []interface{}, error) { + nodePath := make([]byte, len(path)) + copy(nodePath, path) + + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return StateNode{}, nil, err + } + + ty, err := CheckKeyType(nodeElements) + if err != nil { + return StateNode{}, nil, err + } + return StateNode{ + NodeType: ty, + Path: nodePath, + NodeValue: node, + }, nodeElements, nil +} + +// ResolveNodeIt return the state diff node pointed by the iterator. +func ResolveNodeIt(it trie.NodeIterator, trieDB *trie.Database) (StateNode, []interface{}, error) { + nodePath := make([]byte, len(it.Path())) + copy(nodePath, it.Path()) + node, err := trieDB.Node(it.Hash()) + if err != nil { + return StateNode{}, nil, err + } + var nodeElements []interface{} + if err = rlp.DecodeBytes(node, &nodeElements); err != nil { + return StateNode{}, nil, err + } + ty, err := CheckKeyType(nodeElements) + if err != nil { + return StateNode{}, nil, err + } + return StateNode{ + NodeType: ty, + Path: nodePath, + NodeValue: node, + }, nodeElements, nil +} From ba5cdecf93e2e7c8235c7cdf10a3057a96e061da Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 13 Mar 2023 07:51:03 -0500 Subject: [PATCH 07/14] new backend methods using ipld_eth_statedb.StateDB --- go.mod | 37 ++++++++++++------------ go.sum | 56 ++++++++++++++++++++++++------------ pkg/eth/backend.go | 72 ++++++++++++++++++++++++++++++++++++++++------ pkg/eth/sql.go | 5 ++-- 4 files changed, 123 insertions(+), 47 deletions(-) diff --git a/go.mod b/go.mod index ca0fd5f6..f25d5c81 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,10 @@ require ( github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha github.com/cerc-io/go-eth-state-node-iterator v1.1.9 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha + github.com/cerc-io/ipld-eth-statedb v0.0.1-alpha github.com/ethereum/go-ethereum v1.10.26 + github.com/google/uuid v1.3.0 github.com/graph-gophers/graphql-go v1.3.0 - github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-ds-help v1.1.0 @@ -17,7 +18,6 @@ require ( github.com/lib/pq v1.10.6 github.com/machinebox/graphql v0.2.2 github.com/mailgun/groupcache/v2 v2.3.0 - github.com/multiformats/go-multihash v0.2.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.19.0 github.com/prometheus/client_golang v1.12.1 @@ -77,7 +77,6 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/graphql-go/graphql v0.7.9 // indirect @@ -97,6 +96,7 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.0.0 // indirect github.com/ipfs/go-bitswap v0.8.0 // indirect + github.com/ipfs/go-block-format v0.0.3 // indirect github.com/ipfs/go-blockservice v0.4.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.5.1 // indirect @@ -139,14 +139,14 @@ require ( github.com/ipld/go-codec-dagpb v1.4.0 // indirect github.com/ipld/go-ipld-prime v0.17.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.12.1 // indirect + github.com/jackc/pgconn v1.14.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.11.0 // indirect - github.com/jackc/pgx/v4 v4.16.1 // indirect - github.com/jackc/puddle v1.2.1 // indirect + github.com/jackc/pgproto3/v2 v2.3.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect + github.com/jackc/pgx/v4 v4.18.1 // indirect + github.com/jackc/puddle v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -211,6 +211,7 @@ require ( github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.0 // indirect github.com/multiformats/go-multicodec v0.5.0 // indirect + github.com/multiformats/go-multihash v0.2.0 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect github.com/multiformats/go-varint v0.0.6 // indirect github.com/nxadm/tail v1.4.8 // indirect @@ -244,8 +245,8 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect - github.com/stretchr/objx v0.2.0 // indirect - github.com/stretchr/testify v1.7.2 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/testify v1.8.1 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/thoas/go-funk v0.9.2 // indirect @@ -275,14 +276,14 @@ require ( go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect - golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/crypto v0.6.0 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/net v0.6.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.org/x/tools v0.1.10 // indirect + golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index 7d885c28..11a7b12e 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha h1:gesMZEbNU+fcAMc github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha h1:5iqNXeitkj3g7FxyKK/Pz+1HN7Ac9JZzCRj3Lv+uHiw= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha/go.mod h1:dPscFRMvTWPKnoZ4U0D9v4bsrw6XdH7sOp8hUrVzOWA= +github.com/cerc-io/ipld-eth-statedb v0.0.1-alpha h1:oUSxncZCzMO4JXep/CoP+u1lgpdf97mbCUYWnjcHw6w= +github.com/cerc-io/ipld-eth-statedb v0.0.1-alpha/go.mod h1:914KQXnRylWQxRQOvlIr74NmGw7t1CyJJaEr5VSKW8E= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -769,8 +771,9 @@ github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7 github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgconn v1.14.0 h1:vrbA9Ud87g6JdFWkHTJXppVce58qPIdP7N8y0Ml/A7Q= +github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -789,11 +792,13 @@ github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0= +github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -803,8 +808,9 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= @@ -815,16 +821,18 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0= +github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1734,8 +1742,10 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1744,8 +1754,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1829,6 +1841,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -1956,8 +1969,8 @@ golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2002,8 +2015,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2067,8 +2080,10 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220517181318-183a9ca12b87/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2093,8 +2108,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2197,11 +2213,14 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2210,8 +2229,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2290,8 +2310,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 00f1216d..b868393b 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -25,10 +25,13 @@ import ( "math/big" "time" + "github.com/jackc/pgx/v4/pgxpool" + validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" + ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" @@ -64,7 +67,7 @@ const ( FROM canonical_header_hash($1) AS block_hash WHERE block_hash IS NOT NULL` RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON ( + INNER JOIN ipld.blocks ON ( header_cids.cid = blocks.key AND header_cids.block_number = blocks.block_number ) @@ -90,7 +93,7 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` + RetrieveCodeByMhKey = `SELECT data FROM ipld.blocks WHERE key = $1` ) const ( @@ -107,6 +110,8 @@ type Backend struct { // ethereum interfaces EthDB ethdb.Database StateDatabase state.Database + // We'll use this state.Database for eth_call and any place we don't need trie access + IpldStateDatabase ipld_eth_statedb.Database Config *Config } @@ -119,7 +124,7 @@ type Config struct { GroupCacheConfig *shared.GroupCacheConfig } -func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { +func NewEthBackend(db *sqlx.DB, pgxdb *pgxpool.Pool, c *Config) (*Backend, error) { gcc := c.GroupCacheConfig groupName := gcc.StateDB.Name @@ -135,13 +140,17 @@ func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { }) logStateDBStatsOnTimer(ethDB.(*ipfsethdb.Database), gcc) - + ipldStateDB, err := ipld_eth_statedb.NewStateDatabaseWithPool(pgxdb) + if err != nil { + return nil, err + } return &Backend{ - DB: db, - Retriever: r, - EthDB: ethDB, - StateDatabase: state.NewDatabase(ethDB), - Config: c, + DB: db, + Retriever: r, + EthDB: ethDB, + StateDatabase: state.NewDatabase(ethDB), + IpldStateDatabase: ipldStateDB, + Config: c, }, nil } @@ -623,6 +632,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64) } // StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash +// TODO: this needs to be updated to use the new StateDB implementation func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { if blockNr, ok := blockNrOrHash.Number(); ok { return b.StateAndHeaderByNumber(ctx, blockNr) @@ -648,6 +658,50 @@ func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHas return nil, nil, errors.New("invalid arguments; neither block nor hash specified") } +// IPLDStateDBAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash +func (b *Backend) IPLDStateDBAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (vm.StateDB, *types.Header, error) { + if blockNr, ok := blockNrOrHash.Number(); ok { + return b.StateAndHeaderByNumber(ctx, blockNr) + } + if hash, ok := blockNrOrHash.Hash(); ok { + header, err := b.HeaderByHash(ctx, hash) + if err != nil { + return nil, nil, err + } + if header == nil { + return nil, nil, errors.New("header for hash not found") + } + canonicalHash, err := b.GetCanonicalHash(header.Number.Uint64()) + if err != nil { + return nil, nil, err + } + if blockNrOrHash.RequireCanonical && canonicalHash != hash { + return nil, nil, errors.New("hash is not currently canonical") + } + stateDB, err := ipld_eth_statedb.New(header.Root, b.IpldStateDatabase) + return stateDB, header, err + } + return nil, nil, errors.New("invalid arguments; neither block nor hash specified") +} + +// IPLDStateDBAndHeaderByNumber returns the statedb and header for a provided block number +func (b *Backend) IPLDStateDBAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (vm.StateDB, *types.Header, error) { + // Pending state is only known by the miner + if number == rpc.PendingBlockNumber { + return nil, nil, errPendingBlockNumber + } + // Otherwise resolve the block number and return its state + header, err := b.HeaderByNumber(ctx, number) + if err != nil { + return nil, nil, err + } + if header == nil { + return nil, nil, errors.New("header not found") + } + stateDb, err := ipld_eth_statedb.New(header.Root, b.IpldStateDatabase) + return stateDb, header, err +} + // StateAndHeaderByNumber returns the statedb and header for a provided block number func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner diff --git a/pkg/eth/sql.go b/pkg/eth/sql.go index 3e41bfea..82f3e3d0 100644 --- a/pkg/eth/sql.go +++ b/pkg/eth/sql.go @@ -93,6 +93,7 @@ const ( ) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` + // TODO: join on ipld.blocks and return IPLD object in this query instead of round tripping back to ipld.blocks RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.block_number, state_cids.removed FROM eth.state_cids INNER JOIN eth.header_cids ON ( @@ -130,8 +131,7 @@ const ( AND receipt_cids.block_number = transaction_cids.block_number AND transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number` - // TODO: replace me with new query now that we have the correct storage index - RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, block_number, removed, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` + RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, val, block_number, removed, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` ) type ipldResult struct { @@ -142,6 +142,7 @@ type ipldResult struct { type nodeInfo struct { CID string `db:"cid"` + Value []byte `db:"val"` BlockNumber string `db:"block_number"` Data []byte `db:"data"` Removed bool `db:"removed"` From aab5a2ef0bd475462b2ab8199892e10a0fd66ac1 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 13 Mar 2023 11:25:19 -0500 Subject: [PATCH 08/14] use ipld-eth-statedb for eth_call and graphql endpoints --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/debug/backend.go | 1 + pkg/eth/api.go | 7 ++++--- pkg/eth/backend.go | 43 +++++++++++++++++++++--------------------- pkg/graphql/graphql.go | 5 ++--- pkg/graphql/service.go | 2 +- 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index f25d5c81..f31c5dc9 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,14 @@ require ( github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha github.com/cerc-io/go-eth-state-node-iterator v1.1.9 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha - github.com/cerc-io/ipld-eth-statedb v0.0.1-alpha + github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha github.com/ethereum/go-ethereum v1.10.26 github.com/google/uuid v1.3.0 github.com/graph-gophers/graphql-go v1.3.0 github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-ds-help v1.1.0 + github.com/jackc/pgx/v4 v4.18.1 github.com/jmoiron/sqlx v1.3.5 github.com/joho/godotenv v1.4.0 github.com/lib/pq v1.10.6 @@ -145,7 +146,6 @@ require ( github.com/jackc/pgproto3/v2 v2.3.2 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect - github.com/jackc/pgx/v4 v4.18.1 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect diff --git a/go.sum b/go.sum index 11a7b12e..4d78b373 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,8 @@ github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha h1:gesMZEbNU+fcAMc github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha h1:5iqNXeitkj3g7FxyKK/Pz+1HN7Ac9JZzCRj3Lv+uHiw= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha/go.mod h1:dPscFRMvTWPKnoZ4U0D9v4bsrw6XdH7sOp8hUrVzOWA= -github.com/cerc-io/ipld-eth-statedb v0.0.1-alpha h1:oUSxncZCzMO4JXep/CoP+u1lgpdf97mbCUYWnjcHw6w= -github.com/cerc-io/ipld-eth-statedb v0.0.1-alpha/go.mod h1:914KQXnRylWQxRQOvlIr74NmGw7t1CyJJaEr5VSKW8E= +github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha h1:Dj6XG6T14iHRBMFYHKcnMkNsdxZ7xqAaah2Z+QiOe1k= +github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha/go.mod h1:914KQXnRylWQxRQOvlIr74NmGw7t1CyJJaEr5VSKW8E= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/pkg/debug/backend.go b/pkg/debug/backend.go index bf4a1b40..258d1708 100644 --- a/pkg/debug/backend.go +++ b/pkg/debug/backend.go @@ -42,6 +42,7 @@ type Backend struct { } // StateAtBlock retrieves the state database associated with a certain block +// We can't sub in our ipld-eth-statedb here because to match the expected interface we need to return *state.StateDB not vm.StateDB func (b *Backend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, error) { rpcBlockNumber := rpc.BlockNumber(block.NumberU64()) statedb, _, err := b.StateAndHeaderByNumberOrHash(ctx, rpc.BlockNumberOrHashWithNumber(rpcBlockNumber)) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 96196df9..fa932e48 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -28,13 +28,13 @@ import ( "time" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" + ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/filters" @@ -875,6 +875,7 @@ func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, s return nil, err } +// this continues to use ipfs-ethdb based geth StateDB as it requires trie access func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { @@ -977,7 +978,7 @@ type OverrideAccount struct { type StateOverride map[common.Address]OverrideAccount // Apply overrides the fields of specified accounts into the given state. -func (diff *StateOverride) Apply(state *state.StateDB) error { +func (diff *StateOverride) Apply(state *ipld_eth_statedb.StateDB) error { if diff == nil { return nil } @@ -1054,7 +1055,7 @@ func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.Bl log.Debugxf(ctx, "Executing EVM call finished %s runtime %s", time.Now().String(), time.Since(start).String()) }(time.Now()) - state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + state, header, err := b.IPLDStateDBAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err } diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index b868393b..2f8432e3 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -632,7 +632,6 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash, number uint64) } // StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash -// TODO: this needs to be updated to use the new StateDB implementation func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { if blockNr, ok := blockNrOrHash.Number(); ok { return b.StateAndHeaderByNumber(ctx, blockNr) @@ -659,9 +658,9 @@ func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHas } // IPLDStateDBAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash -func (b *Backend) IPLDStateDBAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (vm.StateDB, *types.Header, error) { +func (b *Backend) IPLDStateDBAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*ipld_eth_statedb.StateDB, *types.Header, error) { if blockNr, ok := blockNrOrHash.Number(); ok { - return b.StateAndHeaderByNumber(ctx, blockNr) + return b.IPLDStateDBAndHeaderByNumber(ctx, blockNr) } if hash, ok := blockNrOrHash.Hash(); ok { header, err := b.HeaderByHash(ctx, hash) @@ -684,24 +683,6 @@ func (b *Backend) IPLDStateDBAndHeaderByNumberOrHash(ctx context.Context, blockN return nil, nil, errors.New("invalid arguments; neither block nor hash specified") } -// IPLDStateDBAndHeaderByNumber returns the statedb and header for a provided block number -func (b *Backend) IPLDStateDBAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (vm.StateDB, *types.Header, error) { - // Pending state is only known by the miner - if number == rpc.PendingBlockNumber { - return nil, nil, errPendingBlockNumber - } - // Otherwise resolve the block number and return its state - header, err := b.HeaderByNumber(ctx, number) - if err != nil { - return nil, nil, err - } - if header == nil { - return nil, nil, errors.New("header not found") - } - stateDb, err := ipld_eth_statedb.New(header.Root, b.IpldStateDatabase) - return stateDb, header, err -} - // StateAndHeaderByNumber returns the statedb and header for a provided block number func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner @@ -720,6 +701,24 @@ func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNu return stateDb, header, err } +// IPLDStateDBAndHeaderByNumber returns the statedb and header for a provided block number +func (b *Backend) IPLDStateDBAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*ipld_eth_statedb.StateDB, *types.Header, error) { + // Pending state is only known by the miner + if number == rpc.PendingBlockNumber { + return nil, nil, errPendingBlockNumber + } + // Otherwise resolve the block number and return its state + header, err := b.HeaderByNumber(ctx, number) + if err != nil { + return nil, nil, err + } + if header == nil { + return nil, nil, errors.New("header not found") + } + stateDb, err := ipld_eth_statedb.New(header.Root, b.IpldStateDatabase) + return stateDb, header, err +} + // GetCanonicalHash gets the canonical hash for the provided number, if there is one func (b *Backend) GetCanonicalHash(number uint64) (common.Hash, error) { var hashResult string @@ -741,7 +740,7 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) { } // GetEVM constructs and returns a vm.EVM -func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { +func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state vm.StateDB, header *types.Header) (*vm.EVM, func() error, error) { vmError := func() error { return nil } txContext := core.NewEVMTxContext(msg) blockContext := core.NewEVMBlockContext(header, b, nil) diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index 1f73b606..715cebb9 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/rlp" @@ -51,8 +50,8 @@ type Account struct { } // getState fetches the StateDB object for an account. -func (a *Account) getState(ctx context.Context) (*state.StateDB, error) { - state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) +func (a *Account) getState(ctx context.Context) (*ipld_eth_statedb.StateDB, error) { + state, _, err := a.backend.IPLDStateDBAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) return state, err } diff --git a/pkg/graphql/service.go b/pkg/graphql/service.go index 95211d7a..f3df0d48 100644 --- a/pkg/graphql/service.go +++ b/pkg/graphql/service.go @@ -81,7 +81,7 @@ func (s *Service) Start(server *p2p.Server) error { return nil } -// newHandler returns a new `http.Handler` that will answer GraphQL queries. +// NewHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. func NewHandler(backend *eth.Backend) (http.Handler, error) { q := Resolver{backend} From df8bec1ad030d1a210de3e08a790b313e314c315 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 13 Mar 2023 12:41:08 -0500 Subject: [PATCH 09/14] adapt uncle retrieval methods to work for v5 --- pkg/eth/backend.go | 44 +++++++++++------------------------------- pkg/eth/retriever.go | 36 ++++++++++++---------------------- pkg/eth/sql.go | 6 ++++-- pkg/graphql/graphql.go | 1 + 4 files changed, 28 insertions(+), 59 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 2f8432e3..b5595778 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -374,21 +374,11 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo return nil, err } - // When num. of uncles = 2, - // Check if calculated uncle hash matches the one in header - // If not, re-order the two uncles - // Assumption: Max num. of uncles in mainnet = 2 - if len(uncles) == 2 { - uncleHash := types.CalcUncleHash(uncles) - if uncleHash != header.UncleHash { - uncles[0], uncles[1] = uncles[1], uncles[0] - - uncleHash = types.CalcUncleHash(uncles) - // Check if uncle hash matches after re-ordering - if uncleHash != header.UncleHash { - log.Error("uncle hash mismatch for block hash: ", hash.Hex()) - } - } + // We should not have any non-determinism in the ordering of the uncles returned to us now + uncleHash := types.CalcUncleHash(uncles) + // Check if uncle hash matches expected hash + if uncleHash != header.UncleHash { + log.Error("uncle hash mismatch for block hash: ", hash.Hex()) } // Fetch transactions @@ -427,15 +417,9 @@ func (b *Backend) GetUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]*types. return nil, err } - uncles := make([]*types.Header, len(uncleBytes)) - for i, bytes := range uncleBytes { - var uncle types.Header - err = rlp.DecodeBytes(bytes, &uncle) - if err != nil { - return nil, err - } - - uncles[i] = &uncle + uncles := make([]*types.Header, 0) + if err := rlp.DecodeBytes(uncleBytes, uncles); err != nil { + return nil, err } return uncles, nil @@ -448,15 +432,9 @@ func (b *Backend) GetUnclesByBlockHashAndNumber(tx *sqlx.Tx, hash common.Hash, n return nil, err } - uncles := make([]*types.Header, len(uncleBytes)) - for i, bytes := range uncleBytes { - var uncle types.Header - err = rlp.DecodeBytes(bytes, &uncle) - if err != nil { - return nil, err - } - - uncles[i] = &uncle + uncles := make([]*types.Header, 0) + if err := rlp.DecodeBytes(uncleBytes, uncles); err != nil { + return nil, err } return uncles, nil diff --git a/pkg/eth/retriever.go b/pkg/eth/retriever.go index 296f14ca..ee68185e 100644 --- a/pkg/eth/retriever.go +++ b/pkg/eth/retriever.go @@ -365,34 +365,22 @@ func (r *Retriever) RetrieveHeaderByHash(tx *sqlx.Tx, hash common.Hash) (string, return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex()) } -// RetrieveUncles returns the cids and rlp bytes for the uncles corresponding to the provided block hash, number (of non-omner root block) -func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) { - uncleResults := make([]ipldResult, 0) - if err := tx.Select(&uncleResults, RetrieveUnclesPgStr, hash.Hex(), number); err != nil { - return nil, nil, err +// RetrieveUncles returns the cid and rlp bytes for the uncle list corresponding to the provided block hash, number (of non-omner root block) +func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) (string, []byte, error) { + uncleResult := new(ipldResult) + if err := tx.Select(uncleResult, RetrieveUnclesPgStr, hash.Hex(), number); err != nil { + return "", nil, err } - cids := make([]string, len(uncleResults)) - uncles := make([][]byte, len(uncleResults)) - for i, res := range uncleResults { - cids[i] = res.CID - uncles[i] = res.Data - } - return cids, uncles, nil + return uncleResult.CID, uncleResult.Data, nil } -// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block) -func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) { - uncleResults := make([]ipldResult, 0) - if err := tx.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil { - return nil, nil, err +// RetrieveUnclesByBlockHash returns the cid and rlp bytes for the uncle list corresponding to the provided block hash (of non-omner root block) +func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) { + uncleResult := new(ipldResult) + if err := tx.Select(uncleResult, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil { + return "", nil, err } - cids := make([]string, len(uncleResults)) - uncles := make([][]byte, len(uncleResults)) - for i, res := range uncleResults { - cids[i] = res.CID - uncles[i] = res.Data - } - return cids, uncles, nil + return uncleResult.CID, uncleResult.Data, nil } // RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number diff --git a/pkg/eth/sql.go b/pkg/eth/sql.go index 82f3e3d0..8da8cd8a 100644 --- a/pkg/eth/sql.go +++ b/pkg/eth/sql.go @@ -20,7 +20,8 @@ const ( ) WHERE header_cids.block_hash = $1 AND header_cids.block_number = $2 - ORDER BY uncle_cids.parent_hash` + ORDER BY uncle_cids.parent_hash + LIMIT 1` RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids INNER JOIN eth.header_cids ON ( @@ -32,7 +33,8 @@ const ( AND uncle_cids.block_number = blocks.block_number ) WHERE header_cids.block_hash = $1 - ORDER BY uncle_cids.parent_hash` + ORDER BY uncle_cids.parent_hash + LIMIT 1` RetrieveTransactionsPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids INNER JOIN eth.header_cids ON ( diff --git a/pkg/graphql/graphql.go b/pkg/graphql/graphql.go index 715cebb9..da6dd827 100644 --- a/pkg/graphql/graphql.go +++ b/pkg/graphql/graphql.go @@ -36,6 +36,7 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" + ipld_eth_statedb "github.com/cerc-io/ipld-eth-statedb" ) var ( From 4678b6a73395ec3b33b937e627459860fad16602 Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 13 Mar 2023 18:23:49 -0500 Subject: [PATCH 10/14] ipld-eth-statedb initialization/instantiation --- go.mod | 4 +++- go.sum | 2 -- pkg/eth/backend.go | 8 +++----- pkg/serve/service.go | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index f31c5dc9..00bc9781 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-ds-help v1.1.0 - github.com/jackc/pgx/v4 v4.18.1 github.com/jmoiron/sqlx v1.3.5 github.com/joho/godotenv v1.4.0 github.com/lib/pq v1.10.6 @@ -146,6 +145,7 @@ require ( github.com/jackc/pgproto3/v2 v2.3.2 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect + github.com/jackc/pgx/v4 v4.18.1 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect @@ -295,3 +295,5 @@ require ( ) replace github.com/ethereum/go-ethereum v1.10.26 => github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha + +replace github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha => /Users/iannorden/go/src/github.com/cerc-io/ipld-eth-statedb diff --git a/go.sum b/go.sum index 4d78b373..1733dac5 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,6 @@ github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha h1:gesMZEbNU+fcAMc github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha h1:5iqNXeitkj3g7FxyKK/Pz+1HN7Ac9JZzCRj3Lv+uHiw= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha/go.mod h1:dPscFRMvTWPKnoZ4U0D9v4bsrw6XdH7sOp8hUrVzOWA= -github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha h1:Dj6XG6T14iHRBMFYHKcnMkNsdxZ7xqAaah2Z+QiOe1k= -github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha/go.mod h1:914KQXnRylWQxRQOvlIr74NmGw7t1CyJJaEr5VSKW8E= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index b5595778..fd25a5c7 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -25,8 +25,6 @@ import ( "math/big" "time" - "github.com/jackc/pgx/v4/pgxpool" - validator "github.com/cerc-io/eth-ipfs-state-validator/v4/pkg" ipfsethdb "github.com/cerc-io/ipfs-ethdb/v4/postgres" "github.com/cerc-io/ipld-eth-server/v4/pkg/log" @@ -111,7 +109,7 @@ type Backend struct { EthDB ethdb.Database StateDatabase state.Database // We'll use this state.Database for eth_call and any place we don't need trie access - IpldStateDatabase ipld_eth_statedb.Database + IpldStateDatabase ipld_eth_statedb.StateDatabase Config *Config } @@ -124,7 +122,7 @@ type Config struct { GroupCacheConfig *shared.GroupCacheConfig } -func NewEthBackend(db *sqlx.DB, pgxdb *pgxpool.Pool, c *Config) (*Backend, error) { +func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) { gcc := c.GroupCacheConfig groupName := gcc.StateDB.Name @@ -140,7 +138,7 @@ func NewEthBackend(db *sqlx.DB, pgxdb *pgxpool.Pool, c *Config) (*Backend, error }) logStateDBStatsOnTimer(ethDB.(*ipfsethdb.Database), gcc) - ipldStateDB, err := ipld_eth_statedb.NewStateDatabaseWithPool(pgxdb) + ipldStateDB, err := ipld_eth_statedb.NewStateDatabaseWithSqlxPool(db) if err != nil { return nil, err } diff --git a/pkg/serve/service.go b/pkg/serve/service.go index 5d92a9f8..d36d96e8 100644 --- a/pkg/serve/service.go +++ b/pkg/serve/service.go @@ -58,7 +58,7 @@ type Service struct { sync.Mutex // Used to signal shutdown of the service QuitChan chan bool - // Underlying db + // Underlying db connection pool db *sqlx.DB // rpc client for forwarding cache misses client *rpc.Client From 20b787e5dcac27e8a56147678f7e814674077d70 Mon Sep 17 00:00:00 2001 From: i-norden Date: Tue, 14 Mar 2023 10:04:41 -0500 Subject: [PATCH 11/14] missed moving some sql into consolidated file --- pkg/eth/backend.go | 34 ---------------------------------- pkg/eth/sql.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index fd25a5c7..11c79342 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -60,40 +60,6 @@ var ( errTxHashInMultipleBlocks = errors.New("transaction for hash found in more than one canonical block") ) -const ( - RetrieveCanonicalBlockHashByNumber = `SELECT block_hash - FROM canonical_header_hash($1) AS block_hash - WHERE block_hash IS NOT NULL` - RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids - INNER JOIN ipld.blocks ON ( - header_cids.cid = blocks.key - AND header_cids.block_number = blocks.block_number - ) - WHERE block_hash = (SELECT canonical_header_hash($1))` - RetrieveTD = `SELECT CAST(td as TEXT) FROM eth.header_cids - WHERE header_cids.block_hash = $1` - RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index - FROM ipld.blocks, eth.transaction_cids - WHERE blocks.key = transaction_cids.cid - AND blocks.block_number = transaction_cids.block_number - AND transaction_cids.tx_hash = $1 - AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids - WHERE state_accounts.header_id = state_cids.header_id - AND state_accounts.state_path = state_cids.state_path - AND state_accounts.block_number = state_cids.block_number - AND state_cids.header_id = header_cids.block_hash - AND state_cids.block_number = header_cids.block_number - AND state_leaf_key = $1 - AND header_cids.block_number <= (SELECT block_number - FROM eth.header_cids - WHERE block_hash = $2) - AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) - ORDER BY header_cids.block_number DESC - LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM ipld.blocks WHERE key = $1` -) - const ( StateDBGroupCacheName = "statedb" ) diff --git a/pkg/eth/sql.go b/pkg/eth/sql.go index 8da8cd8a..05aae02e 100644 --- a/pkg/eth/sql.go +++ b/pkg/eth/sql.go @@ -134,6 +134,37 @@ const ( AND transaction_cids.header_id = header_cids.block_hash AND transaction_cids.block_number = header_cids.block_number` RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT cid, val, block_number, removed, state_leaf_removed FROM get_storage_at_by_hash($1, $2, $3)` + RetrieveCanonicalBlockHashByNumber = `SELECT block_hash + FROM canonical_header_hash($1) AS block_hash + WHERE block_hash IS NOT NULL` + RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids + INNER JOIN ipld.blocks ON ( + header_cids.cid = blocks.key + AND header_cids.block_number = blocks.block_number + ) + WHERE block_hash = (SELECT canonical_header_hash($1))` + RetrieveTD = `SELECT CAST(td as TEXT) FROM eth.header_cids + WHERE header_cids.block_hash = $1` + RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index + FROM ipld.blocks, eth.transaction_cids + WHERE blocks.key = transaction_cids.cid + AND blocks.block_number = transaction_cids.block_number + AND transaction_cids.tx_hash = $1 + AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` + RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids + WHERE state_accounts.header_id = state_cids.header_id + AND state_accounts.state_path = state_cids.state_path + AND state_accounts.block_number = state_cids.block_number + AND state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + AND state_leaf_key = $1 + AND header_cids.block_number <= (SELECT block_number + FROM eth.header_cids + WHERE block_hash = $2) + AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) + ORDER BY header_cids.block_number DESC + LIMIT 1` + RetrieveCodeByMhKey = `SELECT data FROM ipld.blocks WHERE key = $1` ) type ipldResult struct { From 2bd6ed9b2486ec3305100436dd2bd69ffeb270bb Mon Sep 17 00:00:00 2001 From: i-norden Date: Thu, 16 Mar 2023 08:24:48 -0500 Subject: [PATCH 12/14] use new release of ipld-eth-statedb --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 00bc9781..649fb16a 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha github.com/cerc-io/go-eth-state-node-iterator v1.1.9 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha - github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha + github.com/cerc-io/ipld-eth-statedb v0.0.3-alpha github.com/ethereum/go-ethereum v1.10.26 github.com/google/uuid v1.3.0 github.com/graph-gophers/graphql-go v1.3.0 @@ -295,5 +295,3 @@ require ( ) replace github.com/ethereum/go-ethereum v1.10.26 => github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha - -replace github.com/cerc-io/ipld-eth-statedb v0.0.2-alpha => /Users/iannorden/go/src/github.com/cerc-io/ipld-eth-statedb diff --git a/go.sum b/go.sum index 1733dac5..fde36a00 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha h1:gesMZEbNU+fcAMc github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha h1:5iqNXeitkj3g7FxyKK/Pz+1HN7Ac9JZzCRj3Lv+uHiw= github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha/go.mod h1:dPscFRMvTWPKnoZ4U0D9v4bsrw6XdH7sOp8hUrVzOWA= +github.com/cerc-io/ipld-eth-statedb v0.0.3-alpha h1:7NCJpF2XmOzD0gmQUkvCf00mylkJ/gBfULhsELV2Awk= +github.com/cerc-io/ipld-eth-statedb v0.0.3-alpha/go.mod h1:Vx8auhxWh7V3qMgx8qo3bdp3ftSiytvXqffEmXvgpIA= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= From ea19cc8d8d1b66687bc2068a21a59caf671dd7f5 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 9 Mar 2023 20:03:41 -0600 Subject: [PATCH 13/14] Use blocks table, not header_cids, for getBlockNumber() (#232) --- pkg/eth/retriever.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/eth/retriever.go b/pkg/eth/retriever.go index ee68185e..09fe947e 100644 --- a/pkg/eth/retriever.go +++ b/pkg/eth/retriever.go @@ -112,14 +112,14 @@ func NewRetriever(db *sqlx.DB) *Retriever { // RetrieveFirstBlockNumber is used to retrieve the first block number in the db func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) { var blockNumber int64 - err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1") + err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number ASC LIMIT 1") return blockNumber, err } // RetrieveLastBlockNumber is used to retrieve the latest block number in the db func (r *Retriever) RetrieveLastBlockNumber() (int64, error) { var blockNumber int64 - err := r.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1") + err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number DESC LIMIT 1") return blockNumber, err } From bf1151d905c8f4dfe5c8a589c7f2af7134797cdd Mon Sep 17 00:00:00 2001 From: i-norden Date: Fri, 17 Mar 2023 15:19:36 -0500 Subject: [PATCH 14/14] updated ipld-eth-db image ver --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2051ec71..768f4e57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: restart: on-failure depends_on: - ipld-eth-db - image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v4.2.3-alpha + image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.0-alpha environment: DATABASE_USER: "vdbm" DATABASE_NAME: "vulcanize_testing"