From 867323a1ab37f24bc130c04122cdc2008710bc06 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Mon, 25 Apr 2022 17:07:11 +0530 Subject: [PATCH 1/7] Update queries to get data from IPLD blocks table --- pkg/eth/ipld_fetcher.go | 38 +++++++-- pkg/eth/ipld_retriever.go | 167 ++++++++++++++++++++++++++++++-------- pkg/shared/functions.go | 7 ++ 3 files changed, 173 insertions(+), 39 deletions(-) diff --git a/pkg/eth/ipld_fetcher.go b/pkg/eth/ipld_fetcher.go index 8396d368..47bd34ac 100644 --- a/pkg/eth/ipld_fetcher.go +++ b/pkg/eth/ipld_fetcher.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "strconv" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff/indexer/models" @@ -102,7 +103,12 @@ func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) { // FetchHeaders fetches headers func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (models.IPLDModel, error) { log.Debug("fetching header ipld") - headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) + blockNumber, err := strconv.ParseUint(c.BlockNumber, 10, 64) + if err != nil { + return models.IPLDModel{}, err + } + + headerBytes, err := shared.FetchIPLDByMhKeyAndBlockNumber(tx, c.MhKey, blockNumber) if err != nil { return models.IPLDModel{}, err } @@ -117,7 +123,11 @@ func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]mode log.Debug("fetching uncle iplds") uncleIPLDs := make([]models.IPLDModel, len(cids)) for i, c := range cids { - uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) + blockNumber, err := strconv.ParseUint(c.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + uncleBytes, err := shared.FetchIPLDByMhKeyAndBlockNumber(tx, c.MhKey, blockNumber) if err != nil { return nil, err } @@ -134,7 +144,11 @@ func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]models.IP log.Debug("fetching transaction iplds") trxIPLDs := make([]models.IPLDModel, len(cids)) for i, c := range cids { - txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey) + blockNumber, err := strconv.ParseUint(c.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + txBytes, err := shared.FetchIPLDByMhKeyAndBlockNumber(tx, c.MhKey, blockNumber) if err != nil { return nil, err } @@ -151,7 +165,11 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]mode log.Debug("fetching receipt iplds") rctIPLDs := make([]models.IPLDModel, len(cids)) for i, c := range cids { - rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey) + blockNumber, err := strconv.ParseUint(c.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + rctBytes, err := shared.FetchIPLDByMhKeyAndBlockNumber(tx, c.LeafMhKey, blockNumber) if err != nil { return nil, err } @@ -172,7 +190,11 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []models.StateNodeModel) ([]S if stateNode.CID == "" { continue } - stateBytes, err := shared.FetchIPLDByMhKey(tx, stateNode.MhKey) + blockNumber, err := strconv.ParseUint(stateNode.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + stateBytes, err := shared.FetchIPLDByMhKeyAndBlockNumber(tx, stateNode.MhKey, blockNumber) if err != nil { return nil, err } @@ -197,7 +219,11 @@ func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []models.StorageNodeWithSta if storageNode.CID == "" || storageNode.StateKey == "" { continue } - storageBytes, err := shared.FetchIPLDByMhKey(tx, storageNode.MhKey) + blockNumber, err := strconv.ParseUint(storageNode.BlockNumber, 10, 64) + if err != nil { + return nil, err + } + storageBytes, err := shared.FetchIPLDByMhKeyAndBlockNumber(tx, storageNode.MhKey, blockNumber) if err != nil { return nil, err } diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 3b6d4073..9caa763d 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -36,82 +36,163 @@ const ( RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + header_cids.mh_key = blocks.key + AND header_cids.block_number = blocks.block_number + ) WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + header_cids.mh_key = blocks.key + AND header_cids.block_number = blocks.block_number + ) WHERE block_number = $1` RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + header_cids.mh_key = blocks.key + AND header_cids.block_number = blocks.block_number + ) WHERE block_hash = $1` RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + uncle_cids.mh_key = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) WHERE block_hash = ANY($1::VARCHAR(66)[])` RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids - INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN eth.header_cids ON ( + uncle_cids.header_id = header_cids.block_hash + AND uncle_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + uncle_cids.mh_key = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) WHERE block_hash = $1` RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data FROM eth.uncle_cids - INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN eth.header_cids ON ( + uncle_cids.header_id = header_cids.block_hash + AND uncle_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + uncle_cids.mh_key = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) WHERE block_number = $1` RetrieveUncleByHashPgStr = `SELECT cid, data FROM eth.uncle_cids - INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + uncle_cids.mh_key = blocks.key + AND uncle_cids.block_number = blocks.block_number + ) WHERE block_hash = $1` RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + transaction_cids.mh_key = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + transaction_cids.mh_key = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data FROM eth.transaction_cids - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + transaction_cids.mh_key = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) WHERE block_number = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids - INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + transaction_cids.mh_key = blocks.key + AND transaction_cids.block_number = blocks.block_number + ) WHERE tx_hash = $1` RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) - INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN public.blocks ON ( + receipt_cids.leaf_mh_key = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) WHERE tx_hash = ANY($1::VARCHAR(66)[])` RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + receipt_cids.leaf_mh_key = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) WHERE block_hash = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) - INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + receipt_cids.leaf_mh_key = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) WHERE block_number = $1 ORDER BY eth.transaction_cids.index ASC` RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data FROM eth.receipt_cids - INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash) - INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key) + INNER JOIN eth.transaction_cids ON ( + receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.block_number = transaction_cids.block_number + ) + INNER JOIN public.blocks ON ( + receipt_cids.leaf_mh_key = blocks.key + AND receipt_cids.block_number = blocks.block_number + ) WHERE tx_hash = $1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) + INNER JOIN eth.header_cids ON ( + state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + state_cids.mh_key = blocks.key + AND state_cids.block_number = blocks.block_number + ) WHERE state_leaf_key = $1 AND block_number <= (SELECT block_number FROM eth.header_cids @@ -121,8 +202,14 @@ const ( LIMIT 1` RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key) + INNER JOIN eth.header_cids ON ( + state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + state_cids.mh_key = blocks.key + AND state_cids.block_number = blocks.block_number + ) WHERE state_leaf_key = $1 AND block_number <= $2 ORDER BY block_number DESC @@ -132,9 +219,16 @@ const ( INNER JOIN eth.state_cids ON ( storage_cids.header_id = state_cids.header_id AND storage_cids.state_path = state_cids.state_path + AND storage_cids.block_number = state_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + storage_cids.mh_key = blocks.key + AND storage_cids.block_number = blocks.block_number ) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) WHERE state_leaf_key = $1 AND storage_leaf_key = $2 AND block_number <= $3 @@ -145,9 +239,16 @@ const ( INNER JOIN eth.state_cids ON ( storage_cids.header_id = state_cids.header_id AND storage_cids.state_path = state_cids.state_path + AND storage_cids.block_number = state_cids.block_number + ) + INNER JOIN eth.header_cids ON ( + state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + ) + INNER JOIN public.blocks ON ( + storage_cids.mh_key = blocks.key + AND storage_cids.block_number = blocks.block_number ) - INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) - INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key) WHERE state_leaf_key = $1 AND storage_leaf_key = $2 AND block_number <= (SELECT block_number diff --git a/pkg/shared/functions.go b/pkg/shared/functions.go index b1ac4550..8989cfb2 100644 --- a/pkg/shared/functions.go +++ b/pkg/shared/functions.go @@ -77,6 +77,13 @@ func FetchIPLDByMhKey(tx *sqlx.Tx, mhKey string) ([]byte, error) { return block, tx.Get(&block, pgStr, mhKey) } +// FetchIPLDByMhKeyAndBlockNumber is used to retrieve an ipld from Postgres blockstore with the provided tx, mhkey string and blockNumber +func FetchIPLDByMhKeyAndBlockNumber(tx *sqlx.Tx, mhKey string, blockNumber uint64) ([]byte, error) { + pgStr := `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` + var block []byte + return block, tx.Get(&block, pgStr, mhKey, blockNumber) +} + // MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string func MultihashKeyFromCID(c cid.Cid) string { dbKey := dshelp.MultihashToDsKey(c.Hash()) From 24a45a3f003794acb0ae03fceb1ae8c89f84e5b1 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 26 Apr 2022 19:40:06 +0530 Subject: [PATCH 2/7] Update unit tests and queries to fetch data from cids tables --- pkg/eth/backend.go | 26 +++++--- pkg/eth/cid_retriever.go | 99 +++++++++++++++++++++---------- pkg/eth/cid_retriever_test.go | 14 +++-- pkg/eth/ipld_retriever.go | 12 ++-- pkg/eth/test_helpers/test_data.go | 99 +++++++++++++++++-------------- 5 files changed, 155 insertions(+), 95 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 8b12ca44..39f499a2 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -64,26 +64,38 @@ var ( const ( RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + header_cids.mh_key = blocks.key + AND header_cids.block_number = blocks.block_number + ) WHERE block_hash = (SELECT canonical_header_hash($1))` RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids - INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key) + INNER JOIN public.blocks ON ( + header_cids.mh_key = blocks.key + AND header_cids.block_number = blocks.block_number + ) WHERE block_hash = (SELECT canonical_header_hash($1))` RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids WHERE header_cids.block_hash = $1` - RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids + RetrieveRPCTransaction = `SELECT blocks.data, block_hash, transaction_cids.block_number, index + FROM public.blocks, eth.transaction_cids, eth.header_cids WHERE blocks.key = transaction_cids.mh_key + AND blocks.block_number = transaction_cids.block_number AND transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number AND transaction_cids.tx_hash = $1` RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids - WHERE state_accounts.header_id = state_cids.header_id AND state_accounts.state_path = state_cids.state_path + WHERE state_accounts.header_id = state_cids.header_id + AND state_accounts.state_path = state_cids.state_path + AND state_accounts.block_number = state_cids.block_number AND state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number AND state_leaf_key = $1 - AND block_number <= (SELECT block_number + AND header_cids.block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $2) - AND header_cids.block_hash = (SELECT canonical_header_hash(block_number)) - ORDER BY block_number DESC + AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) + ORDER BY header_cids.block_number DESC LIMIT 1` RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) diff --git a/pkg/eth/cid_retriever.go b/pkg/eth/cid_retriever.go index bcebcde7..f2414acb 100644 --- a/pkg/eth/cid_retriever.go +++ b/pkg/eth/cid_retriever.go @@ -19,6 +19,7 @@ package eth import ( "fmt" "math/big" + "strconv" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -59,7 +60,7 @@ func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) { // RetrieveLastBlockNumber is used to retrieve the latest block number in the db func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) { var blockNumber int64 - err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1 ") + err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1") return blockNumber, err } @@ -167,9 +168,9 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64 func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]models.HeaderModel, error) { log.Debug("retrieving header cids for block ", blockNumber) headers := make([]models.HeaderModel, 0) - pgStr := `SELECT CAST(block_number as Text), block_hash,parent_hash,cid,mh_key,CAST(td as Text),node_id, - CAST(reward as Text), state_root,uncle_root,tx_root,receipt_root,bloom,timestamp,times_validated, - coinbase FROM eth.header_cids + pgStr := `SELECT CAST(block_number as Text), block_hash, parent_hash, cid, mh_key, CAST(td as Text), node_id, + CAST(reward as Text), state_root, uncle_root,tx_root, receipt_root,bloom, timestamp, times_validated, coinbase + FROM eth.header_cids WHERE block_number = $1` return headers, tx.Select(&headers, pgStr, blockNumber) } @@ -178,7 +179,8 @@ func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]m func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID string) ([]models.UncleModel, error) { log.Debug("retrieving uncle cids for block id ", headerID) headers := make([]models.UncleModel, 0) - pgStr := `SELECT header_id,block_hash,parent_hash,cid,mh_key, CAST(reward as text) FROM eth.uncle_cids + pgStr := `SELECT CAST(block_number as Text), header_id, block_hash, parent_hash, cid, mh_key, CAST(reward as text) + FROM eth.uncle_cids WHERE header_id = $1` return headers, tx.Select(&headers, pgStr, headerID) } @@ -190,10 +192,15 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID args := make([]interface{}, 0, 3) results := make([]models.TxModel, 0) id := 1 - pgStr := fmt.Sprintf(`SELECT transaction_cids.tx_hash, transaction_cids.header_id,transaction_cids.cid, transaction_cids.mh_key, - transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data - FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash) - WHERE header_cids.block_hash = $%d`, id) + pgStr := fmt.Sprintf(`SELECT CAST(transaction_cids.block_number as Text), transaction_cids.tx_hash, + transaction_cids.header_id,transaction_cids.cid, transaction_cids.mh_key, transaction_cids.dst, + transaction_cids.src, transaction_cids.index, transaction_cids.tx_data + FROM eth.transaction_cids + INNER JOIN eth.header_cids ON ( + transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + ) + WHERE header_cids.block_hash = $%d`, id) args = append(args, headerID) id++ if len(txFilter.Dst) > 0 { @@ -292,11 +299,13 @@ func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilte func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID string, trxHashes []string) ([]models.ReceiptModel, error) { log.Debug("retrieving receipt cids for header id ", headerID) args := make([]interface{}, 0, 4) - pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, - receipt_cids.contract, receipt_cids.contract_hash + pgStr := `SELECT CAST(receipt_cids.block_number as Text), receipt_cids.tx_id, receipt_cids.leaf_cid, + receipt_cids.leaf_mh_key, receipt_cids.contract, receipt_cids.contract_hash FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.block_number = transaction_cids.block_number AND transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number AND header_cids.block_hash = $1` id := 2 args = append(args, headerID) @@ -308,20 +317,25 @@ func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter Receip return receiptCids, tx.Select(&receiptCids, pgStr, args...) } -// RetrieveFilteredGQLLogs retrieves and returns all the log cIDs provided blockHash that conform to the provided +// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided // filter parameters. func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) { log.Debug("retrieving log cids for receipt ids") args := make([]interface{}, 0, 4) id := 1 - pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, + pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks WHERE eth.log_cids.rct_id = receipt_cids.tx_id + AND eth.log_cids.block_number = eth.receipt_cids.block_number AND receipt_cids.tx_id = transaction_cids.tx_hash - AND transaction_cids.header_id = header_cids.block_hash - AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1` + AND receipt_cids.block_number = transaction_cids.block_number + AND transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number + AND log_cids.leaf_mh_key = blocks.key + AND log_cids.block_number = blocks.block_number + AND header_cids.block_hash = $1` args = append(args, blockHash.String()) id++ @@ -343,14 +357,17 @@ func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptF func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) { log.Debug("retrieving log cids for receipt ids") args := make([]interface{}, 0, 4) - pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, + pgStr := `SELECT CAST(eth.log_cids.block_number as Text), eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3, eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index, header_cids.block_hash, CAST(header_cids.block_number as Text) FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE eth.log_cids.rct_id = receipt_cids.tx_id + AND eth.log_cids.block_number = eth.receipt_cids.block_number AND receipt_cids.tx_id = transaction_cids.tx_hash - AND transaction_cids.header_id = header_cids.block_hash` + AND receipt_cids.block_number = transaction_cids.block_number + AND transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number` id := 1 if blockNumber > 0 { pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) @@ -380,10 +397,13 @@ func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilte func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, txHashes []string) ([]models.ReceiptModel, error) { log.Debug("retrieving receipt cids for block ", blockNumber) args := make([]interface{}, 0, 5) - pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id + pgStr := `SELECT CAST(receipt_cids.block_number as Text), receipt_cids.tx_id, receipt_cids.leaf_cid, + receipt_cids.leaf_mh_key, FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids WHERE receipt_cids.tx_id = transaction_cids.tx_hash - AND transaction_cids.header_id = header_cids.block_hash` + AND receipt_cids.block_number = transaction_cids.block_number + AND transaction_cids.header_id = header_cids.block_hash + AND transaction_cids.block_number = header_cids.block_number` id := 1 if blockNumber > 0 { pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id) @@ -416,9 +436,13 @@ func hasTopics(topics [][]string) bool { func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID string) ([]models.StateNodeModel, error) { log.Debug("retrieving state cids for header id ", headerID) args := make([]interface{}, 0, 2) - pgStr := `SELECT state_cids.header_id, + pgStr := `SELECT CAST(state_cids.block_number as Text), state_cids.header_id, state_cids.state_leaf_key, state_cids.node_type, state_cids.cid, state_cids.mh_key, state_cids.state_path - FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash) + FROM eth.state_cids + INNER JOIN eth.header_cids ON ( + state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number + ) WHERE header_cids.block_hash = $1` args = append(args, headerID) addrLen := len(stateFilter.Addresses) @@ -441,11 +465,15 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID string) ([]models.StorageNodeWithStateKeyModel, error) { log.Debug("retrieving storage cids for header id ", headerID) args := make([]interface{}, 0, 3) - pgStr := `SELECT storage_cids.header_id, storage_cids.storage_leaf_key, storage_cids.node_type, - storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, storage_cids.state_path, state_cids.state_leaf_key + pgStr := `SELECT CAST(storage_cids.block_number as Text), storage_cids.header_id, storage_cids.storage_leaf_key, + storage_cids.node_type, storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, storage_cids.state_path, + state_cids.state_leaf_key FROM eth.storage_cids, eth.state_cids, eth.header_cids - WHERE storage_cids.header_id = state_cids.header_id AND storage_cids.state_path = state_cids.state_path + WHERE storage_cids.header_id = state_cids.header_id + AND storage_cids.state_path = state_cids.state_path + AND storage_cids.block_number = state_cids.block_number AND state_cids.header_id = header_cids.block_hash + AND state_cids.block_number = header_cids.block_number AND header_cids.block_hash = $1` args = append(args, headerID) id := 2 @@ -496,6 +524,10 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (models.Head log.Error("header cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err } + blockNumber, err := strconv.ParseInt(headerCID.BlockNumber, 10, 64) + if err != nil { + return models.HeaderModel{}, nil, nil, nil, err + } var uncleCIDs []models.UncleModel uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.BlockHash) if err != nil { @@ -503,7 +535,7 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (models.Head return models.HeaderModel{}, nil, nil, nil, err } var txCIDs []models.TxModel - txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.BlockHash) + txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.BlockHash, blockNumber) if err != nil { log.Error("tx cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err @@ -556,7 +588,7 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.Header return models.HeaderModel{}, nil, nil, nil, err } var txCIDs []models.TxModel - txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].BlockHash) + txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].BlockHash, blockNumber) if err != nil { log.Error("tx cid retrieval error") return models.HeaderModel{}, nil, nil, nil, err @@ -576,30 +608,33 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.Header // RetrieveHeaderCIDByHash returns the header for the given block hash func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) { log.Debug("retrieving header cids for block hash ", blockHash.String()) - pgStr := `SELECT block_hash,cid,mh_key FROM eth.header_cids + pgStr := `SELECT block_hash, CAST(block_number as Text), cid, mh_key FROM eth.header_cids WHERE block_hash = $1` var headerCID models.HeaderModel return headerCID, tx.Get(&headerCID, pgStr, blockHash.String()) } // RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id -func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID string) ([]models.TxModel, error) { +func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID string, blockNumber int64) ([]models.TxModel, error) { log.Debug("retrieving tx cids for block id ", headerID) - pgStr := `SELECT * FROM eth.transaction_cids - WHERE header_id = $1 + pgStr := `SELECT CAST(block_number as Text), header_id, index, tx_hash, cid, mh_key, + dst, src, tx_data, tx_type, value + FROM eth.transaction_cids + WHERE header_id = $1 AND block_number = $2 ORDER BY index` var txCIDs []models.TxModel - return txCIDs, tx.Select(&txCIDs, pgStr, headerID) + return txCIDs, tx.Select(&txCIDs, pgStr, headerID, blockNumber) } // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txHashes []string) ([]models.ReceiptModel, error) { log.Debugf("retrieving receipt cids for tx hashes %v", txHashes) - pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, + pgStr := `SELECT CAST(receipt_cids.block_number as Text), receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.contract, receipt_cids.contract_hash FROM eth.receipt_cids, eth.transaction_cids WHERE tx_id = ANY($1) AND receipt_cids.tx_id = transaction_cids.tx_hash + AND receipt_cids.block_number = transaction_cids.block_number ORDER BY transaction_cids.index` var rctCIDs []models.ReceiptModel return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txHashes)) diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/cid_retriever_test.go index abe56659..c4c13ce6 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/cid_retriever_test.go @@ -249,6 +249,7 @@ var _ = Describe("Retriever", func() { AND header_cids.block_number = $1 ORDER BY transaction_cids.index` err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) + Expect(err).ToNot(HaveOccurred()) cids, empty, err := retriever.Retrieve(openFilter, 1) Expect(err).ToNot(HaveOccurred()) Expect(empty).ToNot(BeTrue()) @@ -412,12 +413,13 @@ var _ = Describe("Retriever", func() { Expect(len(cids7[0].StorageNodes)).To(Equal(0)) Expect(len(cids7[0].StateNodes)).To(Equal(1)) Expect(cids7[0].StateNodes[0]).To(Equal(models.StateNodeModel{ - HeaderID: cids7[0].StateNodes[0].HeaderID, - NodeType: 2, - StateKey: common.BytesToHash(test_helpers.AccountLeafKey).Hex(), - CID: test_helpers.State2CID.String(), - MhKey: test_helpers.State2MhKey, - Path: []byte{'\x0c'}, + BlockNumber: "1", + HeaderID: cids7[0].StateNodes[0].HeaderID, + NodeType: 2, + StateKey: common.BytesToHash(test_helpers.AccountLeafKey).Hex(), + CID: test_helpers.State2CID.String(), + MhKey: test_helpers.State2MhKey, + Path: []byte{'\x0c'}, })) _, empty, err = retriever.Retrieve(rctTopicsAndAddressFilterFail, 1) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 9caa763d..654b2db8 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -194,11 +194,11 @@ const ( AND state_cids.block_number = blocks.block_number ) WHERE state_leaf_key = $1 - AND block_number <= (SELECT block_number + AND header_cids.block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $2) - AND header_cids.block_hash = (SELECT canonical_header_hash(block_number)) - ORDER BY block_number DESC + AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) + ORDER BY header_cids.block_number DESC LIMIT 1` RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.node_type FROM eth.state_cids @@ -251,11 +251,11 @@ const ( ) WHERE state_leaf_key = $1 AND storage_leaf_key = $2 - AND block_number <= (SELECT block_number + AND header_cids.block_number <= (SELECT block_number FROM eth.header_cids WHERE block_hash = $3) - AND header_cids.block_hash = (SELECT canonical_header_hash(block_number)) - ORDER BY block_number DESC + AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) + ORDER BY header_cids.block_number DESC LIMIT 1` ) diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index 650b7cbc..b55fe202 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -231,40 +231,44 @@ var ( } MockTrxMetaPostPublsh = []models.TxModel{ { - CID: Trx1CID.String(), // This is empty until we go to publish to ipfs - MhKey: Trx1MhKey, - Src: SenderAddr.Hex(), - Dst: Address.String(), - Index: 0, - TxHash: MockTransactions[0].Hash().String(), - Data: []byte{}, + BlockNumber: "1", + CID: Trx1CID.String(), // This is empty until we go to publish to ipfs + MhKey: Trx1MhKey, + Src: SenderAddr.Hex(), + Dst: Address.String(), + Index: 0, + TxHash: MockTransactions[0].Hash().String(), + Data: []byte{}, }, { - CID: Trx2CID.String(), - MhKey: Trx2MhKey, - Src: SenderAddr.Hex(), - Dst: AnotherAddress.String(), - Index: 1, - TxHash: MockTransactions[1].Hash().String(), - Data: []byte{}, + BlockNumber: "1", + CID: Trx2CID.String(), + MhKey: Trx2MhKey, + Src: SenderAddr.Hex(), + Dst: AnotherAddress.String(), + Index: 1, + TxHash: MockTransactions[1].Hash().String(), + Data: []byte{}, }, { - CID: Trx3CID.String(), - MhKey: Trx3MhKey, - Src: SenderAddr.Hex(), - Dst: "", - Index: 2, - TxHash: MockTransactions[2].Hash().String(), - Data: MockContractByteCode, + BlockNumber: "1", + CID: Trx3CID.String(), + MhKey: Trx3MhKey, + Src: SenderAddr.Hex(), + Dst: "", + Index: 2, + TxHash: MockTransactions[2].Hash().String(), + Data: MockContractByteCode, }, { - CID: Trx4CID.String(), - MhKey: Trx4MhKey, - Src: SenderAddr.Hex(), - Dst: AnotherAddress1.String(), - Index: 3, - TxHash: MockTransactions[3].Hash().String(), - Data: []byte{}, + BlockNumber: "1", + CID: Trx4CID.String(), + MhKey: Trx4MhKey, + Src: SenderAddr.Hex(), + Dst: AnotherAddress1.String(), + Index: 3, + TxHash: MockTransactions[3].Hash().String(), + Data: []byte{}, }, } MockRctMeta = []models.ReceiptModel{ @@ -296,24 +300,28 @@ var ( MockRctMetaPostPublish = []models.ReceiptModel{ { + BlockNumber: "1", LeafCID: Rct1CID.String(), LeafMhKey: Rct1MhKey, Contract: "", ContractHash: "", }, { + BlockNumber: "1", LeafCID: Rct2CID.String(), LeafMhKey: Rct2MhKey, Contract: "", ContractHash: "", }, { + BlockNumber: "1", LeafCID: Rct3CID.String(), LeafMhKey: Rct3MhKey, Contract: ContractAddress.String(), ContractHash: ContractHash, }, { + BlockNumber: "1", LeafCID: Rct4CID.String(), LeafMhKey: Rct4MhKey, Contract: "", @@ -391,18 +399,20 @@ var ( } MockStateMetaPostPublish = []models.StateNodeModel{ { - CID: State1CID.String(), - MhKey: State1MhKey, - Path: []byte{'\x06'}, - NodeType: 2, - StateKey: common.BytesToHash(ContractLeafKey).Hex(), + BlockNumber: "1", + CID: State1CID.String(), + MhKey: State1MhKey, + Path: []byte{'\x06'}, + NodeType: 2, + StateKey: common.BytesToHash(ContractLeafKey).Hex(), }, { - CID: State2CID.String(), - MhKey: State2MhKey, - Path: []byte{'\x0c'}, - NodeType: 2, - StateKey: common.BytesToHash(AccountLeafKey).Hex(), + BlockNumber: "1", + CID: State2CID.String(), + MhKey: State2MhKey, + Path: []byte{'\x0c'}, + NodeType: 2, + StateKey: common.BytesToHash(AccountLeafKey).Hex(), }, } MockStorageNodes = map[string][]sdtypes.StorageNode{ @@ -461,12 +471,13 @@ var ( StateNodes: MockStateMetaPostPublish, StorageNodes: []models.StorageNodeWithStateKeyModel{ { - Path: []byte{}, - CID: StorageCID.String(), - MhKey: StorageMhKey, - NodeType: 2, - StateKey: common.BytesToHash(ContractLeafKey).Hex(), - StorageKey: common.BytesToHash(StorageLeafKey).Hex(), + BlockNumber: "1", + Path: []byte{}, + CID: StorageCID.String(), + MhKey: StorageMhKey, + NodeType: 2, + StateKey: common.BytesToHash(ContractLeafKey).Hex(), + StorageKey: common.BytesToHash(StorageLeafKey).Hex(), }, }, } From 384d939dc269d19c4ca7de1d2df31c48498ea5d4 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Fri, 6 May 2022 13:56:43 +0530 Subject: [PATCH 3/7] Upgrade dependencies --- cmd/validate.go | 4 ++-- go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/eth/backend.go | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/validate.go b/cmd/validate.go index 3c95edbf..04338a4a 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -22,8 +22,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - validator "github.com/vulcanize/eth-ipfs-state-validator/v3/pkg" - ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres" + validator "github.com/vulcanize/eth-ipfs-state-validator/v4/pkg" + ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres" s "github.com/vulcanize/ipld-eth-server/v3/pkg/serve" ) diff --git a/go.mod b/go.mod index aa292c91..f52ce67f 100644 --- a/go.mod +++ b/go.mod @@ -23,9 +23,9 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.11.0 - github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.0 + github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.0-alpha github.com/vulcanize/gap-filler v0.3.1 - github.com/vulcanize/ipfs-ethdb/v3 v3.0.1 + github.com/vulcanize/ipfs-ethdb/v4 v4.0.0-alpha ) -replace github.com/ethereum/go-ethereum v1.10.17 => github.com/vulcanize/go-ethereum v1.10.17-statediff-3.2.1 +replace github.com/ethereum/go-ethereum v1.10.17 => github.com/vulcanize/go-ethereum v1.10.17-statediff-4.0.1-alpha diff --git a/go.sum b/go.sum index 2118e8df..e802faef 100644 --- a/go.sum +++ b/go.sum @@ -1834,14 +1834,14 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.0 h1:hQcbewsJX5Xpd3xdE9n3JpGyuzUqMhwhpQCpbN+9Uiw= -github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.0/go.mod h1:CuuygZPciwwLV5vNY1LcIi0PZxXZbOmicw1/IkrPpPI= +github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.0-alpha h1:9TtxGBNl/hVQ8wegrZx//UnAMivImbcNcYTQxHfhjlQ= +github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.0-alpha/go.mod h1:cY+XmYdOtBc84s02GdDTFoybJF+mR7WDcr3Z7l4Fb1g= github.com/vulcanize/gap-filler v0.3.1 h1:N5d+jCJo/VTWFvBSbTD7biRhK/OqDZzi1tgA85SIBKs= github.com/vulcanize/gap-filler v0.3.1/go.mod h1:qowG1cgshVpBqMokiWro/1xhh0uypw7oAu8FQ42JMy4= -github.com/vulcanize/go-ethereum v1.10.17-statediff-3.2.1 h1:BUIaiqqsWM61KfzIowqrGJ8TM2SWwmral4iJ/nhTSe0= -github.com/vulcanize/go-ethereum v1.10.17-statediff-3.2.1/go.mod h1:mDwZX+QoWdqzQo6SDG3YVqCYACutcSG6uzpziMvTu28= -github.com/vulcanize/ipfs-ethdb/v3 v3.0.1 h1:gm+6SgxcNOCz6mgwvhsxORLXsr9yIFvbJDuoAK85ONs= -github.com/vulcanize/ipfs-ethdb/v3 v3.0.1/go.mod h1:SPBTTl5CqRexYfkI66pbzE5nQziYpoQ+7hH0XveA9IU= +github.com/vulcanize/go-ethereum v1.10.17-statediff-4.0.1-alpha h1:ANyTG2JJkOaQD3IqkLLl2OjtTC+PKlrdmy6HIddpqZw= +github.com/vulcanize/go-ethereum v1.10.17-statediff-4.0.1-alpha/go.mod h1:mDwZX+QoWdqzQo6SDG3YVqCYACutcSG6uzpziMvTu28= +github.com/vulcanize/ipfs-ethdb/v4 v4.0.0-alpha h1:XaltpDMGFqcEe+tsYTmqICeFhR+jTHHzc85YTVigEdE= +github.com/vulcanize/ipfs-ethdb/v4 v4.0.0-alpha/go.mod h1:sSTqwalaV7CiXq83YkMJpZetoSxgcdzr0AG0NnuaGMM= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 39f499a2..a8b4bda8 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -43,8 +43,8 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" - validator "github.com/vulcanize/eth-ipfs-state-validator/v3/pkg" - ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres" + validator "github.com/vulcanize/eth-ipfs-state-validator/v4/pkg" + ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres" ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" From 36e590c0926a2c2ffcec759a83979ec50500e5b5 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 18 May 2022 20:23:40 +0530 Subject: [PATCH 4/7] Delete from all tables when tearing down db for tests --- pkg/eth/backend.go | 1 - pkg/shared/test_helpers.go | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index a8b4bda8..5cc1f878 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -59,7 +59,6 @@ var ( // errMissingSignature is returned if a block's extra-data section doesn't seem // to contain a 65 byte secp256k1 signature. - errMissingSignature = errors.New("extra-data 65 byte signature suffix missing") ) const ( diff --git a/pkg/shared/test_helpers.go b/pkg/shared/test_helpers.go index d9795b59..f8d32dbf 100644 --- a/pkg/shared/test_helpers.go +++ b/pkg/shared/test_helpers.go @@ -58,8 +58,12 @@ func SetupDB() *sqlx.DB { func TearDownDB(db *sqlx.DB) { tx, err := db.Beginx() Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM nodes`) + Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM eth.header_cids`) Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.uncle_cids`) + Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM eth.transaction_cids`) Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM eth.receipt_cids`) @@ -68,6 +72,10 @@ func TearDownDB(db *sqlx.DB) { Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM eth.storage_cids`) Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.state_accounts`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.access_list_elements`) + Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM blocks`) Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM eth.log_cids`) From 9676690a2dff5a3642122146709e5c882045bf34 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 18 May 2022 17:48:17 +0530 Subject: [PATCH 5/7] Remove unused functions --- pkg/eth/filterer.go | 25 +++++++------ pkg/eth/filterer_test.go | 40 ++++++++++++--------- pkg/eth/ipld_fetcher.go | 30 +++++++++------- pkg/eth/test_helpers/test_data.go | 60 ++++++++++++++++++------------- pkg/shared/functions.go | 41 --------------------- 5 files changed, 93 insertions(+), 103 deletions(-) diff --git a/pkg/eth/filterer.go b/pkg/eth/filterer.go index 3fee2209..8a17f446 100644 --- a/pkg/eth/filterer.go +++ b/pkg/eth/filterer.go @@ -82,8 +82,9 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP return err } response.Header = models.IPLDModel{ - Data: headerRLP, - Key: cid.String(), + BlockNumber: payload.Block.Number().String(), + Data: headerRLP, + Key: cid.String(), } if headerFilter.Uncles { response.Uncles = make([]models.IPLDModel, len(payload.Block.Body().Uncles)) @@ -97,8 +98,9 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP return err } response.Uncles[i] = models.IPLDModel{ - Data: uncleRlp, - Key: cid.String(), + BlockNumber: uncle.Number.String(), + Data: uncleRlp, + Key: cid.String(), } } } @@ -183,8 +185,9 @@ func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response * // TODO: Verify this filter logic. if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) { response.Receipts = append(response.Receipts, models.IPLDModel{ - Data: rctIPLDData[idx], - Key: rctLeafCID[idx].String(), + BlockNumber: payload.Block.Number().String(), + Data: rctIPLDData[idx], + Key: rctLeafCID[idx].String(), }) } } @@ -282,8 +285,9 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag StateLeafKey: common.BytesToHash(stateNode.LeafKey), Path: stateNode.Path, IPLD: models.IPLDModel{ - Data: stateNode.NodeValue, - Key: cid.String(), + BlockNumber: payload.Block.Number().String(), + Data: stateNode.NodeValue, + Key: cid.String(), }, Type: stateNode.NodeType, }) @@ -300,8 +304,9 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag StateLeafKey: common.BytesToHash(stateNode.LeafKey), StorageLeafKey: common.BytesToHash(storageNode.LeafKey), IPLD: models.IPLDModel{ - Data: storageNode.NodeValue, - Key: cid.String(), + BlockNumber: payload.Block.Number().String(), + Data: storageNode.NodeValue, + Key: cid.String(), }, Type: storageNode.NodeType, Path: storageNode.Path, diff --git a/pkg/eth/filterer_test.go b/pkg/eth/filterer_test.go index 21cabc54..73b428cc 100644 --- a/pkg/eth/filterer_test.go +++ b/pkg/eth/filterer_test.go @@ -61,14 +61,16 @@ var _ = Describe("Filterer", func() { Expect(stateNode.Type).To(Equal(sdtypes.Leaf)) if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.AccountLeafKey) { Expect(stateNode.IPLD).To(Equal(models.IPLDModel{ - Data: test_helpers.State2IPLD.RawData(), - Key: test_helpers.State2IPLD.Cid().String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.State2IPLD.RawData(), + Key: test_helpers.State2IPLD.Cid().String(), })) } if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.ContractLeafKey) { Expect(stateNode.IPLD).To(Equal(models.IPLDModel{ - Data: test_helpers.State1IPLD.RawData(), - Key: test_helpers.State1IPLD.Cid().String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.State1IPLD.RawData(), + Key: test_helpers.State1IPLD.Cid().String(), })) } } @@ -87,8 +89,9 @@ var _ = Describe("Filterer", func() { Expect(len(iplds1.StateNodes)).To(Equal(0)) Expect(len(iplds1.Receipts)).To(Equal(1)) Expect(iplds1.Receipts[0]).To(Equal(models.IPLDModel{ - Data: test_helpers.Rct1IPLD, - Key: test_helpers.Rct1CID.String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.Rct1IPLD, + Key: test_helpers.Rct1CID.String(), })) iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload) @@ -102,8 +105,9 @@ var _ = Describe("Filterer", func() { Expect(len(iplds2.StateNodes)).To(Equal(0)) Expect(len(iplds2.Receipts)).To(Equal(1)) Expect(iplds2.Receipts[0]).To(Equal(models.IPLDModel{ - Data: test_helpers.Rct1IPLD, - Key: test_helpers.Rct1CID.String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.Rct1IPLD, + Key: test_helpers.Rct1CID.String(), })) iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload) @@ -117,8 +121,9 @@ var _ = Describe("Filterer", func() { Expect(len(iplds3.StateNodes)).To(Equal(0)) Expect(len(iplds3.Receipts)).To(Equal(1)) Expect(iplds3.Receipts[0]).To(Equal(models.IPLDModel{ - Data: test_helpers.Rct1IPLD, - Key: test_helpers.Rct1CID.String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.Rct1IPLD, + Key: test_helpers.Rct1CID.String(), })) iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload) @@ -132,8 +137,9 @@ var _ = Describe("Filterer", func() { Expect(len(iplds4.StateNodes)).To(Equal(0)) Expect(len(iplds4.Receipts)).To(Equal(1)) Expect(iplds4.Receipts[0]).To(Equal(models.IPLDModel{ - Data: test_helpers.Rct2IPLD, - Key: test_helpers.Rct2CID.String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.Rct2IPLD, + Key: test_helpers.Rct2CID.String(), })) iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload) @@ -165,8 +171,9 @@ var _ = Describe("Filterer", func() { Expect(len(iplds6.StateNodes)).To(Equal(0)) Expect(len(iplds6.Receipts)).To(Equal(1)) Expect(iplds4.Receipts[0]).To(Equal(models.IPLDModel{ - Data: test_helpers.Rct2IPLD, - Key: test_helpers.Rct2CID.String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.Rct2IPLD, + Key: test_helpers.Rct2CID.String(), })) iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload) @@ -181,8 +188,9 @@ var _ = Describe("Filterer", func() { Expect(len(iplds7.StateNodes)).To(Equal(1)) Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(test_helpers.AccountLeafKey)) Expect(iplds7.StateNodes[0].IPLD).To(Equal(models.IPLDModel{ - Data: test_helpers.State2IPLD.RawData(), - Key: test_helpers.State2IPLD.Cid().String(), + BlockNumber: test_helpers.BlockNumber.String(), + Data: test_helpers.State2IPLD.RawData(), + Key: test_helpers.State2IPLD.Cid().String(), })) iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, test_helpers.MockConvertedPayload) diff --git a/pkg/eth/ipld_fetcher.go b/pkg/eth/ipld_fetcher.go index 47bd34ac..9e539b8c 100644 --- a/pkg/eth/ipld_fetcher.go +++ b/pkg/eth/ipld_fetcher.go @@ -113,8 +113,9 @@ func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (models.IPL return models.IPLDModel{}, err } return models.IPLDModel{ - Data: headerBytes, - Key: c.CID, + BlockNumber: c.BlockNumber, + Data: headerBytes, + Key: c.CID, }, nil } @@ -132,8 +133,9 @@ func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]mode return nil, err } uncleIPLDs[i] = models.IPLDModel{ - Data: uncleBytes, - Key: c.CID, + BlockNumber: c.BlockNumber, + Data: uncleBytes, + Key: c.CID, } } return uncleIPLDs, nil @@ -153,8 +155,9 @@ func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]models.IP return nil, err } trxIPLDs[i] = models.IPLDModel{ - Data: txBytes, - Key: c.CID, + BlockNumber: c.BlockNumber, + Data: txBytes, + Key: c.CID, } } return trxIPLDs, nil @@ -175,8 +178,9 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]mode } //nodeVal, err := DecodeLeafNode(rctBytes) rctIPLDs[i] = models.IPLDModel{ - Data: rctBytes, - Key: c.LeafCID, + BlockNumber: c.BlockNumber, + Data: rctBytes, + Key: c.LeafCID, } } return rctIPLDs, nil @@ -200,8 +204,9 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []models.StateNodeModel) ([]S } stateNodes = append(stateNodes, StateNode{ IPLD: models.IPLDModel{ - Data: stateBytes, - Key: stateNode.CID, + BlockNumber: stateNode.BlockNumber, + Data: stateBytes, + Key: stateNode.CID, }, StateLeafKey: common.HexToHash(stateNode.StateKey), Type: ResolveToNodeType(stateNode.NodeType), @@ -229,8 +234,9 @@ func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []models.StorageNodeWithSta } storageNodes = append(storageNodes, StorageNode{ IPLD: models.IPLDModel{ - Data: storageBytes, - Key: storageNode.CID, + BlockNumber: storageNode.BlockNumber, + Data: storageBytes, + Key: storageNode.CID, }, StateLeafKey: common.HexToHash(storageNode.StateKey), StorageLeafKey: common.HexToHash(storageNode.StorageKey), diff --git a/pkg/eth/test_helpers/test_data.go b/pkg/eth/test_helpers/test_data.go index b55fe202..f1154470 100644 --- a/pkg/eth/test_helpers/test_data.go +++ b/pkg/eth/test_helpers/test_data.go @@ -494,43 +494,52 @@ var ( MockIPLDs = eth.IPLDs{ BlockNumber: new(big.Int).Set(BlockNumber), Header: models.IPLDModel{ - Data: HeaderIPLD.RawData(), - Key: HeaderIPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: HeaderIPLD.RawData(), + Key: HeaderIPLD.Cid().String(), }, Transactions: []models.IPLDModel{ { - Data: Trx1IPLD.RawData(), - Key: Trx1IPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: Trx1IPLD.RawData(), + Key: Trx1IPLD.Cid().String(), }, { - Data: Trx2IPLD.RawData(), - Key: Trx2IPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: Trx2IPLD.RawData(), + Key: Trx2IPLD.Cid().String(), }, { - Data: Trx3IPLD.RawData(), - Key: Trx3IPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: Trx3IPLD.RawData(), + Key: Trx3IPLD.Cid().String(), }, { - Data: Trx4IPLD.RawData(), - Key: Trx4IPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: Trx4IPLD.RawData(), + Key: Trx4IPLD.Cid().String(), }, }, Receipts: []models.IPLDModel{ { - Data: Rct1IPLD, - Key: Rct1CID.String(), + BlockNumber: BlockNumber.String(), + Data: Rct1IPLD, + Key: Rct1CID.String(), }, { - Data: Rct2IPLD, - Key: Rct2CID.String(), + BlockNumber: BlockNumber.String(), + Data: Rct2IPLD, + Key: Rct2CID.String(), }, { - Data: Rct3IPLD, - Key: Rct3CID.String(), + BlockNumber: BlockNumber.String(), + Data: Rct3IPLD, + Key: Rct3CID.String(), }, { - Data: Rct4IPLD, - Key: Rct4CID.String(), + BlockNumber: BlockNumber.String(), + Data: Rct4IPLD, + Key: Rct4CID.String(), }, }, StateNodes: []eth.StateNode{ @@ -538,8 +547,9 @@ var ( StateLeafKey: common.BytesToHash(ContractLeafKey), Type: sdtypes.Leaf, IPLD: models.IPLDModel{ - Data: State1IPLD.RawData(), - Key: State1IPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: State1IPLD.RawData(), + Key: State1IPLD.Cid().String(), }, Path: []byte{'\x06'}, }, @@ -547,8 +557,9 @@ var ( StateLeafKey: common.BytesToHash(AccountLeafKey), Type: sdtypes.Leaf, IPLD: models.IPLDModel{ - Data: State2IPLD.RawData(), - Key: State2IPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: State2IPLD.RawData(), + Key: State2IPLD.Cid().String(), }, Path: []byte{'\x0c'}, }, @@ -559,8 +570,9 @@ var ( StorageLeafKey: common.BytesToHash(StorageLeafKey), Type: sdtypes.Leaf, IPLD: models.IPLDModel{ - Data: StorageIPLD.RawData(), - Key: StorageIPLD.Cid().String(), + BlockNumber: BlockNumber.String(), + Data: StorageIPLD.RawData(), + Key: StorageIPLD.Cid().String(), }, Path: []byte{}, }, diff --git a/pkg/shared/functions.go b/pkg/shared/functions.go index 8989cfb2..8c78d0d4 100644 --- a/pkg/shared/functions.go +++ b/pkg/shared/functions.go @@ -18,11 +18,9 @@ package shared import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" - node "github.com/ipfs/go-ipld-format" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" ) @@ -50,33 +48,6 @@ func Rollback(tx *sqlx.Tx) { } } -// PublishIPLD is used to insert an ipld into Postgres blockstore with the provided tx -func PublishIPLD(tx *sqlx.Tx, i node.Node) error { - dbKey := dshelp.MultihashToDsKey(i.Cid().Hash()) - prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() - raw := i.RawData() - _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) - return err -} - -// FetchIPLD is used to retrieve an ipld from Postgres blockstore with the provided tx and cid string -func FetchIPLD(tx *sqlx.Tx, cid string) ([]byte, error) { - mhKey, err := MultihashKeyFromCIDString(cid) - if err != nil { - return nil, err - } - pgStr := `SELECT data FROM public.blocks WHERE key = $1` - var block []byte - return block, tx.Get(&block, pgStr, mhKey) -} - -// FetchIPLDByMhKey is used to retrieve an ipld from Postgres blockstore with the provided tx and mhkey string -func FetchIPLDByMhKey(tx *sqlx.Tx, mhKey string) ([]byte, error) { - pgStr := `SELECT data FROM public.blocks WHERE key = $1` - var block []byte - return block, tx.Get(&block, pgStr, mhKey) -} - // FetchIPLDByMhKeyAndBlockNumber is used to retrieve an ipld from Postgres blockstore with the provided tx, mhkey string and blockNumber func FetchIPLDByMhKeyAndBlockNumber(tx *sqlx.Tx, mhKey string, blockNumber uint64) ([]byte, error) { pgStr := `SELECT data FROM public.blocks WHERE key = $1 AND block_number = $2` @@ -99,15 +70,3 @@ func MultihashKeyFromCIDString(c string) (string, error) { dbKey := dshelp.MultihashToDsKey(dc.Hash()) return blockstore.BlockPrefix.String() + dbKey.String(), nil } - -// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx -func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) { - c, err := ipld.RawdataToCid(codec, raw, mh) - if err != nil { - return "", err - } - dbKey := dshelp.MultihashToDsKey(c.Hash()) - prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() - _, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) - return c.String(), err -} From dc14656bbc493953b20b92d4926eb0dbe48a5303 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 19 May 2022 10:13:33 +0530 Subject: [PATCH 6/7] Update scripts to run unit tests using TimescaleDB --- .github/workflows/run_unit_test.sh | 22 +++++++++++++++------- docker-compose.yml | 19 +++++++++---------- scripts/run_unit_test.sh | 30 ++++++++++++++++++++++++------ 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/.github/workflows/run_unit_test.sh b/.github/workflows/run_unit_test.sh index 8b80c5ab..d1854407 100755 --- a/.github/workflows/run_unit_test.sh +++ b/.github/workflows/run_unit_test.sh @@ -8,22 +8,30 @@ temp_dir=$(mktemp -d) cd $temp_dir git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git" cd ipld-eth-server +mkdir -p out ## Remove the branch and github related info. This way future runs wont be confused. rm -f /tmp/git_head_ref /tmp/git_repository -# Spin up DB -docker-compose up -d ipld-eth-db -trap "docker-compose down --remove-orphans; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR -sleep 10 +# Remove existing docker-tsdb directory +rm -rf out/docker-tsdb/ + +# Copy over files to setup TimescaleDB +ID=$(docker create vulcanize/ipld-eth-db:v4.1.1-alpha) +docker cp $ID:/app/docker-tsdb out/docker-tsdb/ +docker rm -v $ID + +# Spin up TimescaleDB +docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml up ipld-eth-db +trap "docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR +sleep 45 # Remove old logs so there's no confusion, then run test rm -f /tmp/test.log /tmp/return_test.txt -PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test > /tmp/test.log +PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test > /tmp/test.log echo $? > /tmp/return_test.txt # Clean up -docker-compose down -v --remove-orphans +docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes cd $start_dir rm -fr $temp_dir - diff --git a/docker-compose.yml b/docker-compose.yml index 132a8d2e..dd5cefa2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,17 +2,16 @@ version: '3.2' services: ipld-eth-db: - restart: always - image: vulcanize/ipld-eth-db:v3.2.0 + restart: on-failure + depends_on: + - access-node + image: vulcanize/ipld-eth-db:v4.1.1-alpha environment: - POSTGRES_USER: "vdbm" - POSTGRES_DB: "vulcanize_testing" - POSTGRES_PASSWORD: "password" - volumes: - - vdb_db_eth_server:/var/lib/postgresql/data - ports: - - "127.0.0.1:8077:5432" - command: ["postgres", "-c", "log_statement=all"] + DATABASE_USER: "vdbm" + DATABASE_NAME: "vulcanize_testing_v4" + DATABASE_PASSWORD: "password" + DATABASE_HOSTNAME: "access-node" + DATABASE_PORT: 5432 eth-server: restart: unless-stopped diff --git a/scripts/run_unit_test.sh b/scripts/run_unit_test.sh index 27f7e2ef..1b9198d6 100755 --- a/scripts/run_unit_test.sh +++ b/scripts/run_unit_test.sh @@ -1,8 +1,26 @@ -# Clear up existing docker images and volume. -docker-compose down --remove-orphans --volumes +#!/bin/bash -docker-compose -f docker-compose.yml up -d ipld-eth-db -sleep 10 -PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test +set -e -docker-compose down --remove-orphans --volumes +mkdir -p out + +# Remove existing docker-tsdb directory +rm -rf out/docker-tsdb/ + +# Copy over files to setup TimescaleDB +ID=$(docker create vulcanize/ipld-eth-db:v4.1.1-alpha) +docker cp $ID:/app/docker-tsdb out/docker-tsdb/ +docker rm -v $ID + +# Spin up TimescaleDB +docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml up ipld-eth-db +trap "docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes; rm -rf out/" SIGINT SIGTERM ERR +sleep 45 + +# Run unit tests +go clean -testcache +PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test + +# Clean up +docker-compose -f out/docker-tsdb/docker-compose.test.yml -f docker-compose.yml down --remove-orphans --volumes +rm -rf out/ From 04096fab0cd8db7c87cbe3af4880abd271611e4d Mon Sep 17 00:00:00 2001 From: nabarun Date: Thu, 19 May 2022 11:21:15 +0530 Subject: [PATCH 7/7] Update stack-orchestrator versions for CI --- .github/workflows/on-pr.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 99f446dd..8bd54f6c 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -60,8 +60,8 @@ jobs: integrationtest: name: Run integration tests env: - STACK_ORCHESTRATOR_REF: fcbc74451c5494664fe21f765e89c9c6565c07cb - GO_ETHEREUM_REF: 498101102c891c4f8c3cab5649158c642ee1fd6b + STACK_ORCHESTRATOR_REF: 38545ad36144800d16d61e53050bb8e784729f26 + GO_ETHEREUM_REF: ef8c9f2580dd577a7c5eca538fb0ed64d53dc4a4 GOPATH: /tmp/go DB_WRITE: true ETH_FORWARD_ETH_CALLS: false @@ -127,8 +127,8 @@ jobs: integrationtest_forwardethcalls: name: Run integration tests for direct proxy fall-through of eth_calls env: - STACK_ORCHESTRATOR_REF: fcbc74451c5494664fe21f765e89c9c6565c07cb - GO_ETHEREUM_REF: 498101102c891c4f8c3cab5649158c642ee1fd6b + STACK_ORCHESTRATOR_REF: 38545ad36144800d16d61e53050bb8e784729f26 + GO_ETHEREUM_REF: ef8c9f2580dd577a7c5eca538fb0ed64d53dc4a4 GOPATH: /tmp/go DB_WRITE: false ETH_FORWARD_ETH_CALLS: true @@ -194,8 +194,8 @@ jobs: integrationtest_watchedaddress_gapfillingservice: name: Run integration tests for watched addresses with gap filling service enabled env: - STACK_ORCHESTRATOR_REF: fcbc74451c5494664fe21f765e89c9c6565c07cb - GO_ETHEREUM_REF: 498101102c891c4f8c3cab5649158c642ee1fd6b + STACK_ORCHESTRATOR_REF: 38545ad36144800d16d61e53050bb8e784729f26 + GO_ETHEREUM_REF: ef8c9f2580dd577a7c5eca538fb0ed64d53dc4a4 GOPATH: /tmp/go DB_WRITE: true ETH_FORWARD_ETH_CALLS: false