From a2a391c45953a3253a0ae7f084f055297ae5405f Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 10:13:08 +0530 Subject: [PATCH 01/21] Implement getSlice API for state nodes --- pkg/eth/api.go | 8 +++ pkg/eth/api_test.go | 4 ++ pkg/eth/backend.go | 125 +++++++++++++++++++++++++++++++++++++- pkg/eth/helpers.go | 82 +++++++++++++++++++++++++ pkg/eth/ipld_retriever.go | 51 ++++++++++++++++ pkg/eth/types.go | 39 ++++++++++++ 6 files changed, 308 insertions(+), 1 deletion(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 5fc64393..b7e4da8e 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -837,6 +837,14 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre }, state.Error() } +// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth +func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) { + if storage { + return pea.B.GetStorageSlice(path, depth, root) + } + return pea.B.GetStateSlice(path, depth, root) +} + // revertError is an API error that encompassas an EVM revertal with JSON error // code and a binary data blob. type revertError struct { diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index feb500bf..91d80b1d 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -1145,4 +1145,8 @@ var _ = Describe("API", func() { Expect(code).To(BeEmpty()) }) }) + + Describe("eth_getSlice", func() { + // TODO Implement + }) }) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 24d86e54..8d1ed56c 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -43,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" @@ -95,7 +96,12 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` + RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` + RetrieveBlockNumberForStateRoot = `SELECT block_number + FROM eth.header_cids + WHERE state_root = $1 + ORDER BY block_number DESC + LIMIT 1` ) const ( @@ -888,6 +894,123 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, return storageRlp, err } +func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { + response := new(GetSliceResponse) + response.init(path, depth, root) + + // Start a timer + trieLoadingStart := makeTimestamp() + + // Get the block height for the input state root + blockHeight, err := b.getBlockHeightForStateRoot(root) + if err != nil { + return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error()) + } + + headPath, stemPaths, slicePaths, err := getPaths(path, depth) + if err != nil { + return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error()) + } + response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart)) + + // Begin tx + tx, err := b.DB.Beginx() + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + + // Fetch stem nodes + // some of the "stem" nodes can be leaf nodes (but not value nodes) + stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, stemPaths) + if err != nil { + return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error()) + } + response.TrieNodes.Stem = stemNodes + response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent + + // Fetch slice nodes + sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, slicePaths) + if err != nil { + return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error()) + } + response.TrieNodes.Slice = sliceNodes + response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent + + // Fetch head node + headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, [][]byte{headPath}) + if err != nil { + return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error()) + } + response.TrieNodes.Head = headNode + + // Fetch leaf contract data and fill in remaining metadata + leafFetchStart := makeTimestamp() + leafNodes := make([]cid.Cid, 0, len(stemLeafCIDs)+len(sliceLeafCIDs)+len(headLeafCID)) + leafNodes = append(leafNodes, stemLeafCIDs...) + leafNodes = append(leafNodes, sliceLeafCIDs...) + leafNodes = append(leafNodes, headLeafCID...) + // TODO: fill in contract data `response.Leaves` + + maxDepth := deepestPath - len(headPath) + if maxDepth < 0 { + maxDepth = 0 + } + response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) + + response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) + response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) + response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) + response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) + + return response, nil +} + +func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { + // TODO Implement + return nil, nil +} + +func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { + var blockHeight uint64 + return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String()) +} + +func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight uint64, paths [][]byte) (map[string]string, []cid.Cid, int, string, error) { + nodes := make(map[string]string) + fetchStart := makeTimestamp() + + // Get CIDs for all nodes at the provided paths + leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStatesByPathsAndBlockNumber(tx, paths, blockHeight) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for leaf nodes + err = populateNodesMap(nodes, leafCIDs, leafIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for intermediate nodes + err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index f9a995fc..6caded31 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -17,7 +17,16 @@ package eth import ( + "bytes" + "fmt" + "math" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" ) func ResolveToNodeType(nodeType int) sdtypes.NodeType { @@ -34,3 +43,76 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType { return sdtypes.Unknown } } + +var pathSteps = []byte{'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f'} + +// Return head, stem, and slice byte paths for the given head path and depth +func getPaths(path string, depth int) ([]byte, [][]byte, [][]byte, error) { + // Convert the head hex path to a decoded byte path + headPath := common.FromHex(path) + + pathLen := len(headPath) + if pathLen > 64 { // max path len is 64 + return nil, nil, nil, fmt.Errorf("path length cannot exceed 64; got %d", pathLen) + } + + maxDepth := 64 - pathLen + if depth > maxDepth { + return nil, nil, nil, fmt.Errorf("max depth for path %s is %d; got %d", path, maxDepth, depth) + } + + // Collect all of the stem paths + stemPaths := make([][]byte, 0, pathLen) + for i := 0; i < pathLen; i++ { + stemPaths = append(stemPaths, headPath[:i]) + } + + // Generate all of the slice paths + slicePaths := make([][]byte, 0, int(math.Pow(16, float64(depth)))) + makeSlicePaths(headPath, depth, &slicePaths) + + return headPath, stemPaths, slicePaths, nil +} + +// iterative function to generate the set of slice paths +func makeSlicePaths(path []byte, depth int, slicePaths *[][]byte) { + depth-- // decrement the depth + nextPaths := make([][]byte, 16) // slice to hold the next 16 paths + for i, step := range pathSteps { // iterate through steps + nextPath := append(path, step) // create next paths by adding steps to current path + nextPaths[i] = nextPath + newSlicePaths := append(*slicePaths, nextPath) // add next paths to the collection of all slice paths + slicePaths = &newSlicePaths + } + + if depth == 0 { // if depth has reach 0, return + return + } + for _, nextPath := range nextPaths { // if not, then we iterate over the next paths + makeSlicePaths(nextPath, depth, slicePaths) // and repeat the process for each one + } +} + +// use to return timestamp in milliseconds +func makeTimestamp() int64 { + return time.Now().UnixNano() / int64(time.Millisecond) +} + +func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) error { + for i, cid := range cids { + decodedMh, err := multihash.Decode(cid.Hash()) + if err != nil { + return err + } + + data := iplds[i] + hash := crypto.Keccak256Hash(data) + if !bytes.Equal(hash.Bytes(), decodedMh.Digest) { + panic("multihash digest should equal keccak of raw data") + } + + nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data) + } + + return nil +} diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 2cbdcaea..fe970a82 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -23,6 +23,7 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/statediff/trie_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" "github.com/ethereum/go-ethereum/common" @@ -238,6 +239,17 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` + RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.mh_key, state_cids.node_type + FROM eth.state_cids + INNER JOIN public.blocks ON ( + state_cids.mh_key = blocks.key + AND state_cids.block_number = blocks.block_number + ) + WHERE state_path = $1 + AND state_cids.block_number <= $2 + AND node_type != 3 + ORDER BY state_cids.block_number DESC + LIMIT 1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON ( @@ -723,3 +735,42 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad } return storageResult.CID, i[1].([]byte), nil } + +// RetrieveStatesByPathsAndBlockNumber returns the cid and rlp bytes for the state nodes corresponding to the provided state paths and block number +func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) { + deepestPath := 0 + + leafNodeCIDs := make([]cid.Cid, 0) + intermediateNodeCIDs := make([]cid.Cid, 0) + + leafNodeIPLDs := make([][]byte, 0) + intermediateNodeIPLDs := make([][]byte, 0) + + for _, path := range paths { + // Create a result object, select: cid, blockNumber and nodeType + res := new(nodeInfo) + if err := r.db.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + return nil, nil, nil, nil, 0, err + } + + pathLen := len(path) + if pathLen > deepestPath { + deepestPath = pathLen + } + + cid, err := cid.Decode(res.CID) + if err != nil { + return nil, nil, nil, nil, 0, err + } + + if res.NodeType == sdtypes.Leaf.Int() { + leafNodeCIDs = append(leafNodeCIDs, cid) + leafNodeIPLDs = append(leafNodeIPLDs, res.Data) + } else { + intermediateNodeCIDs = append(intermediateNodeCIDs, cid) + intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) + } + } + + return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil +} diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 5118eda2..ada668b5 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -18,6 +18,7 @@ package eth import ( "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -264,3 +265,41 @@ type LogResult struct { TxnIndex int64 `db:"txn_index"` TxHash string `db:"tx_hash"` } + +// GetSliceResponse holds response for the eth_getSlice method +type GetSliceResponse struct { + SliceID string `json:"sliceId"` + MetaData GetSliceResponseMetadata `json:"metadata"` + TrieNodes GetSliceResponseTrieNodes `json:"trieNodes"` + Leaves map[string]GetSliceResponseAccount `json:"leaves"` // we won't be using addresses, but keccak256(address) // TODO: address comment +} + +func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) { + sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String()) + sr.MetaData = GetSliceResponseMetadata{ + NodeStats: make(map[string]string, 0), + TimeStats: make(map[string]string, 0), + } + sr.Leaves = make(map[string]GetSliceResponseAccount) + sr.TrieNodes = GetSliceResponseTrieNodes{ + Stem: make(map[string]string), + Head: make(map[string]string), + Slice: make(map[string]string), + } +} + +type GetSliceResponseMetadata struct { + TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one) + NodeStats map[string]string `json:"trieNodes"` // total, leaves, smart contracts +} + +type GetSliceResponseTrieNodes struct { + Stem map[string]string `json:"stem"` + Head map[string]string `json:"head"` + Slice map[string]string `json:"sliceNodes"` +} + +type GetSliceResponseAccount struct { + StorageRoot string `json:"storageRoot"` + EVMCode string `json:"evmCode"` +} -- 2.45.2 From 7cf38e742aa7a86b49d7476f0f16364e03462f0e Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 11:31:47 +0530 Subject: [PATCH 02/21] Implement getSlice API for storage nodes --- pkg/eth/backend.go | 127 +++++++++++++++++++++++++++++++++++--- pkg/eth/helpers.go | 2 +- pkg/eth/ipld_retriever.go | 64 ++++++++++++++++++- 3 files changed, 182 insertions(+), 11 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 8d1ed56c..817b677e 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -102,6 +102,14 @@ const ( WHERE state_root = $1 ORDER BY block_number DESC LIMIT 1` + RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT block_number, state_leaf_key + FROM eth.state_cids, eth.state_accounts + WHERE state_accounts.storage_root = $1 + AND state_cids.state_id = state_accounts.state_path + AND state_cids.storage_root = state_accounts.header_id + AND state_cids.storage_root = state_accounts.block_number + ORDER BY block_number DESC + LIMIT 1` ) const ( @@ -907,6 +915,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error()) } + // Get all the paths headPath, stemPaths, slicePaths, err := getPaths(path, depth) if err != nil { return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error()) @@ -931,7 +940,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS // Fetch stem nodes // some of the "stem" nodes can be leaf nodes (but not value nodes) - stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, stemPaths) + stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, stemPaths, blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error()) } @@ -939,7 +948,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent // Fetch slice nodes - sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, slicePaths) + sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, slicePaths, blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error()) } @@ -947,7 +956,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent // Fetch head node - headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, [][]byte{headPath}) + headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, [][]byte{headPath}, blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error()) } @@ -960,6 +969,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS leafNodes = append(leafNodes, sliceLeafCIDs...) leafNodes = append(leafNodes, headLeafCID...) // TODO: fill in contract data `response.Leaves` + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) maxDepth := deepestPath - len(headPath) if maxDepth < 0 { @@ -970,15 +980,84 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts - response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) return response, nil } func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { - // TODO Implement - return nil, nil + response := new(GetSliceResponse) + response.init(path, depth, root) + + // Start a timer + trieLoadingStart := makeTimestamp() + + // Get the block height and state leaf key for the input storage root + blockHeight, stateLeafKey, err := b.getBlockHeightAndStateLeafKeyForStorageRoot(root) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice blockheight and state key lookup error: %s", err.Error()) + } + + // Get all the paths + headPath, stemPaths, slicePaths, err := getPaths(path, depth) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice path generation error: %s", err.Error()) + } + response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart)) + + // Begin tx + tx, err := b.DB.Beginx() + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + + // Fetch stem nodes + // some of the "stem" nodes can be leaf nodes (but not value nodes) + stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, stemPaths, blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice stem node lookup error: %s", err.Error()) + } + response.TrieNodes.Stem = stemNodes + response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent + + // Fetch slice nodes + sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, slicePaths, blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice slice node lookup error: %s", err.Error()) + } + response.TrieNodes.Slice = sliceNodes + response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent + + // Fetch head node + headNode, headLeafCID, _, _, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, [][]byte{headPath}, blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice head node lookup error: %s", err.Error()) + } + response.TrieNodes.Head = headNode + + // Fill in metadata + maxDepth := deepestPath - len(headPath) + if maxDepth < 0 { + maxDepth = 0 + } + response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) + + response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) + response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(stemLeafCIDs) + len(sliceLeafCIDs) + len(headLeafCID)) + response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(0) + + return response, nil } func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { @@ -986,7 +1065,16 @@ func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String()) } -func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight uint64, paths [][]byte) (map[string]string, []cid.Cid, int, string, error) { +func (b *Backend) getBlockHeightAndStateLeafKeyForStorageRoot(root common.Hash) (uint64, string, error) { + var res struct { + BlockNumber uint64 `db:"block_number"` + StateLeafKey string `db:"state_leaf_key"` + } + + return res.BlockNumber, res.StateLeafKey, b.DB.Get(&res, RetrieveBlockNumberAndStateLeafKeyForStorageRoot, root.String()) +} + +func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { nodes := make(map[string]string) fetchStart := makeTimestamp() @@ -1011,6 +1099,31 @@ func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight ui return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil } +func (b *Backend) getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { + nodes := make(map[string]string) + fetchStart := makeTimestamp() + + // Get CIDs for all nodes at the provided paths + leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, paths, blockHeight) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for leaf nodes + err = populateNodesMap(nodes, leafCIDs, leafIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for intermediate nodes + err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index 6caded31..31783bb6 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -108,7 +108,7 @@ func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) e data := iplds[i] hash := crypto.Keccak256Hash(data) if !bytes.Equal(hash.Bytes(), decodedMh.Digest) { - panic("multihash digest should equal keccak of raw data") + return fmt.Errorf("multihash digest should equal keccak of raw data") } nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index fe970a82..24a628c3 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -239,7 +239,7 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.mh_key, state_cids.node_type + RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids INNER JOIN public.blocks ON ( state_cids.mh_key = blocks.key @@ -250,6 +250,23 @@ const ( AND node_type != 3 ORDER BY state_cids.block_number DESC LIMIT 1` + RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data + FROM eth.storage_cids + INNER JOIN eth.state_cids ON ( + storage_cids.state_path = state_cids.state_path + AND storage_cids.header_id = state_cids.header_id + AND storage_cids.block_number = state_cids.block_number + ) + INNER JOIN public.blocks ON ( + storage_cids.mh_key = blocks.key + AND storage_cids.block_number = blocks.block_number + ) + WHERE state_leaf_key = $1 + AND storage_path = $2 + AND storage_cids.block_number <= $3 + AND node_type != 3 + ORDER BY storage_cids.block_number DESC + LIMIT 1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON ( @@ -746,10 +763,51 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ leafNodeIPLDs := make([][]byte, 0) intermediateNodeIPLDs := make([][]byte, 0) + // TODO: fetch all nodes in a single query for _, path := range paths { - // Create a result object, select: cid, blockNumber and nodeType + // Create a result object, select: cid, data res := new(nodeInfo) - if err := r.db.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + if err := tx.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + return nil, nil, nil, nil, 0, err + } + + pathLen := len(path) + if pathLen > deepestPath { + deepestPath = pathLen + } + + cid, err := cid.Decode(res.CID) + if err != nil { + return nil, nil, nil, nil, 0, err + } + + if res.NodeType == sdtypes.Leaf.Int() { + leafNodeCIDs = append(leafNodeCIDs, cid) + leafNodeIPLDs = append(leafNodeIPLDs, res.Data) + } else { + intermediateNodeCIDs = append(intermediateNodeCIDs, cid) + intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) + } + } + + return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil +} + +// RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber returns the cid and rlp bytes for the storage nodes corresponding to the provided state leaf key, storage paths and block number +func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) { + deepestPath := 0 + + leafNodeCIDs := make([]cid.Cid, 0) + intermediateNodeCIDs := make([]cid.Cid, 0) + + leafNodeIPLDs := make([][]byte, 0) + intermediateNodeIPLDs := make([][]byte, 0) + + // TODO: fetch all nodes in a single query + for _, path := range paths { + // Create a result object, select: cid, data + res := new(nodeInfo) + if err := tx.Get(res, RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr, stateLeafKey, path, number); err != nil { return nil, nil, nil, nil, 0, err } -- 2.45.2 From bc408ec6804ff16dc4c5c9cf155a5d47f4a55573 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 14:42:05 +0530 Subject: [PATCH 03/21] Fix the helper function to create a slice of required paths --- pkg/eth/helpers.go | 39 +++++++++++++++++++++++++-------------- pkg/eth/ipld_retriever.go | 19 +++++++++++++++---- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index 31783bb6..498b6ae6 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -74,26 +74,37 @@ func getPaths(path string, depth int) ([]byte, [][]byte, [][]byte, error) { return headPath, stemPaths, slicePaths, nil } -// iterative function to generate the set of slice paths +// An iterative function to generate the set of slice paths func makeSlicePaths(path []byte, depth int, slicePaths *[][]byte) { - depth-- // decrement the depth - nextPaths := make([][]byte, 16) // slice to hold the next 16 paths - for i, step := range pathSteps { // iterate through steps - nextPath := append(path, step) // create next paths by adding steps to current path - nextPaths[i] = nextPath - newSlicePaths := append(*slicePaths, nextPath) // add next paths to the collection of all slice paths - slicePaths = &newSlicePaths - } - - if depth == 0 { // if depth has reach 0, return + // return if depth has reached 0 + if depth <= 0 { return } - for _, nextPath := range nextPaths { // if not, then we iterate over the next paths - makeSlicePaths(nextPath, depth, slicePaths) // and repeat the process for each one + depth-- + + // slice to hold the next 16 paths + nextPaths := make([][]byte, 0, 16) + for _, step := range pathSteps { + // create next paths by adding steps to current path + nextPath := make([]byte, len(path)) + copy(nextPath, path) + nextPath = append(nextPath, step) + + nextPaths = append(nextPaths, nextPath) + + // also add the next path to the collection of all slice paths + dst := make([]byte, len(nextPath)) + copy(dst, nextPath) + *slicePaths = append(*slicePaths, dst) + } + + // iterate over the next paths to repeat the process if not + for _, nextPath := range nextPaths { + makeSlicePaths(nextPath, depth, slicePaths) } } -// use to return timestamp in milliseconds +// Timestamp in milliseconds func makeTimestamp() int64 { return time.Now().UnixNano() / int64(time.Millisecond) } diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 24a628c3..fe36f991 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -17,6 +17,7 @@ package eth import ( + "database/sql" "fmt" "strconv" @@ -239,7 +240,7 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data + RetrieveStateByPathAndBlockNumberPgStr = `SELECT cid, data, node_type FROM eth.state_cids INNER JOIN public.blocks ON ( state_cids.mh_key = blocks.key @@ -250,7 +251,7 @@ const ( AND node_type != 3 ORDER BY state_cids.block_number DESC LIMIT 1` - RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data + RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type FROM eth.storage_cids INNER JOIN eth.state_cids ON ( storage_cids.state_path = state_cids.state_path @@ -765,9 +766,14 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ // TODO: fetch all nodes in a single query for _, path := range paths { - // Create a result object, select: cid, data + // Create a result object, select: cid, data, node_type res := new(nodeInfo) if err := tx.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + // we will not find a node for each path + if err == sql.ErrNoRows { + continue + } + return nil, nil, nil, nil, 0, err } @@ -805,9 +811,14 @@ func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx * // TODO: fetch all nodes in a single query for _, path := range paths { - // Create a result object, select: cid, data + // Create a result object, select: cid, data, node_type res := new(nodeInfo) if err := tx.Get(res, RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr, stateLeafKey, path, number); err != nil { + // we will not find a node for each path + if err == sql.ErrNoRows { + continue + } + return nil, nil, nil, nil, 0, err } -- 2.45.2 From fc8f19d791fa62bbebaec2b41a8f30571bc3f008 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 17:33:06 +0530 Subject: [PATCH 04/21] Fix query to get state leaf key for given storage root --- pkg/eth/backend.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 817b677e..a09184ac 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -105,10 +105,10 @@ const ( RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT block_number, state_leaf_key FROM eth.state_cids, eth.state_accounts WHERE state_accounts.storage_root = $1 - AND state_cids.state_id = state_accounts.state_path - AND state_cids.storage_root = state_accounts.header_id - AND state_cids.storage_root = state_accounts.block_number - ORDER BY block_number DESC + AND state_cids.state_path = state_accounts.state_path + AND state_cids.header_id = state_accounts.header_id + AND state_cids.block_number = state_accounts.block_number + ORDER BY state_accounts.block_number DESC LIMIT 1` ) -- 2.45.2 From 9cdef929f8bb0d40ea3d455994c44006e6915cca Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 18:55:30 +0530 Subject: [PATCH 05/21] Add a test to get state slice for root path --- pkg/eth/eth_state_test.go | 57 +++++++++++++++++++++++++++++++++++++++ pkg/eth/types.go | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 2fe62e89..3756eaab 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -19,6 +19,7 @@ package eth_test import ( "bytes" "context" + "fmt" "io/ioutil" "math/big" "time" @@ -44,6 +45,21 @@ import ( var ( parsedABI abi.ABI + + stateRoot = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43") + rootDataHash = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43" + rootData = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080" + + account1DataHash = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4" + account1Data = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659" + account2DataHash = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35" + account2Data = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account3DataHash = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2" + account3Data = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account4DataHash = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951" + account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91" + account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" ) func init() { @@ -529,4 +545,45 @@ var _ = Describe("eth state reading tests", func() { Expect(header).To(Equal(expectedCanonicalHeader)) }) }) + + Describe("eth_getSlice", func() { + It("Retrieves the state nodes for root path", func() { + path := "0x" + depth := 3 + sliceResponse, err := api.GetSlice(ctx, "0x", 3, stateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, stateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "1", + "02-total-trie-nodes": "6", + "03-leaves": "5", + "04-smart-contracts": "", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + rootDataHash: rootData, + }, + Slice: map[string]string{ + account1DataHash: account1Data, + account2DataHash: account2Data, + account3DataHash: account3Data, + account4DataHash: account4Data, + account5DataHash: account5Data, + }, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID)) + Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats)) + Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes)) + Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves)) + }) + }) }) diff --git a/pkg/eth/types.go b/pkg/eth/types.go index ada668b5..6849b9d3 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -290,7 +290,7 @@ func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) { type GetSliceResponseMetadata struct { TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one) - NodeStats map[string]string `json:"trieNodes"` // total, leaves, smart contracts + NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts } type GetSliceResponseTrieNodes struct { -- 2.45.2 From 8e061504a2329b2071dc1de959af49efbd05f4ec Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 20:04:03 +0530 Subject: [PATCH 06/21] Add checks in queries to get canonical data --- pkg/eth/backend.go | 6 ++++-- pkg/eth/ipld_retriever.go | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index a09184ac..6c2d6caf 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -100,6 +100,7 @@ const ( RetrieveBlockNumberForStateRoot = `SELECT block_number FROM eth.header_cids WHERE state_root = $1 + AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY block_number DESC LIMIT 1` RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT block_number, state_leaf_key @@ -108,6 +109,7 @@ const ( AND state_cids.state_path = state_accounts.state_path AND state_cids.header_id = state_accounts.header_id AND state_cids.block_number = state_accounts.block_number + AND state_accounts.header_id = (SELECT canonical_header_hash(state_accounts.block_number)) ORDER BY state_accounts.block_number DESC LIMIT 1` ) @@ -977,10 +979,10 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS } response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) - response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) + response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head) + len(response.TrieNodes.Slice)) response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts - response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) + response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head)) return response, nil } diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index fe36f991..c12ed34d 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -240,7 +240,7 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveStateByPathAndBlockNumberPgStr = `SELECT cid, data, node_type + RetrieveStateByPathAndBlockNumberPgStr = `SELECT cid, data, node_type, state_cids.block_number FROM eth.state_cids INNER JOIN public.blocks ON ( state_cids.mh_key = blocks.key @@ -249,6 +249,7 @@ const ( WHERE state_path = $1 AND state_cids.block_number <= $2 AND node_type != 3 + AND state_cids.header_id = (SELECT canonical_header_hash(state_cids.block_number)) ORDER BY state_cids.block_number DESC LIMIT 1` RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type @@ -266,6 +267,7 @@ const ( AND storage_path = $2 AND storage_cids.block_number <= $3 AND node_type != 3 + AND storage_cids.header_id = (SELECT canonical_header_hash(storage_cids.block_number)) ORDER BY storage_cids.block_number DESC LIMIT 1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type -- 2.45.2 From 24a68fd7bae699b7e792fa0c28bfdd1bd398ad22 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 20:12:40 +0530 Subject: [PATCH 07/21] Add tests to get state slice --- pkg/eth/eth_state_test.go | 112 ++++++++++++++++++++++++++++++++++---- pkg/eth/test_helpers.go | 8 +++ 2 files changed, 109 insertions(+), 11 deletions(-) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 3756eaab..f8d27ab7 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -46,9 +46,13 @@ import ( var ( parsedABI abi.ABI - stateRoot = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43") - rootDataHash = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43" - rootData = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080" + block1StateRoot = common.HexToHash("0xa1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21") + rootDataHashBlock1 = "a1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21" + rootDataBlock1 = "f871a0577652b625b77bdb5bf77bc43f3125cad7464d679d1575565277d3611b8053e780808080a0fe889f10e5db8f2c2bf355928152a17f6e3bb99a9241ac6d84c77e6264509c798080808080808080a011db0cda34a896dabeb6839bb06a38f49514cfa486435984eb013b7df9ee85c58080" + + block5StateRoot = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43") + rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43" + rootDataBlock5 = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080" account1DataHash = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4" account1Data = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659" @@ -547,14 +551,14 @@ var _ = Describe("eth state reading tests", func() { }) Describe("eth_getSlice", func() { - It("Retrieves the state nodes for root path", func() { + It("Retrieves the state slice for root path", func() { path := "0x" depth := 3 - sliceResponse, err := api.GetSlice(ctx, "0x", 3, stateRoot, false) + sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false) Expect(err).ToNot(HaveOccurred()) expectedResponse := eth.GetSliceResponse{ - SliceID: fmt.Sprintf("%s-%d-%s", path, depth, stateRoot.String()), + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()), MetaData: eth.GetSliceResponseMetadata{ NodeStats: map[string]string{ "00-stem-and-head-nodes": "1", @@ -567,7 +571,7 @@ var _ = Describe("eth state reading tests", func() { TrieNodes: eth.GetSliceResponseTrieNodes{ Stem: map[string]string{}, Head: map[string]string{ - rootDataHash: rootData, + rootDataHashBlock5: rootDataBlock5, }, Slice: map[string]string{ account1DataHash: account1Data, @@ -580,10 +584,96 @@ var _ = Describe("eth state reading tests", func() { Leaves: map[string]eth.GetSliceResponseAccount{}, } - Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID)) - Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats)) - Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes)) - Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves)) + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the state slice for root path with 0 depth", func() { + path := "0x" + depth := 0 + sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "0", + "04-smart-contracts": "", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + rootDataHashBlock5: rootDataBlock5, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the state slice for a path to an account", func() { + path := "0x06" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "2", + "01-max-depth": "0", + "02-total-trie-nodes": "2", + "03-leaves": "1", + "04-smart-contracts": "", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{ + rootDataHashBlock5: rootDataBlock5, + }, + Head: map[string]string{ + account1DataHash: account1Data, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the state slice for a path to a non-existing account", func() { + path := "0x06" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, block1StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block1StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "0", + "04-smart-contracts": "", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{ + rootDataHashBlock1: rootDataBlock1, + }, + Head: map[string]string{}, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) }) }) }) diff --git a/pkg/eth/test_helpers.go b/pkg/eth/test_helpers.go index f59f4b43..f2eb1d76 100644 --- a/pkg/eth/test_helpers.go +++ b/pkg/eth/test_helpers.go @@ -18,6 +18,7 @@ package eth import ( "github.com/ethereum/go-ethereum/statediff/indexer/models" + . "github.com/onsi/gomega" ) // TxModelsContainsCID used to check if a list of TxModels contains a specific cid string @@ -39,3 +40,10 @@ func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool { } return false } + +func CheckGetSliceResponse(sliceResponse GetSliceResponse, expectedResponse GetSliceResponse) { + Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID)) + Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats)) + Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes)) + Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves)) +} -- 2.45.2 From 13c75f50e2072385494a59f88d0ef1fa85e50664 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 20:24:11 +0530 Subject: [PATCH 08/21] Add a todo for using an iterator --- pkg/eth/backend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 6c2d6caf..5b5a0114 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -917,6 +917,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error()) } + // TODO: Use an iterator instead of doing an exhausitive database search for all possible paths at the given depth // Get all the paths headPath, stemPaths, slicePaths, err := getPaths(path, depth) if err != nil { -- 2.45.2 From a549bd408f1d75e9bf8b2aadb928b9ee4e0b658e Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 14 Dec 2022 10:00:34 +0530 Subject: [PATCH 09/21] Avoid filtering out removed nodes --- pkg/eth/backend.go | 2 +- pkg/eth/ipld_retriever.go | 34 +++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 5b5a0114..152290a9 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -103,7 +103,7 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY block_number DESC LIMIT 1` - RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT block_number, state_leaf_key + RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT state_accounts.block_number, state_leaf_key FROM eth.state_cids, eth.state_accounts WHERE state_accounts.storage_root = $1 AND state_cids.state_path = state_accounts.state_path diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index c12ed34d..ca5b1a21 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -34,10 +34,6 @@ import ( ) const ( - // node type removed value. - // https://github.com/cerc-io/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34 - removedNode = 3 - RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON ( @@ -248,11 +244,10 @@ const ( ) WHERE state_path = $1 AND state_cids.block_number <= $2 - AND node_type != 3 AND state_cids.header_id = (SELECT canonical_header_hash(state_cids.block_number)) ORDER BY state_cids.block_number DESC LIMIT 1` - RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type + RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, storage_cids.block_number FROM eth.storage_cids INNER JOIN eth.state_cids ON ( storage_cids.state_path = state_cids.state_path @@ -266,7 +261,6 @@ const ( WHERE state_leaf_key = $1 AND storage_path = $2 AND storage_cids.block_number <= $3 - AND node_type != 3 AND storage_cids.header_id = (SELECT canonical_header_hash(storage_cids.block_number)) ORDER BY storage_cids.block_number DESC LIMIT 1` @@ -643,7 +637,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr return "", nil, err } - if accountResult.NodeType == removedNode { + if accountResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, nil } @@ -675,7 +669,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad return "", nil, err } - if accountResult.NodeType == removedNode { + if accountResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, nil } @@ -703,7 +697,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } - if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { + if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, EmptyNodeValue, nil } @@ -736,7 +730,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad return "", nil, err } - if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { + if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, nil } @@ -771,7 +765,7 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ // Create a result object, select: cid, data, node_type res := new(nodeInfo) if err := tx.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { - // we will not find a node for each path + // Skip if node not found for a path if err == sql.ErrNoRows { continue } @@ -779,6 +773,11 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ return nil, nil, nil, nil, 0, err } + // Skip if node is of removed type + if res.NodeType == sdtypes.Removed.Int() { + continue + } + pathLen := len(path) if pathLen > deepestPath { deepestPath = pathLen @@ -790,9 +789,11 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ } if res.NodeType == sdtypes.Leaf.Int() { + fmt.Println("found leaf node for path", path, res.BlockNumber) leafNodeCIDs = append(leafNodeCIDs, cid) leafNodeIPLDs = append(leafNodeIPLDs, res.Data) } else { + fmt.Println("found intermediate node for path", path, res.NodeType, res.BlockNumber) intermediateNodeCIDs = append(intermediateNodeCIDs, cid) intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) } @@ -816,7 +817,7 @@ func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx * // Create a result object, select: cid, data, node_type res := new(nodeInfo) if err := tx.Get(res, RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr, stateLeafKey, path, number); err != nil { - // we will not find a node for each path + // Skip if node not found for a path if err == sql.ErrNoRows { continue } @@ -824,6 +825,11 @@ func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx * return nil, nil, nil, nil, 0, err } + // Skip if node is of removed type + if res.NodeType == sdtypes.Removed.Int() { + continue + } + pathLen := len(path) if pathLen > deepestPath { deepestPath = pathLen @@ -835,9 +841,11 @@ func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx * } if res.NodeType == sdtypes.Leaf.Int() { + fmt.Println("found leaf node for path", path, res.BlockNumber) leafNodeCIDs = append(leafNodeCIDs, cid) leafNodeIPLDs = append(leafNodeIPLDs, res.Data) } else { + fmt.Println("found intermediate node for path", path, res.NodeType, res.BlockNumber) intermediateNodeCIDs = append(intermediateNodeCIDs, cid) intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) } -- 2.45.2 From 33f762bea9929c08f0f2815f73e8861c1ae84e70 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 14 Dec 2022 10:31:45 +0530 Subject: [PATCH 10/21] Add tests to get storage slice --- pkg/eth/eth_state_test.go | 131 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index f8d27ab7..12b9e117 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -64,6 +64,19 @@ var ( account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91" account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + + contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0") + storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0" + storageRootDataBlock5 = "f838a120290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7" + + contractStorageRootBlock4 = common.HexToHash("0x64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68") + storageRootDataHashBlock4 = "64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68" + storageRootDataBlock4 = "f8518080a08e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d558080808080808080a07b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af8080808080" + + storageNode1DataHash = "7b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af" + storageNode1Data = "e2a0310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf609" + storageNode2DataHash = "8e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d55" + storageNode2Data = "f7a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7" ) func init() { @@ -675,5 +688,123 @@ var _ = Describe("eth state reading tests", func() { eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) }) + + It("Retrieves the storage slice for root path", func() { + path := "0x" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "1", + "02-total-trie-nodes": "3", + "03-leaves": "2", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + storageRootDataHashBlock4: storageRootDataBlock4, + }, + Slice: map[string]string{ + storageNode1DataHash: storageNode1Data, + storageNode2DataHash: storageNode2Data, + }, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the storage slice for root path with 0 depth", func() { + path := "0x" + depth := 0 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + storageRootDataHashBlock4: storageRootDataBlock4, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the storage slice for root path with deleted nodes", func() { + path := "0x" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock5, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock5.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "1", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + storageRootDataHashBlock5: storageRootDataBlock5, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the storage slice for a path to a storage node", func() { + path := "0x0b" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "2", + "01-max-depth": "0", + "02-total-trie-nodes": "2", + "03-leaves": "1", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{ + storageRootDataHashBlock4: storageRootDataBlock4, + }, + Head: map[string]string{ + storageNode1DataHash: storageNode1Data, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) }) }) -- 2.45.2 From 16fc1d4957a2de31e03a6211fa143d2d4e9c870e Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 14 Dec 2022 10:35:54 +0530 Subject: [PATCH 11/21] Remove logs --- pkg/eth/ipld_retriever.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index ca5b1a21..adb9e46f 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -236,7 +236,7 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveStateByPathAndBlockNumberPgStr = `SELECT cid, data, node_type, state_cids.block_number + RetrieveStateByPathAndBlockNumberPgStr = `SELECT cid, data, node_type FROM eth.state_cids INNER JOIN public.blocks ON ( state_cids.mh_key = blocks.key @@ -247,7 +247,7 @@ const ( AND state_cids.header_id = (SELECT canonical_header_hash(state_cids.block_number)) ORDER BY state_cids.block_number DESC LIMIT 1` - RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, storage_cids.block_number + RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type FROM eth.storage_cids INNER JOIN eth.state_cids ON ( storage_cids.state_path = state_cids.state_path @@ -789,11 +789,9 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ } if res.NodeType == sdtypes.Leaf.Int() { - fmt.Println("found leaf node for path", path, res.BlockNumber) leafNodeCIDs = append(leafNodeCIDs, cid) leafNodeIPLDs = append(leafNodeIPLDs, res.Data) } else { - fmt.Println("found intermediate node for path", path, res.NodeType, res.BlockNumber) intermediateNodeCIDs = append(intermediateNodeCIDs, cid) intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) } @@ -841,11 +839,9 @@ func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx * } if res.NodeType == sdtypes.Leaf.Int() { - fmt.Println("found leaf node for path", path, res.BlockNumber) leafNodeCIDs = append(leafNodeCIDs, cid) leafNodeIPLDs = append(leafNodeIPLDs, res.Data) } else { - fmt.Println("found intermediate node for path", path, res.NodeType, res.BlockNumber) intermediateNodeCIDs = append(intermediateNodeCIDs, cid) intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) } -- 2.45.2 From 4ea8620bc80ea55a3584688b3e272eaa0b6d806f Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 14 Dec 2022 13:46:58 +0530 Subject: [PATCH 12/21] Populate extra contract leaves field in the response --- pkg/eth/api_test.go | 4 --- pkg/eth/backend.go | 59 +++++++++++++++++++++++++++++++++++++++++++-- pkg/eth/types.go | 4 +-- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index 91d80b1d..feb500bf 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -1145,8 +1145,4 @@ var _ = Describe("API", func() { Expect(code).To(BeEmpty()) }) }) - - Describe("eth_getSlice", func() { - // TODO Implement - }) }) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 152290a9..fa1f8fb7 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -109,9 +109,21 @@ const ( AND state_cids.state_path = state_accounts.state_path AND state_cids.header_id = state_accounts.header_id AND state_cids.block_number = state_accounts.block_number + AND state_cids.node_type != 3 AND state_accounts.header_id = (SELECT canonical_header_hash(state_accounts.block_number)) ORDER BY state_accounts.block_number DESC LIMIT 1` + RetrieveAccountByStateCID = `SELECT state_leaf_key, storage_root, code_hash + FROM eth.state_cids + INNER JOIN eth.state_accounts ON ( + state_cids.state_path = state_accounts.state_path + AND state_cids.header_id = state_accounts.header_id + AND state_cids.block_number = state_accounts.block_number + ) + WHERE state_cids.cid = $1 + AND state_cids.block_number <= $2 + ORDER BY state_cids.block_number DESC + LIMIT 1` ) const ( @@ -971,7 +983,24 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS leafNodes = append(leafNodes, stemLeafCIDs...) leafNodes = append(leafNodes, sliceLeafCIDs...) leafNodes = append(leafNodes, headLeafCID...) - // TODO: fill in contract data `response.Leaves` + + contractCount := 0 + for _, leafNodeCID := range leafNodes { + stateLeafKey, storageRoot, code, err := b.getAccountByStateCID(tx, leafNodeCID.String(), blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStateSlice account lookup error: %s", err.Error()) + } + + response.Leaves[stateLeafKey] = GetSliceResponseAccount{ + StorageRoot: storageRoot, + EVMCode: common.Bytes2Hex(code), + } + + if len(code) > 0 { + contractCount++ + } + } + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) maxDepth := deepestPath - len(headPath) @@ -982,7 +1011,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head) + len(response.TrieNodes.Slice)) response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) - response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts + response.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(contractCount) response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head)) return response, nil @@ -1102,6 +1131,32 @@ func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil } +func (b *Backend) getAccountByStateCID(tx *sqlx.Tx, cid string, blockHeight uint64) (string, string, []byte, error) { + var err error + var res struct { + StateLeafKey string `db:"state_leaf_key"` + StorageRoot string `db:"storage_root"` + CodeHash []byte `db:"code_hash"` + } + + if err = tx.Get(&res, RetrieveAccountByStateCID, cid, blockHeight); err != nil { + return "", "", nil, err + } + + mhKey, err := ethServerShared.MultihashKeyFromKeccak256(common.BytesToHash(res.CodeHash)) + if err != nil { + return "", "", nil, err + } + + code := make([]byte, 0) + err = tx.Get(&code, RetrieveCodeByMhKey, mhKey) + if err != nil { + return "", "", nil, err + } + + return res.StateLeafKey, res.StorageRoot, code, nil +} + func (b *Backend) getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { nodes := make(map[string]string) fetchStart := makeTimestamp() diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 6849b9d3..d9631786 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -271,7 +271,7 @@ type GetSliceResponse struct { SliceID string `json:"sliceId"` MetaData GetSliceResponseMetadata `json:"metadata"` TrieNodes GetSliceResponseTrieNodes `json:"trieNodes"` - Leaves map[string]GetSliceResponseAccount `json:"leaves"` // we won't be using addresses, but keccak256(address) // TODO: address comment + Leaves map[string]GetSliceResponseAccount `json:"leaves"` // key: Keccak256Hash(address) in hex (leafKey) } func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) { @@ -294,7 +294,7 @@ type GetSliceResponseMetadata struct { } type GetSliceResponseTrieNodes struct { - Stem map[string]string `json:"stem"` + Stem map[string]string `json:"stem"` // key: Keccak256Hash(data) in hex, value: trie node data in hex Head map[string]string `json:"head"` Slice map[string]string `json:"sliceNodes"` } -- 2.45.2 From 2b8de130dc871bb19de9f876b4deb55340c2f5d0 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 14 Dec 2022 14:14:34 +0530 Subject: [PATCH 13/21] Update tests --- pkg/eth/api_test.go | 5 +-- pkg/eth/cid_retriever_test.go | 1 + pkg/eth/eth_state_test.go | 72 ++++++++++++++++++++++++++--------- pkg/eth/filterer.go | 9 ++--- 4 files changed, 59 insertions(+), 28 deletions(-) diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index feb500bf..bbbd1a06 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -24,7 +24,6 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" - ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -202,8 +201,8 @@ var _ = Describe("API", func() { ChainConfig: chainConfig, VMConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call. - GroupCacheConfig: ðServerShared.GroupCacheConfig{ - StateDB: ethServerShared.GroupConfig{ + GroupCacheConfig: &shared.GroupCacheConfig{ + StateDB: shared.GroupConfig{ Name: "api_test", CacheSizeInMB: 8, CacheExpiryInMins: 60, diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/cid_retriever_test.go index 29f9c6c8..5ab41827 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/cid_retriever_test.go @@ -300,6 +300,7 @@ var _ = Describe("Retriever", func() { AND header_cids.block_number = $1 ORDER BY transaction_cids.index` err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) + Expect(err).ToNot(HaveOccurred()) cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1) Expect(err).ToNot(HaveOccurred()) Expect(empty).ToNot(BeTrue()) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 12b9e117..d3cb664a 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -27,7 +27,6 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" - ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -54,16 +53,25 @@ var ( rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43" rootDataBlock5 = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080" - account1DataHash = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4" - account1Data = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659" - account2DataHash = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35" - account2Data = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account3DataHash = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2" - account3Data = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account4DataHash = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951" - account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91" - account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account1DataHash = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4" + account1Data = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659" + account1StateLeafKey = "0x6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45" + account1Code = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032" + account2DataHash = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35" + account2Data = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account2StateLeafKey = "0xe926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2" + account3DataHash = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2" + account3Data = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account3StateLeafKey = "0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a" + account4DataHash = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951" + account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account4StateLeafKey = "0xc957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45" + account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91" + account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account5StateLeafKey = "0x5380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a" + + emptyStorageRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + emptyCode = "" contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0") storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0" @@ -114,8 +122,8 @@ var _ = Describe("eth state reading tests", func() { ChainConfig: chainConfig, VMConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call. - GroupCacheConfig: ðServerShared.GroupCacheConfig{ - StateDB: ethServerShared.GroupConfig{ + GroupCacheConfig: &shared.GroupCacheConfig{ + StateDB: shared.GroupConfig{ Name: "eth_state_test", CacheSizeInMB: 8, CacheExpiryInMins: 60, @@ -578,7 +586,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "1", "02-total-trie-nodes": "6", "03-leaves": "5", - "04-smart-contracts": "", + "04-smart-contracts": "1", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ @@ -594,7 +602,28 @@ var _ = Describe("eth state reading tests", func() { account5DataHash: account5Data, }, }, - Leaves: map[string]eth.GetSliceResponseAccount{}, + Leaves: map[string]eth.GetSliceResponseAccount{ + account1StateLeafKey: { + StorageRoot: contractStorageRootBlock5.Hex(), + EVMCode: account1Code, + }, + account2StateLeafKey: { + StorageRoot: emptyStorageRoot.Hex(), + EVMCode: emptyCode, + }, + account3StateLeafKey: { + StorageRoot: emptyStorageRoot.Hex(), + EVMCode: emptyCode, + }, + account4StateLeafKey: { + StorageRoot: emptyStorageRoot.Hex(), + EVMCode: emptyCode, + }, + account5StateLeafKey: { + StorageRoot: emptyStorageRoot.Hex(), + EVMCode: emptyCode, + }, + }, } eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) @@ -613,7 +642,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "0", "02-total-trie-nodes": "1", "03-leaves": "0", - "04-smart-contracts": "", + "04-smart-contracts": "0", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ @@ -642,7 +671,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "0", "02-total-trie-nodes": "2", "03-leaves": "1", - "04-smart-contracts": "", + "04-smart-contracts": "1", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ @@ -654,7 +683,12 @@ var _ = Describe("eth state reading tests", func() { }, Slice: map[string]string{}, }, - Leaves: map[string]eth.GetSliceResponseAccount{}, + Leaves: map[string]eth.GetSliceResponseAccount{ + account1StateLeafKey: { + StorageRoot: contractStorageRootBlock5.Hex(), + EVMCode: account1Code, + }, + }, } eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) @@ -673,7 +707,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "0", "02-total-trie-nodes": "1", "03-leaves": "0", - "04-smart-contracts": "", + "04-smart-contracts": "0", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ diff --git a/pkg/eth/filterer.go b/pkg/eth/filterer.go index 8a17f446..d4cc936a 100644 --- a/pkg/eth/filterer.go +++ b/pkg/eth/filterer.go @@ -208,7 +208,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa } // If there are no wanted contract addresses, we keep all receipts that match the topic filter if len(wantedAddresses) == 0 { - if match := filterMatch(wantedTopics, actualTopics); match == true { + if match := filterMatch(wantedTopics, actualTopics); match { return true } } @@ -218,7 +218,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa for _, actualAddr := range actualAddresses { if wantedAddr == actualAddr { // we keep the receipt if it matches on the topic filter - if match := filterMatch(wantedTopics, actualTopics); match == true { + if match := filterMatch(wantedTopics, actualTopics); match { return true } } @@ -240,10 +240,7 @@ func filterMatch(wantedTopics, actualTopics [][]string) bool { matches++ } } - if matches == 4 { - return true - } - return false + return matches == 4 } // returns 1 if the two slices have a string in common, 0 if they do not -- 2.45.2 From a338bd1a59550c32324e70ee6c7b184e82c746de Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 15 Dec 2022 15:13:14 +0530 Subject: [PATCH 14/21] Avoid EOAs in additional data in response --- pkg/eth/backend.go | 13 +++++-------- pkg/eth/eth_state_test.go | 23 ----------------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index fa1f8fb7..439743b5 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -984,20 +984,17 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS leafNodes = append(leafNodes, sliceLeafCIDs...) leafNodes = append(leafNodes, headLeafCID...) - contractCount := 0 for _, leafNodeCID := range leafNodes { stateLeafKey, storageRoot, code, err := b.getAccountByStateCID(tx, leafNodeCID.String(), blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice account lookup error: %s", err.Error()) } - response.Leaves[stateLeafKey] = GetSliceResponseAccount{ - StorageRoot: storageRoot, - EVMCode: common.Bytes2Hex(code), - } - if len(code) > 0 { - contractCount++ + response.Leaves[stateLeafKey] = GetSliceResponseAccount{ + StorageRoot: storageRoot, + EVMCode: common.Bytes2Hex(code), + } } } @@ -1011,7 +1008,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head) + len(response.TrieNodes.Slice)) response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) - response.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(contractCount) + response.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(response.Leaves)) response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head)) return response, nil diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index d3cb664a..32d78d24 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -59,19 +59,12 @@ var ( account1Code = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032" account2DataHash = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35" account2Data = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account2StateLeafKey = "0xe926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2" account3DataHash = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2" account3Data = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account3StateLeafKey = "0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a" account4DataHash = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951" account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account4StateLeafKey = "0xc957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45" account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91" account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - account5StateLeafKey = "0x5380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a" - - emptyStorageRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - emptyCode = "" contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0") storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0" @@ -607,22 +600,6 @@ var _ = Describe("eth state reading tests", func() { StorageRoot: contractStorageRootBlock5.Hex(), EVMCode: account1Code, }, - account2StateLeafKey: { - StorageRoot: emptyStorageRoot.Hex(), - EVMCode: emptyCode, - }, - account3StateLeafKey: { - StorageRoot: emptyStorageRoot.Hex(), - EVMCode: emptyCode, - }, - account4StateLeafKey: { - StorageRoot: emptyStorageRoot.Hex(), - EVMCode: emptyCode, - }, - account5StateLeafKey: { - StorageRoot: emptyStorageRoot.Hex(), - EVMCode: emptyCode, - }, }, } -- 2.45.2 From 2a194a94882607a2f7c167a39c8d25cc1a69db91 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 15 Dec 2022 15:21:10 +0530 Subject: [PATCH 15/21] Use iterator based approach for getSlice --- pkg/eth/api.go | 5 +- pkg/eth/backend.go | 354 ++++++++++---------------------------- pkg/eth/backend_utils.go | 117 +++++++++++++ pkg/eth/helpers.go | 86 --------- pkg/eth/ipld_retriever.go | 130 -------------- pkg/eth/types.go | 23 +++ 6 files changed, 236 insertions(+), 479 deletions(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index b7e4da8e..a3605a4d 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -839,10 +839,7 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre // GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) { - if storage { - return pea.B.GetStorageSlice(path, depth, root) - } - return pea.B.GetStateSlice(path, depth, root) + return pea.B.GetSlice(path, depth, root, storage) } // revertError is an API error that encompassas an EVM revertal with JSON error diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 439743b5..6f7e9385 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -42,13 +42,11 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/ethereum/go-ethereum/trie" - "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" - ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" - "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" ) @@ -96,34 +94,7 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` - RetrieveBlockNumberForStateRoot = `SELECT block_number - FROM eth.header_cids - WHERE state_root = $1 - AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) - ORDER BY block_number DESC - LIMIT 1` - RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT state_accounts.block_number, state_leaf_key - FROM eth.state_cids, eth.state_accounts - WHERE state_accounts.storage_root = $1 - AND state_cids.state_path = state_accounts.state_path - AND state_cids.header_id = state_accounts.header_id - AND state_cids.block_number = state_accounts.block_number - AND state_cids.node_type != 3 - AND state_accounts.header_id = (SELECT canonical_header_hash(state_accounts.block_number)) - ORDER BY state_accounts.block_number DESC - LIMIT 1` - RetrieveAccountByStateCID = `SELECT state_leaf_key, storage_root, code_hash - FROM eth.state_cids - INNER JOIN eth.state_accounts ON ( - state_cids.state_path = state_accounts.state_path - AND state_cids.header_id = state_accounts.header_id - AND state_cids.block_number = state_accounts.block_number - ) - WHERE state_cids.cid = $1 - AND state_cids.block_number <= $2 - ORDER BY state_cids.block_number DESC - LIMIT 1` + RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` ) const ( @@ -916,267 +887,132 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, return storageRlp, err } -func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { +func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) { response := new(GetSliceResponse) response.init(path, depth, root) - // Start a timer - trieLoadingStart := makeTimestamp() + t, _ := b.StateDatabase.OpenTrie(root) + headPath := common.FromHex(path) - // Get the block height for the input state root - blockHeight, err := b.getBlockHeightForStateRoot(root) - if err != nil { - return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error()) - } + // Metadata fields + metaData := metaDataFields{} - // TODO: Use an iterator instead of doing an exhausitive database search for all possible paths at the given depth - // Get all the paths - headPath, stemPaths, slicePaths, err := getPaths(path, depth) - if err != nil { - return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error()) - } - response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart)) - - // Begin tx - tx, err := b.DB.Beginx() + // Get Stem nodes + err := b.getSliceStem(headPath, t, response, &metaData, storage) if err != nil { return nil, err } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) - } else { - err = tx.Commit() - } - }() - // Fetch stem nodes - // some of the "stem" nodes can be leaf nodes (but not value nodes) - stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, stemPaths, blockHeight) + // Get Head node + err = b.getSliceHead(headPath, t, response, &metaData, storage) if err != nil { - return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error()) + return nil, err } - response.TrieNodes.Stem = stemNodes - response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent - // Fetch slice nodes - sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, slicePaths, blockHeight) - if err != nil { - return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error()) - } - response.TrieNodes.Slice = sliceNodes - response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent - - // Fetch head node - headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, [][]byte{headPath}, blockHeight) - if err != nil { - return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error()) - } - response.TrieNodes.Head = headNode - - // Fetch leaf contract data and fill in remaining metadata - leafFetchStart := makeTimestamp() - leafNodes := make([]cid.Cid, 0, len(stemLeafCIDs)+len(sliceLeafCIDs)+len(headLeafCID)) - leafNodes = append(leafNodes, stemLeafCIDs...) - leafNodes = append(leafNodes, sliceLeafCIDs...) - leafNodes = append(leafNodes, headLeafCID...) - - for _, leafNodeCID := range leafNodes { - stateLeafKey, storageRoot, code, err := b.getAccountByStateCID(tx, leafNodeCID.String(), blockHeight) + if depth > 0 { + // Get Slice nodes + err = b.getSliceTrie(headPath, t, response, &metaData, depth, storage) if err != nil { - return nil, fmt.Errorf("GetStateSlice account lookup error: %s", err.Error()) - } - - if len(code) > 0 { - response.Leaves[stateLeafKey] = GetSliceResponseAccount{ - StorageRoot: storageRoot, - EVMCode: common.Bytes2Hex(code), - } + return nil, err } } - response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) - - maxDepth := deepestPath - len(headPath) - if maxDepth < 0 { - maxDepth = 0 - } - response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) - - response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head) + len(response.TrieNodes.Slice)) - response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) - response.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(response.Leaves)) - response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Head)) + response.populateMetaData(metaData) return response, nil } -func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { - response := new(GetSliceResponse) - response.init(path, depth, root) +func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error { + for i := 0; i < len(headPath); i++ { + // Create path for each node along the stem + startPath := make([]byte, len(headPath[:i])) + copy(startPath, headPath[:i]) - // Start a timer - trieLoadingStart := makeTimestamp() + // Create an iterator initialized at startPath + it, timeTaken := getIteratorAtPath(t, startPath) + metaData.trieLoadingTime += timeTaken - // Get the block height and state leaf key for the input storage root - blockHeight, stateLeafKey, err := b.getBlockHeightAndStateLeafKeyForStorageRoot(root) - if err != nil { - return nil, fmt.Errorf("GetStorageSlice blockheight and state key lookup error: %s", err.Error()) + sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Stem, response.Leaves, it, storage) + if err != nil { + return err + } + + // Update metadata + if (sliceNodeMetrics.pathLen - len(headPath)) > metaData.maxDepth { + metaData.maxDepth = sliceNodeMetrics.pathLen + } + if sliceNodeMetrics.isLeaf { + metaData.leafCount++ + } + metaData.stemNodesFetchTime += sliceNodeMetrics.nodeFetchTime + metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime } - // Get all the paths - headPath, stemPaths, slicePaths, err := getPaths(path, depth) - if err != nil { - return nil, fmt.Errorf("GetStorageSlice path generation error: %s", err.Error()) - } - response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart)) + return nil +} - // Begin tx - tx, err := b.DB.Beginx() +func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error { + // Create an iterator initialized at headPath + it, timeTaken := getIteratorAtPath(t, headPath) + metaData.trieLoadingTime += timeTaken + + sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Head, response.Leaves, it, storage) if err != nil { - return nil, err + return err } - defer func() { - if p := recover(); p != nil { - shared.Rollback(tx) - panic(p) - } else if err != nil { - shared.Rollback(tx) + + // Update metadata + if (sliceNodeMetrics.pathLen - len(headPath)) > metaData.maxDepth { + metaData.maxDepth = sliceNodeMetrics.pathLen + } + if sliceNodeMetrics.isLeaf { + metaData.leafCount++ + } + metaData.stemNodesFetchTime += sliceNodeMetrics.nodeFetchTime + metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime + + return nil +} + +func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, depth int, storage bool) error { + it, timeTaken := getIteratorAtPath(t, headPath) + metaData.trieLoadingTime += timeTaken + + headPathLen := len(headPath) + maxPathLen := headPathLen + depth + descend := true + for it.Next(descend) { + pathLen := len(it.Path()) + + // End iteration on coming out of subtrie + if pathLen <= headPathLen { + break + } + + // Avoid descending further if max depth reached + if pathLen >= maxPathLen { + descend = false } else { - err = tx.Commit() + descend = true } - }() - // Fetch stem nodes - // some of the "stem" nodes can be leaf nodes (but not value nodes) - stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, stemPaths, blockHeight) - if err != nil { - return nil, fmt.Errorf("GetStorageSlice stem node lookup error: %s", err.Error()) - } - response.TrieNodes.Stem = stemNodes - response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent + sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Slice, response.Leaves, it, storage) + if err != nil { + return err + } - // Fetch slice nodes - sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, slicePaths, blockHeight) - if err != nil { - return nil, fmt.Errorf("GetStorageSlice slice node lookup error: %s", err.Error()) - } - response.TrieNodes.Slice = sliceNodes - response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent - - // Fetch head node - headNode, headLeafCID, _, _, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, [][]byte{headPath}, blockHeight) - if err != nil { - return nil, fmt.Errorf("GetStorageSlice head node lookup error: %s", err.Error()) - } - response.TrieNodes.Head = headNode - - // Fill in metadata - maxDepth := deepestPath - len(headPath) - if maxDepth < 0 { - maxDepth = 0 - } - response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) - - response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) - response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(stemLeafCIDs) + len(sliceLeafCIDs) + len(headLeafCID)) - response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) - response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(0) - - return response, nil -} - -func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { - var blockHeight uint64 - return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String()) -} - -func (b *Backend) getBlockHeightAndStateLeafKeyForStorageRoot(root common.Hash) (uint64, string, error) { - var res struct { - BlockNumber uint64 `db:"block_number"` - StateLeafKey string `db:"state_leaf_key"` + // Update metadata + if (sliceNodeMetrics.pathLen - len(headPath)) > metaData.maxDepth { + metaData.maxDepth = sliceNodeMetrics.pathLen + } + if sliceNodeMetrics.isLeaf { + metaData.leafCount++ + } + metaData.sliceNodesFetchTime += sliceNodeMetrics.nodeFetchTime + metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime } - return res.BlockNumber, res.StateLeafKey, b.DB.Get(&res, RetrieveBlockNumberAndStateLeafKeyForStorageRoot, root.String()) -} - -func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { - nodes := make(map[string]string) - fetchStart := makeTimestamp() - - // Get CIDs for all nodes at the provided paths - leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStatesByPathsAndBlockNumber(tx, paths, blockHeight) - if err != nil { - return nil, nil, 0, "", err - } - - // Populate the nodes map for leaf nodes - err = populateNodesMap(nodes, leafCIDs, leafIPLDs) - if err != nil { - return nil, nil, 0, "", err - } - - // Populate the nodes map for intermediate nodes - err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs) - if err != nil { - return nil, nil, 0, "", err - } - - return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil -} - -func (b *Backend) getAccountByStateCID(tx *sqlx.Tx, cid string, blockHeight uint64) (string, string, []byte, error) { - var err error - var res struct { - StateLeafKey string `db:"state_leaf_key"` - StorageRoot string `db:"storage_root"` - CodeHash []byte `db:"code_hash"` - } - - if err = tx.Get(&res, RetrieveAccountByStateCID, cid, blockHeight); err != nil { - return "", "", nil, err - } - - mhKey, err := ethServerShared.MultihashKeyFromKeccak256(common.BytesToHash(res.CodeHash)) - if err != nil { - return "", "", nil, err - } - - code := make([]byte, 0) - err = tx.Get(&code, RetrieveCodeByMhKey, mhKey) - if err != nil { - return "", "", nil, err - } - - return res.StateLeafKey, res.StorageRoot, code, nil -} - -func (b *Backend) getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { - nodes := make(map[string]string) - fetchStart := makeTimestamp() - - // Get CIDs for all nodes at the provided paths - leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, paths, blockHeight) - if err != nil { - return nil, nil, 0, "", err - } - - // Populate the nodes map for leaf nodes - err = populateNodesMap(nodes, leafCIDs, leafIPLDs) - if err != nil { - return nil, nil, 0, "", err - } - - // Populate the nodes map for intermediate nodes - err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs) - if err != nil { - return nil, nil, 0, "", err - } - - return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil + return nil } // Engine satisfied the ChainContext interface diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index 7adf9792..bee26972 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -17,20 +17,32 @@ package eth import ( + "bytes" "context" "encoding/json" "fmt" "math/big" + nodeiter "github.com/cerc-io/go-eth-state-node-iterator" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" ) +var nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") +var emptyCodeHash = crypto.Keccak256([]byte{}) + // RPCMarshalHeader converts the given header to the RPC output. // This function is eth/internal so we have to make our own version here... func RPCMarshalHeader(head *types.Header) map[string]interface{} { @@ -300,3 +312,108 @@ func toBlockNumArg(number *big.Int) string { } return hexutil.EncodeBig(number) } + +func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64) { + startTime := makeTimestamp() + var it trie.NodeIterator + + if len(startKey)%2 != 0 { + // Zero-pad for odd-length keys, required by HexToKeyBytes() + startKey = append(startKey, 0) + it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey)) + } else { + it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey)) + // Step to the required node (not required if original startKey was odd-length) + it.Next(true) + } + + return it, makeTimestamp() - startTime +} + +type SliceNodeMetrics struct { + pathLen int + isLeaf bool + nodeFetchTime int64 + leafFetchTime int64 +} + +func fillSliceNodeData( + ethDB ethdb.KeyValueReader, + trieDB *trie.Database, + nodesMap map[string]string, + leavesMap map[string]GetSliceResponseAccount, + it trie.NodeIterator, + storage bool, +) (SliceNodeMetrics, error) { + // Skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + return SliceNodeMetrics{}, nil + } + + nodeStartTime := makeTimestamp() + + node, nodeElements, err := sdtrie.ResolveNode(it, trieDB) + if err != nil { + return SliceNodeMetrics{}, err + } + + sliceNodeMetrics := SliceNodeMetrics{ + pathLen: len(it.Path()), + isLeaf: node.NodeType == sdtypes.Leaf, + } + + // Populate the nodes map + nodeVal := node.NodeValue + nodeValHash := crypto.Keccak256Hash(nodeVal) + nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(nodeVal) + + sliceNodeMetrics.nodeFetchTime = makeTimestamp() - nodeStartTime + + // Extract account data if it's a Leaf node + if node.NodeType == sdtypes.Leaf && !storage { + leafStartTime := makeTimestamp() + + stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements) + if err != nil { + return SliceNodeMetrics{}, fmt.Errorf("GetSlice account lookup error: %s", err.Error()) + } + + if len(code) > 0 { + // Populate the leaves map + leavesMap[stateLeafKey] = GetSliceResponseAccount{ + StorageRoot: storageRoot, + EVMCode: common.Bytes2Hex(code), + } + } + + sliceNodeMetrics.leafFetchTime = makeTimestamp() - leafStartTime + } + + return sliceNodeMetrics, nil +} + +func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) { + var account types.StateAccount + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + + if bytes.Equal(account.CodeHash, emptyCodeHash) { + return "", "", nil, nil + } + + // Extract state leaf key + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + stateLeafKeyString := common.BytesToHash(leafKey).String() + + storageRootString := account.Root.String() + + // Extract codeHash and get code + codeHash := common.BytesToHash(account.CodeHash) + codeBytes := rawdb.ReadCode(ethDB, codeHash) + + return stateLeafKeyString, storageRootString, codeBytes, nil +} diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index 498b6ae6..cce23579 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -17,16 +17,9 @@ package eth import ( - "bytes" - "fmt" - "math" "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" sdtypes "github.com/ethereum/go-ethereum/statediff/types" - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" ) func ResolveToNodeType(nodeType int) sdtypes.NodeType { @@ -44,86 +37,7 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType { } } -var pathSteps = []byte{'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f'} - -// Return head, stem, and slice byte paths for the given head path and depth -func getPaths(path string, depth int) ([]byte, [][]byte, [][]byte, error) { - // Convert the head hex path to a decoded byte path - headPath := common.FromHex(path) - - pathLen := len(headPath) - if pathLen > 64 { // max path len is 64 - return nil, nil, nil, fmt.Errorf("path length cannot exceed 64; got %d", pathLen) - } - - maxDepth := 64 - pathLen - if depth > maxDepth { - return nil, nil, nil, fmt.Errorf("max depth for path %s is %d; got %d", path, maxDepth, depth) - } - - // Collect all of the stem paths - stemPaths := make([][]byte, 0, pathLen) - for i := 0; i < pathLen; i++ { - stemPaths = append(stemPaths, headPath[:i]) - } - - // Generate all of the slice paths - slicePaths := make([][]byte, 0, int(math.Pow(16, float64(depth)))) - makeSlicePaths(headPath, depth, &slicePaths) - - return headPath, stemPaths, slicePaths, nil -} - -// An iterative function to generate the set of slice paths -func makeSlicePaths(path []byte, depth int, slicePaths *[][]byte) { - // return if depth has reached 0 - if depth <= 0 { - return - } - depth-- - - // slice to hold the next 16 paths - nextPaths := make([][]byte, 0, 16) - for _, step := range pathSteps { - // create next paths by adding steps to current path - nextPath := make([]byte, len(path)) - copy(nextPath, path) - nextPath = append(nextPath, step) - - nextPaths = append(nextPaths, nextPath) - - // also add the next path to the collection of all slice paths - dst := make([]byte, len(nextPath)) - copy(dst, nextPath) - *slicePaths = append(*slicePaths, dst) - } - - // iterate over the next paths to repeat the process if not - for _, nextPath := range nextPaths { - makeSlicePaths(nextPath, depth, slicePaths) - } -} - // Timestamp in milliseconds func makeTimestamp() int64 { return time.Now().UnixNano() / int64(time.Millisecond) } - -func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) error { - for i, cid := range cids { - decodedMh, err := multihash.Decode(cid.Hash()) - if err != nil { - return err - } - - data := iplds[i] - hash := crypto.Keccak256Hash(data) - if !bytes.Equal(hash.Bytes(), decodedMh.Digest) { - return fmt.Errorf("multihash digest should equal keccak of raw data") - } - - nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data) - } - - return nil -} diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index adb9e46f..3cfb6530 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -17,14 +17,12 @@ package eth import ( - "database/sql" "fmt" "strconv" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/statediff/trie_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" - "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" "github.com/ethereum/go-ethereum/common" @@ -236,34 +234,6 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveStateByPathAndBlockNumberPgStr = `SELECT cid, data, node_type - FROM eth.state_cids - INNER JOIN public.blocks ON ( - state_cids.mh_key = blocks.key - AND state_cids.block_number = blocks.block_number - ) - WHERE state_path = $1 - AND state_cids.block_number <= $2 - AND state_cids.header_id = (SELECT canonical_header_hash(state_cids.block_number)) - ORDER BY state_cids.block_number DESC - LIMIT 1` - RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type - FROM eth.storage_cids - INNER JOIN eth.state_cids ON ( - storage_cids.state_path = state_cids.state_path - AND storage_cids.header_id = state_cids.header_id - AND storage_cids.block_number = state_cids.block_number - ) - INNER JOIN public.blocks ON ( - storage_cids.mh_key = blocks.key - AND storage_cids.block_number = blocks.block_number - ) - WHERE state_leaf_key = $1 - AND storage_path = $2 - AND storage_cids.block_number <= $3 - AND storage_cids.header_id = (SELECT canonical_header_hash(storage_cids.block_number)) - ORDER BY storage_cids.block_number DESC - LIMIT 1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON ( @@ -749,103 +719,3 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad } return storageResult.CID, i[1].([]byte), nil } - -// RetrieveStatesByPathsAndBlockNumber returns the cid and rlp bytes for the state nodes corresponding to the provided state paths and block number -func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) { - deepestPath := 0 - - leafNodeCIDs := make([]cid.Cid, 0) - intermediateNodeCIDs := make([]cid.Cid, 0) - - leafNodeIPLDs := make([][]byte, 0) - intermediateNodeIPLDs := make([][]byte, 0) - - // TODO: fetch all nodes in a single query - for _, path := range paths { - // Create a result object, select: cid, data, node_type - res := new(nodeInfo) - if err := tx.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { - // Skip if node not found for a path - if err == sql.ErrNoRows { - continue - } - - return nil, nil, nil, nil, 0, err - } - - // Skip if node is of removed type - if res.NodeType == sdtypes.Removed.Int() { - continue - } - - pathLen := len(path) - if pathLen > deepestPath { - deepestPath = pathLen - } - - cid, err := cid.Decode(res.CID) - if err != nil { - return nil, nil, nil, nil, 0, err - } - - if res.NodeType == sdtypes.Leaf.Int() { - leafNodeCIDs = append(leafNodeCIDs, cid) - leafNodeIPLDs = append(leafNodeIPLDs, res.Data) - } else { - intermediateNodeCIDs = append(intermediateNodeCIDs, cid) - intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) - } - } - - return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil -} - -// RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber returns the cid and rlp bytes for the storage nodes corresponding to the provided state leaf key, storage paths and block number -func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) { - deepestPath := 0 - - leafNodeCIDs := make([]cid.Cid, 0) - intermediateNodeCIDs := make([]cid.Cid, 0) - - leafNodeIPLDs := make([][]byte, 0) - intermediateNodeIPLDs := make([][]byte, 0) - - // TODO: fetch all nodes in a single query - for _, path := range paths { - // Create a result object, select: cid, data, node_type - res := new(nodeInfo) - if err := tx.Get(res, RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr, stateLeafKey, path, number); err != nil { - // Skip if node not found for a path - if err == sql.ErrNoRows { - continue - } - - return nil, nil, nil, nil, 0, err - } - - // Skip if node is of removed type - if res.NodeType == sdtypes.Removed.Int() { - continue - } - - pathLen := len(path) - if pathLen > deepestPath { - deepestPath = pathLen - } - - cid, err := cid.Decode(res.CID) - if err != nil { - return nil, nil, nil, nil, 0, err - } - - if res.NodeType == sdtypes.Leaf.Int() { - leafNodeCIDs = append(leafNodeCIDs, cid) - leafNodeIPLDs = append(leafNodeIPLDs, res.Data) - } else { - intermediateNodeCIDs = append(intermediateNodeCIDs, cid) - intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) - } - } - - return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil -} diff --git a/pkg/eth/types.go b/pkg/eth/types.go index d9631786..a1f1e9c8 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "strconv" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -288,6 +289,19 @@ func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) { } } +func (sr *GetSliceResponse) populateMetaData(metaData metaDataFields) { + sr.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head)) + sr.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(metaData.maxDepth) + sr.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head) + len(sr.TrieNodes.Slice)) + sr.MetaData.NodeStats["03-leaves"] = strconv.Itoa(metaData.leafCount) + sr.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(sr.Leaves)) + + sr.MetaData.TimeStats["00-trie-loading"] = strconv.FormatInt(metaData.trieLoadingTime, 10) + sr.MetaData.TimeStats["01-fetch-stem-keys"] = strconv.FormatInt(metaData.stemNodesFetchTime, 10) + sr.MetaData.TimeStats["02-fetch-slice-keys"] = strconv.FormatInt(metaData.sliceNodesFetchTime, 10) + sr.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.FormatInt(metaData.leavesFetchTime, 10) +} + type GetSliceResponseMetadata struct { TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one) NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts @@ -303,3 +317,12 @@ type GetSliceResponseAccount struct { StorageRoot string `json:"storageRoot"` EVMCode string `json:"evmCode"` } + +type metaDataFields struct { + maxDepth int + leafCount int + trieLoadingTime int64 + stemNodesFetchTime int64 + sliceNodesFetchTime int64 + leavesFetchTime int64 +} -- 2.45.2 From 8e4d67fcd1f32b23af00cf5b52753e1a52f8c603 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 15 Dec 2022 16:02:15 +0530 Subject: [PATCH 16/21] Skip undesired nodes from stem and head iterators --- pkg/eth/backend.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 6f7e9385..a3eb6ea8 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -17,6 +17,7 @@ package eth import ( + "bytes" "context" "database/sql" "errors" @@ -932,6 +933,11 @@ func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSlice it, timeTaken := getIteratorAtPath(t, startPath) metaData.trieLoadingTime += timeTaken + // Skip if iterator not at required path (might happen if node not present at given path) + if !bytes.Equal(it.Path(), startPath) { + continue + } + sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Stem, response.Leaves, it, storage) if err != nil { return err @@ -956,6 +962,11 @@ func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSlice it, timeTaken := getIteratorAtPath(t, headPath) metaData.trieLoadingTime += timeTaken + // Skip if iterator not at required path (might happen if node not present at given path) + if !bytes.Equal(it.Path(), headPath) { + return nil + } + sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Head, response.Leaves, it, storage) if err != nil { return err -- 2.45.2 From d5d03a0633e61a74e8e32762a49c71804cbafeb1 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 15 Dec 2022 16:03:17 +0530 Subject: [PATCH 17/21] Update storage slice tests --- pkg/eth/eth_state_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 32d78d24..cd793c83 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -714,6 +714,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "1", "02-total-trie-nodes": "3", "03-leaves": "2", + "04-smart-contracts": "0", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ @@ -745,6 +746,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "0", "02-total-trie-nodes": "1", "03-leaves": "0", + "04-smart-contracts": "0", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ @@ -773,6 +775,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "0", "02-total-trie-nodes": "1", "03-leaves": "1", + "04-smart-contracts": "0", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ @@ -801,6 +804,7 @@ var _ = Describe("eth state reading tests", func() { "01-max-depth": "0", "02-total-trie-nodes": "2", "03-leaves": "1", + "04-smart-contracts": "0", }, }, TrieNodes: eth.GetSliceResponseTrieNodes{ -- 2.45.2 From 865b9656691cde57e90a1897b7765bbdb4db2cc3 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Thu, 15 Dec 2022 16:28:19 +0530 Subject: [PATCH 18/21] Fix meta data updates --- pkg/eth/backend.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index a3eb6ea8..c440430a 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -892,12 +892,15 @@ func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage boo response := new(GetSliceResponse) response.init(path, depth, root) - t, _ := b.StateDatabase.OpenTrie(root) - headPath := common.FromHex(path) - // Metadata fields metaData := metaDataFields{} + startTime := makeTimestamp() + t, _ := b.StateDatabase.OpenTrie(root) + metaData.trieLoadingTime = makeTimestamp() - startTime + + headPath := common.FromHex(path) + // Get Stem nodes err := b.getSliceStem(headPath, t, response, &metaData, storage) if err != nil { @@ -944,8 +947,9 @@ func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSlice } // Update metadata - if (sliceNodeMetrics.pathLen - len(headPath)) > metaData.maxDepth { - metaData.maxDepth = sliceNodeMetrics.pathLen + depthReached := sliceNodeMetrics.pathLen - len(headPath) + if depthReached > metaData.maxDepth { + metaData.maxDepth = depthReached } if sliceNodeMetrics.isLeaf { metaData.leafCount++ @@ -973,8 +977,9 @@ func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSlice } // Update metadata - if (sliceNodeMetrics.pathLen - len(headPath)) > metaData.maxDepth { - metaData.maxDepth = sliceNodeMetrics.pathLen + depthReached := sliceNodeMetrics.pathLen - len(headPath) + if depthReached > metaData.maxDepth { + metaData.maxDepth = depthReached } if sliceNodeMetrics.isLeaf { metaData.leafCount++ @@ -1013,8 +1018,9 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice } // Update metadata - if (sliceNodeMetrics.pathLen - len(headPath)) > metaData.maxDepth { - metaData.maxDepth = sliceNodeMetrics.pathLen + depthReached := sliceNodeMetrics.pathLen - len(headPath) + if depthReached > metaData.maxDepth { + metaData.maxDepth = depthReached } if sliceNodeMetrics.isLeaf { metaData.leafCount++ -- 2.45.2 From b5bb47e77d9a92b0648792ed63e16b933198e11f Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Fri, 16 Dec 2022 17:21:40 +0530 Subject: [PATCH 19/21] Use state trie to get stem nodes directly using paths --- go.mod | 2 +- pkg/eth/backend.go | 99 +++++++++++++++++++++++++++++----------- pkg/eth/backend_utils.go | 67 ++++++++++++--------------- 3 files changed, 102 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 28e6e31a..ca0fd5f6 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha + github.com/cerc-io/go-eth-state-node-iterator v1.1.9 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha github.com/ethereum/go-ethereum v1.10.26 github.com/graph-gophers/graphql-go v1.3.0 @@ -41,7 +42,6 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cerc-io/go-eth-state-node-iterator v1.1.9 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cheekybits/genny v1.0.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index c440430a..c4afe783 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -44,6 +44,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" @@ -899,6 +901,7 @@ func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage boo t, _ := b.StateDatabase.OpenTrie(root) metaData.trieLoadingTime = makeTimestamp() - startTime + // Convert the head hex path to a decoded byte path headPath := common.FromHex(path) // Get Stem nodes @@ -927,65 +930,90 @@ func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage boo } func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error { + leavesFetchTime := int64(0) + totalStemStartTime := makeTimestamp() + for i := 0; i < len(headPath); i++ { // Create path for each node along the stem - startPath := make([]byte, len(headPath[:i])) - copy(startPath, headPath[:i]) + nodePath := make([]byte, len(headPath[:i])) + copy(nodePath, headPath[:i]) - // Create an iterator initialized at startPath - it, timeTaken := getIteratorAtPath(t, startPath) - metaData.trieLoadingTime += timeTaken + // TODO: verify doesn't return for non-existent node + rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(nodePath)) + if err != nil { + return err + } - // Skip if iterator not at required path (might happen if node not present at given path) - if !bytes.Equal(it.Path(), startPath) { + // Skip if node not found + if rawNode == nil { continue } - sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Stem, response.Leaves, it, storage) + node, nodeElements, err := ResolveNode(nodePath, rawNode, b.StateDatabase.TrieDB()) + if err != nil { + return err + } + + leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Stem, response.Leaves, node, nodeElements, storage) if err != nil { return err } // Update metadata - depthReached := sliceNodeMetrics.pathLen - len(headPath) + depthReached := len(node.Path) - len(headPath) if depthReached > metaData.maxDepth { metaData.maxDepth = depthReached } - if sliceNodeMetrics.isLeaf { + if node.NodeType == sdtypes.Leaf { metaData.leafCount++ } - metaData.stemNodesFetchTime += sliceNodeMetrics.nodeFetchTime - metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime + leavesFetchTime += leafFetchTime } + // Update metadata time metrics + totalStemTime := makeTimestamp() - totalStemStartTime + metaData.sliceNodesFetchTime = totalStemTime - leavesFetchTime + metaData.leavesFetchTime += leavesFetchTime + return nil } func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error { - // Create an iterator initialized at headPath - it, timeTaken := getIteratorAtPath(t, headPath) - metaData.trieLoadingTime += timeTaken + totalHeadStartTime := makeTimestamp() - // Skip if iterator not at required path (might happen if node not present at given path) - if !bytes.Equal(it.Path(), headPath) { + rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(headPath)) + if err != nil { + return err + } + + // Skip if node not found + if rawNode == nil { return nil } - sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Head, response.Leaves, it, storage) + node, nodeElements, err := ResolveNode(headPath, rawNode, b.StateDatabase.TrieDB()) + if err != nil { + return err + } + + leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Head, response.Leaves, node, nodeElements, storage) if err != nil { return err } // Update metadata - depthReached := sliceNodeMetrics.pathLen - len(headPath) + depthReached := len(node.Path) - len(headPath) if depthReached > metaData.maxDepth { metaData.maxDepth = depthReached } - if sliceNodeMetrics.isLeaf { + if node.NodeType == sdtypes.Leaf { metaData.leafCount++ } - metaData.stemNodesFetchTime += sliceNodeMetrics.nodeFetchTime - metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime + + // Update metadata time metrics + totalHeadTime := makeTimestamp() - totalHeadStartTime + metaData.stemNodesFetchTime = totalHeadTime - leafFetchTime + metaData.leavesFetchTime += leafFetchTime return nil } @@ -994,6 +1022,9 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice it, timeTaken := getIteratorAtPath(t, headPath) metaData.trieLoadingTime += timeTaken + leavesFetchTime := int64(0) + totalSliceStartTime := makeTimestamp() + headPathLen := len(headPath) maxPathLen := headPathLen + depth descend := true @@ -1012,23 +1043,37 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice descend = true } - sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Slice, response.Leaves, it, storage) + // Skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + return nil + } + + node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB()) + if err != nil { + return err + } + + leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Slice, response.Leaves, node, nodeElements, storage) if err != nil { return err } // Update metadata - depthReached := sliceNodeMetrics.pathLen - len(headPath) + depthReached := len(node.Path) - len(headPath) if depthReached > metaData.maxDepth { metaData.maxDepth = depthReached } - if sliceNodeMetrics.isLeaf { + if node.NodeType == sdtypes.Leaf { metaData.leafCount++ } - metaData.sliceNodesFetchTime += sliceNodeMetrics.nodeFetchTime - metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime + leavesFetchTime += leafFetchTime } + // Update metadata time metrics + totalSliceTime := makeTimestamp() - totalSliceStartTime + metaData.sliceNodesFetchTime = totalSliceTime - leavesFetchTime + metaData.leavesFetchTime += leavesFetchTime + return nil } diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index bee26972..cca1c52f 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -330,52 +330,25 @@ func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64) return it, makeTimestamp() - startTime } -type SliceNodeMetrics struct { - pathLen int - isLeaf bool - nodeFetchTime int64 - leafFetchTime int64 -} - func fillSliceNodeData( ethDB ethdb.KeyValueReader, - trieDB *trie.Database, nodesMap map[string]string, leavesMap map[string]GetSliceResponseAccount, - it trie.NodeIterator, + node sdtypes.StateNode, + nodeElements []interface{}, storage bool, -) (SliceNodeMetrics, error) { - // Skip value nodes - if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { - return SliceNodeMetrics{}, nil - } - - nodeStartTime := makeTimestamp() - - node, nodeElements, err := sdtrie.ResolveNode(it, trieDB) - if err != nil { - return SliceNodeMetrics{}, err - } - - sliceNodeMetrics := SliceNodeMetrics{ - pathLen: len(it.Path()), - isLeaf: node.NodeType == sdtypes.Leaf, - } - +) (int64, error) { // Populate the nodes map - nodeVal := node.NodeValue - nodeValHash := crypto.Keccak256Hash(nodeVal) - nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(nodeVal) - - sliceNodeMetrics.nodeFetchTime = makeTimestamp() - nodeStartTime + // nodeVal := node.NodeValue + nodeValHash := crypto.Keccak256Hash(node.NodeValue) + nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue) // Extract account data if it's a Leaf node + leafStartTime := makeTimestamp() if node.NodeType == sdtypes.Leaf && !storage { - leafStartTime := makeTimestamp() - stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements) if err != nil { - return SliceNodeMetrics{}, fmt.Errorf("GetSlice account lookup error: %s", err.Error()) + return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error()) } if len(code) > 0 { @@ -385,11 +358,9 @@ func fillSliceNodeData( EVMCode: common.Bytes2Hex(code), } } - - sliceNodeMetrics.leafFetchTime = makeTimestamp() - leafStartTime } - return sliceNodeMetrics, nil + return makeTimestamp() - leafStartTime, nil } func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) { @@ -417,3 +388,23 @@ func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNo return stateLeafKeyString, storageRootString, codeBytes, nil } + +func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) { + nodePath := make([]byte, len(path)) + copy(nodePath, path) + + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return sdtypes.StateNode{}, nil, err + } + + ty, err := sdtrie.CheckKeyType(nodeElements) + if err != nil { + return sdtypes.StateNode{}, nil, err + } + return sdtypes.StateNode{ + NodeType: ty, + Path: nodePath, + NodeValue: node, + }, nodeElements, nil +} -- 2.45.2 From d40bbc46ebe0a0a65cd3c5368bd4a542e318f5f3 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Fri, 16 Dec 2022 19:50:01 +0530 Subject: [PATCH 20/21] Bugfix - Continue processing other trie nodes on encountering a leaf --- pkg/eth/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index c4afe783..d2773204 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -1045,7 +1045,7 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice // Skip value nodes if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { - return nil + continue } node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB()) -- 2.45.2 From 7a04ee73ff92b703d484043bd8e82655f21a1198 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Mon, 19 Dec 2022 11:48:57 +0530 Subject: [PATCH 21/21] Remove unnecessary TODO --- pkg/eth/backend.go | 1 - pkg/eth/backend_utils.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index d2773204..1123a875 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -938,7 +938,6 @@ func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSlice nodePath := make([]byte, len(headPath[:i])) copy(nodePath, headPath[:i]) - // TODO: verify doesn't return for non-existent node rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(nodePath)) if err != nil { return err diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index cca1c52f..68655604 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -339,7 +339,6 @@ func fillSliceNodeData( storage bool, ) (int64, error) { // Populate the nodes map - // nodeVal := node.NodeValue nodeValHash := crypto.Keccak256Hash(node.NodeValue) nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue) -- 2.45.2