From a2a391c45953a3253a0ae7f084f055297ae5405f Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Tue, 13 Dec 2022 10:13:08 +0530 Subject: [PATCH] Implement getSlice API for state nodes --- pkg/eth/api.go | 8 +++ pkg/eth/api_test.go | 4 ++ pkg/eth/backend.go | 125 +++++++++++++++++++++++++++++++++++++- pkg/eth/helpers.go | 82 +++++++++++++++++++++++++ pkg/eth/ipld_retriever.go | 51 ++++++++++++++++ pkg/eth/types.go | 39 ++++++++++++ 6 files changed, 308 insertions(+), 1 deletion(-) diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 5fc64393..b7e4da8e 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -837,6 +837,14 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre }, state.Error() } +// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth +func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) { + if storage { + return pea.B.GetStorageSlice(path, depth, root) + } + return pea.B.GetStateSlice(path, depth, root) +} + // revertError is an API error that encompassas an EVM revertal with JSON error // code and a binary data blob. type revertError struct { diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index feb500bf..91d80b1d 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -1145,4 +1145,8 @@ var _ = Describe("API", func() { Expect(code).To(BeEmpty()) }) }) + + Describe("eth_getSlice", func() { + // TODO Implement + }) }) diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 24d86e54..8d1ed56c 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -43,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" @@ -95,7 +96,12 @@ const ( AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number)) ORDER BY header_cids.block_number DESC LIMIT 1` - RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` + RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1` + RetrieveBlockNumberForStateRoot = `SELECT block_number + FROM eth.header_cids + WHERE state_root = $1 + ORDER BY block_number DESC + LIMIT 1` ) const ( @@ -888,6 +894,123 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, return storageRlp, err } +func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { + response := new(GetSliceResponse) + response.init(path, depth, root) + + // Start a timer + trieLoadingStart := makeTimestamp() + + // Get the block height for the input state root + blockHeight, err := b.getBlockHeightForStateRoot(root) + if err != nil { + return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error()) + } + + headPath, stemPaths, slicePaths, err := getPaths(path, depth) + if err != nil { + return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error()) + } + response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart)) + + // Begin tx + tx, err := b.DB.Beginx() + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + + // Fetch stem nodes + // some of the "stem" nodes can be leaf nodes (but not value nodes) + stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, stemPaths) + if err != nil { + return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error()) + } + response.TrieNodes.Stem = stemNodes + response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent + + // Fetch slice nodes + sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, slicePaths) + if err != nil { + return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error()) + } + response.TrieNodes.Slice = sliceNodes + response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent + + // Fetch head node + headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, [][]byte{headPath}) + if err != nil { + return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error()) + } + response.TrieNodes.Head = headNode + + // Fetch leaf contract data and fill in remaining metadata + leafFetchStart := makeTimestamp() + leafNodes := make([]cid.Cid, 0, len(stemLeafCIDs)+len(sliceLeafCIDs)+len(headLeafCID)) + leafNodes = append(leafNodes, stemLeafCIDs...) + leafNodes = append(leafNodes, sliceLeafCIDs...) + leafNodes = append(leafNodes, headLeafCID...) + // TODO: fill in contract data `response.Leaves` + + maxDepth := deepestPath - len(headPath) + if maxDepth < 0 { + maxDepth = 0 + } + response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) + + response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) + response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) + response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) + response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) + + return response, nil +} + +func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { + // TODO Implement + return nil, nil +} + +func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { + var blockHeight uint64 + return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String()) +} + +func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight uint64, paths [][]byte) (map[string]string, []cid.Cid, int, string, error) { + nodes := make(map[string]string) + fetchStart := makeTimestamp() + + // Get CIDs for all nodes at the provided paths + leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStatesByPathsAndBlockNumber(tx, paths, blockHeight) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for leaf nodes + err = populateNodesMap(nodes, leafCIDs, leafIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for intermediate nodes + err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index f9a995fc..6caded31 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -17,7 +17,16 @@ package eth import ( + "bytes" + "fmt" + "math" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" ) func ResolveToNodeType(nodeType int) sdtypes.NodeType { @@ -34,3 +43,76 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType { return sdtypes.Unknown } } + +var pathSteps = []byte{'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f'} + +// Return head, stem, and slice byte paths for the given head path and depth +func getPaths(path string, depth int) ([]byte, [][]byte, [][]byte, error) { + // Convert the head hex path to a decoded byte path + headPath := common.FromHex(path) + + pathLen := len(headPath) + if pathLen > 64 { // max path len is 64 + return nil, nil, nil, fmt.Errorf("path length cannot exceed 64; got %d", pathLen) + } + + maxDepth := 64 - pathLen + if depth > maxDepth { + return nil, nil, nil, fmt.Errorf("max depth for path %s is %d; got %d", path, maxDepth, depth) + } + + // Collect all of the stem paths + stemPaths := make([][]byte, 0, pathLen) + for i := 0; i < pathLen; i++ { + stemPaths = append(stemPaths, headPath[:i]) + } + + // Generate all of the slice paths + slicePaths := make([][]byte, 0, int(math.Pow(16, float64(depth)))) + makeSlicePaths(headPath, depth, &slicePaths) + + return headPath, stemPaths, slicePaths, nil +} + +// iterative function to generate the set of slice paths +func makeSlicePaths(path []byte, depth int, slicePaths *[][]byte) { + depth-- // decrement the depth + nextPaths := make([][]byte, 16) // slice to hold the next 16 paths + for i, step := range pathSteps { // iterate through steps + nextPath := append(path, step) // create next paths by adding steps to current path + nextPaths[i] = nextPath + newSlicePaths := append(*slicePaths, nextPath) // add next paths to the collection of all slice paths + slicePaths = &newSlicePaths + } + + if depth == 0 { // if depth has reach 0, return + return + } + for _, nextPath := range nextPaths { // if not, then we iterate over the next paths + makeSlicePaths(nextPath, depth, slicePaths) // and repeat the process for each one + } +} + +// use to return timestamp in milliseconds +func makeTimestamp() int64 { + return time.Now().UnixNano() / int64(time.Millisecond) +} + +func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) error { + for i, cid := range cids { + decodedMh, err := multihash.Decode(cid.Hash()) + if err != nil { + return err + } + + data := iplds[i] + hash := crypto.Keccak256Hash(data) + if !bytes.Equal(hash.Bytes(), decodedMh.Digest) { + panic("multihash digest should equal keccak of raw data") + } + + nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data) + } + + return nil +} diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 2cbdcaea..fe970a82 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -23,6 +23,7 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/statediff/trie_helpers" sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ipfs/go-cid" "github.com/jmoiron/sqlx" "github.com/ethereum/go-ethereum/common" @@ -238,6 +239,17 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` + RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.mh_key, state_cids.node_type + FROM eth.state_cids + INNER JOIN public.blocks ON ( + state_cids.mh_key = blocks.key + AND state_cids.block_number = blocks.block_number + ) + WHERE state_path = $1 + AND state_cids.block_number <= $2 + AND node_type != 3 + ORDER BY state_cids.block_number DESC + LIMIT 1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON ( @@ -723,3 +735,42 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad } return storageResult.CID, i[1].([]byte), nil } + +// RetrieveStatesByPathsAndBlockNumber returns the cid and rlp bytes for the state nodes corresponding to the provided state paths and block number +func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) { + deepestPath := 0 + + leafNodeCIDs := make([]cid.Cid, 0) + intermediateNodeCIDs := make([]cid.Cid, 0) + + leafNodeIPLDs := make([][]byte, 0) + intermediateNodeIPLDs := make([][]byte, 0) + + for _, path := range paths { + // Create a result object, select: cid, blockNumber and nodeType + res := new(nodeInfo) + if err := r.db.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + return nil, nil, nil, nil, 0, err + } + + pathLen := len(path) + if pathLen > deepestPath { + deepestPath = pathLen + } + + cid, err := cid.Decode(res.CID) + if err != nil { + return nil, nil, nil, nil, 0, err + } + + if res.NodeType == sdtypes.Leaf.Int() { + leafNodeCIDs = append(leafNodeCIDs, cid) + leafNodeIPLDs = append(leafNodeIPLDs, res.Data) + } else { + intermediateNodeCIDs = append(intermediateNodeCIDs, cid) + intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) + } + } + + return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil +} diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 5118eda2..ada668b5 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -18,6 +18,7 @@ package eth import ( "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -264,3 +265,41 @@ type LogResult struct { TxnIndex int64 `db:"txn_index"` TxHash string `db:"tx_hash"` } + +// GetSliceResponse holds response for the eth_getSlice method +type GetSliceResponse struct { + SliceID string `json:"sliceId"` + MetaData GetSliceResponseMetadata `json:"metadata"` + TrieNodes GetSliceResponseTrieNodes `json:"trieNodes"` + Leaves map[string]GetSliceResponseAccount `json:"leaves"` // we won't be using addresses, but keccak256(address) // TODO: address comment +} + +func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) { + sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String()) + sr.MetaData = GetSliceResponseMetadata{ + NodeStats: make(map[string]string, 0), + TimeStats: make(map[string]string, 0), + } + sr.Leaves = make(map[string]GetSliceResponseAccount) + sr.TrieNodes = GetSliceResponseTrieNodes{ + Stem: make(map[string]string), + Head: make(map[string]string), + Slice: make(map[string]string), + } +} + +type GetSliceResponseMetadata struct { + TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one) + NodeStats map[string]string `json:"trieNodes"` // total, leaves, smart contracts +} + +type GetSliceResponseTrieNodes struct { + Stem map[string]string `json:"stem"` + Head map[string]string `json:"head"` + Slice map[string]string `json:"sliceNodes"` +} + +type GetSliceResponseAccount struct { + StorageRoot string `json:"storageRoot"` + EVMCode string `json:"evmCode"` +}