Implement getSlice API for state nodes

pkg/eth/api.go

@@ -837,6 +837,14 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre
 	}, state.Error()
 }
 
+// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth
+func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) {
+	if storage {
+		return pea.B.GetStorageSlice(path, depth, root)
+	}
+	return pea.B.GetStateSlice(path, depth, root)
+}
+
 // revertError is an API error that encompassas an EVM revertal with JSON error
 // code and a binary data blob.
 type revertError struct {

pkg/eth/api_test.go

@@ -1145,4 +1145,8 @@ var _ = Describe("API", func() {
 			Expect(code).To(BeEmpty())
 		})
 	})
+
+	Describe("eth_getSlice", func() {
+		// TODO Implement
+	})
 })

pkg/eth/backend.go

@@ -43,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ipfs/go-cid"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
 
@@ -96,6 +97,11 @@ const (
 											ORDER BY header_cids.block_number DESC
 											LIMIT 1`
 	RetrieveCodeByMhKey             = `SELECT data FROM public.blocks WHERE key = $1`
+	RetrieveBlockNumberForStateRoot = `SELECT block_number
+			FROM eth.header_cids
+			WHERE state_root = $1
+			ORDER BY block_number DESC
+			LIMIT 1`
 )
 
 const (
@@ -888,6 +894,123 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address,
 	return storageRlp, err
 }
 
+func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) {
+	response := new(GetSliceResponse)
+	response.init(path, depth, root)
+
+	// Start a timer
+	trieLoadingStart := makeTimestamp()
+
+	// Get the block height for the input state root
+	blockHeight, err := b.getBlockHeightForStateRoot(root)
+	if err != nil {
+		return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error())
+	}
+
+	headPath, stemPaths, slicePaths, err := getPaths(path, depth)
+	if err != nil {
+		return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error())
+	}
+	response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart))
+
+	// Begin tx
+	tx, err := b.DB.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	// Fetch stem nodes
+	// some of the "stem" nodes can be leaf nodes (but not value nodes)
+	stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, stemPaths)
+	if err != nil {
+		return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error())
+	}
+	response.TrieNodes.Stem = stemNodes
+	response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent
+
+	// Fetch slice nodes
+	sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, slicePaths)
+	if err != nil {
+		return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error())
+	}
+	response.TrieNodes.Slice = sliceNodes
+	response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent
+
+	// Fetch head node
+	headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, [][]byte{headPath})
+	if err != nil {
+		return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error())
+	}
+	response.TrieNodes.Head = headNode
+
+	// Fetch leaf contract data and fill in remaining metadata
+	leafFetchStart := makeTimestamp()
+	leafNodes := make([]cid.Cid, 0, len(stemLeafCIDs)+len(sliceLeafCIDs)+len(headLeafCID))
+	leafNodes = append(leafNodes, stemLeafCIDs...)
+	leafNodes = append(leafNodes, sliceLeafCIDs...)
+	leafNodes = append(leafNodes, headLeafCID...)
+	// TODO: fill in contract data `response.Leaves`
+
+	maxDepth := deepestPath - len(headPath)
+	if maxDepth < 0 {
+		maxDepth = 0
+	}
+	response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth)
+
+	response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1)
+	response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes))
+	response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts
+	response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart))
+	response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1)
+
+	return response, nil
+}
+
+func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) {
+	// TODO Implement
+	return nil, nil
+}
+
+func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) {
+	var blockHeight uint64
+	return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String())
+}
+
+func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight uint64, paths [][]byte) (map[string]string, []cid.Cid, int, string, error) {
+	nodes := make(map[string]string)
+	fetchStart := makeTimestamp()
+
+	// Get CIDs for all nodes at the provided paths
+	leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStatesByPathsAndBlockNumber(tx, paths, blockHeight)
+	if err != nil {
+		return nil, nil, 0, "", err
+	}
+
+	// Populate the nodes map for leaf nodes
+	err = populateNodesMap(nodes, leafCIDs, leafIPLDs)
+	if err != nil {
+		return nil, nil, 0, "", err
+	}
+
+	// Populate the nodes map for intermediate nodes
+	err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs)
+	if err != nil {
+		return nil, nil, 0, "", err
+	}
+
+	return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil
+}
+
 // Engine satisfied the ChainContext interface
 func (b *Backend) Engine() consensus.Engine {
 	// TODO: we need to support more than just ethash based engines

pkg/eth/helpers.go

@@ -17,7 +17,16 @@
 package eth
 
 import (
+	"bytes"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
 )
 
 func ResolveToNodeType(nodeType int) sdtypes.NodeType {
@@ -34,3 +43,76 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType {
 		return sdtypes.Unknown
 	}
 }
+
+var pathSteps = []byte{'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f'}
+
+// Return head, stem, and slice byte paths for the given head path and depth
+func getPaths(path string, depth int) ([]byte, [][]byte, [][]byte, error) {
+	// Convert the head hex path to a decoded byte path
+	headPath := common.FromHex(path)
+
+	pathLen := len(headPath)
+	if pathLen > 64 { // max path len is 64
+		return nil, nil, nil, fmt.Errorf("path length cannot exceed 64; got %d", pathLen)
+	}
+
+	maxDepth := 64 - pathLen
+	if depth > maxDepth {
+		return nil, nil, nil, fmt.Errorf("max depth for path %s is %d; got %d", path, maxDepth, depth)
+	}
+
+	// Collect all of the stem paths
+	stemPaths := make([][]byte, 0, pathLen)
+	for i := 0; i < pathLen; i++ {
+		stemPaths = append(stemPaths, headPath[:i])
+	}
+
+	// Generate all of the slice paths
+	slicePaths := make([][]byte, 0, int(math.Pow(16, float64(depth))))
+	makeSlicePaths(headPath, depth, &slicePaths)
+
+	return headPath, stemPaths, slicePaths, nil
+}
+
+// iterative function to generate the set of slice paths
+func makeSlicePaths(path []byte, depth int, slicePaths *[][]byte) {
+	depth--                          // decrement the depth
+	nextPaths := make([][]byte, 16)  // slice to hold the next 16 paths
+	for i, step := range pathSteps { // iterate through steps
+		nextPath := append(path, step) // create next paths by adding steps to current path
+		nextPaths[i] = nextPath
+		newSlicePaths := append(*slicePaths, nextPath) // add next paths to the collection of all slice paths
+		slicePaths = &newSlicePaths
+	}
+
+	if depth == 0 { // if depth has reach 0, return
+		return
+	}
+	for _, nextPath := range nextPaths { // if not, then we iterate over the next paths
+		makeSlicePaths(nextPath, depth, slicePaths) // and repeat the process for each one
+	}
+}
+
+// use to return timestamp in milliseconds
+func makeTimestamp() int64 {
+	return time.Now().UnixNano() / int64(time.Millisecond)
+}
+
+func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) error {
+	for i, cid := range cids {
+		decodedMh, err := multihash.Decode(cid.Hash())
+		if err != nil {
+			return err
+		}
+
+		data := iplds[i]
+		hash := crypto.Keccak256Hash(data)
+		if !bytes.Equal(hash.Bytes(), decodedMh.Digest) {
+			panic("multihash digest should equal keccak of raw data")
+		}
+
+		nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data)
+	}
+
+	return nil
+}

pkg/eth/ipld_retriever.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
 	"github.com/ethereum/go-ethereum/statediff/trie_helpers"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ipfs/go-cid"
 	"github.com/jmoiron/sqlx"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -238,6 +239,17 @@ const (
 										)
 									WHERE tx_hash = $1
 									AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))`
+	RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.mh_key, state_cids.node_type
+									FROM eth.state_cids
+									INNER JOIN public.blocks ON (
+										state_cids.mh_key = blocks.key
+										AND state_cids.block_number = blocks.block_number
+									)
+									WHERE state_path = $1
+									AND state_cids.block_number <= $2
+									AND node_type != 3
+									ORDER BY state_cids.block_number DESC
+									LIMIT 1`
 	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type
 												FROM eth.state_cids
 													INNER JOIN eth.header_cids ON (
@@ -723,3 +735,42 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad
 	}
 	return storageResult.CID, i[1].([]byte), nil
 }
+
+// RetrieveStatesByPathsAndBlockNumber returns the cid and rlp bytes for the state nodes corresponding to the provided state paths and block number
+func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) {
+	deepestPath := 0
+
+	leafNodeCIDs := make([]cid.Cid, 0)
+	intermediateNodeCIDs := make([]cid.Cid, 0)
+
+	leafNodeIPLDs := make([][]byte, 0)
+	intermediateNodeIPLDs := make([][]byte, 0)
+
+	for _, path := range paths {
+		// Create a result object, select: cid, blockNumber and nodeType
+		res := new(nodeInfo)
+		if err := r.db.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil {
+			return nil, nil, nil, nil, 0, err
+		}
+
+		pathLen := len(path)
+		if pathLen > deepestPath {
+			deepestPath = pathLen
+		}
+
+		cid, err := cid.Decode(res.CID)
+		if err != nil {
+			return nil, nil, nil, nil, 0, err
+		}
+
+		if res.NodeType == sdtypes.Leaf.Int() {
+			leafNodeCIDs = append(leafNodeCIDs, cid)
+			leafNodeIPLDs = append(leafNodeIPLDs, res.Data)
+		} else {
+			intermediateNodeCIDs = append(intermediateNodeCIDs, cid)
+			intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data)
+		}
+	}
+
+	return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil
+}

pkg/eth/types.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"errors"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -264,3 +265,41 @@ type LogResult struct {
 	TxnIndex    int64  `db:"txn_index"`
 	TxHash      string `db:"tx_hash"`
 }
+
+// GetSliceResponse holds response for the eth_getSlice method
+type GetSliceResponse struct {
+	SliceID   string                             `json:"sliceId"`
+	MetaData  GetSliceResponseMetadata           `json:"metadata"`
+	TrieNodes GetSliceResponseTrieNodes          `json:"trieNodes"`
+	Leaves    map[string]GetSliceResponseAccount `json:"leaves"` // we won't be using addresses, but keccak256(address) // TODO: address comment
+}
+
+func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) {
+	sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String())
+	sr.MetaData = GetSliceResponseMetadata{
+		NodeStats: make(map[string]string, 0),
+		TimeStats: make(map[string]string, 0),
+	}
+	sr.Leaves = make(map[string]GetSliceResponseAccount)
+	sr.TrieNodes = GetSliceResponseTrieNodes{
+		Stem:  make(map[string]string),
+		Head:  make(map[string]string),
+		Slice: make(map[string]string),
+	}
+}
+
+type GetSliceResponseMetadata struct {
+	TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one)
+	NodeStats map[string]string `json:"trieNodes"` // total, leaves, smart contracts
+}
+
+type GetSliceResponseTrieNodes struct {
+	Stem  map[string]string `json:"stem"`
+	Head  map[string]string `json:"head"`
+	Slice map[string]string `json:"sliceNodes"`
+}
+
+type GetSliceResponseAccount struct {
+	StorageRoot string `json:"storageRoot"`
+	EVMCode     string `json:"evmCode"`
+}