Implement getSlice API (#206)

* Implement getSlice API for state nodes

* Implement getSlice API for storage nodes

* Fix the helper function to create a slice of required paths

* Fix query to get state leaf key for given storage root

* Add a test to get state slice for root path

* Add checks in queries to get canonical data

* Add tests to get state slice

* Add a todo for using an iterator

* Avoid filtering out removed nodes

* Add tests to get storage slice

* Remove logs

* Populate extra contract leaves field in the response

* Update tests

* Avoid EOAs in additional data in response

* Use iterator based approach for getSlice

* Skip undesired nodes from stem and head iterators

* Update storage slice tests

* Fix meta data updates

* Use state trie to get stem nodes directly using paths

* Bugfix - Continue processing other trie nodes on encountering a leaf

* Remove unnecessary TODO

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha
+	github.com/cerc-io/go-eth-state-node-iterator v1.1.9
 	github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/graph-gophers/graphql-go v1.3.0
@@ -41,7 +42,6 @@ require (
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
-	github.com/cerc-io/go-eth-state-node-iterator v1.1.9 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/containerd/cgroups v1.0.3 // indirect

pkg/eth/api.go

@@ -837,6 +837,11 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre
 	}, state.Error()
 }
 
+// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth
+func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) {
+	return pea.B.GetSlice(path, depth, root, storage)
+}
+
 // revertError is an API error that encompassas an EVM revertal with JSON error
 // code and a binary data blob.
 type revertError struct {

pkg/eth/api_test.go

@@ -24,7 +24,6 @@ import (
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
-	ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -202,8 +201,8 @@ var _ = Describe("API", func() {
 			ChainConfig: chainConfig,
 			VMConfig:    vm.Config{},
 			RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
-			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+			GroupCacheConfig: &shared.GroupCacheConfig{
-				StateDB: ethServerShared.GroupConfig{
+				StateDB: shared.GroupConfig{
 					Name:                   "api_test",
 					CacheSizeInMB:          8,
 					CacheExpiryInMins:      60,

pkg/eth/backend.go

@@ -17,6 +17,7 @@
 package eth
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"errors"
@@ -42,12 +43,13 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
 
-	ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
-
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
 )
 
@@ -888,6 +890,192 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address,
 	return storageRlp, err
 }
 
+func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) {
+	response := new(GetSliceResponse)
+	response.init(path, depth, root)
+
+	// Metadata fields
+	metaData := metaDataFields{}
+
+	startTime := makeTimestamp()
+	t, _ := b.StateDatabase.OpenTrie(root)
+	metaData.trieLoadingTime = makeTimestamp() - startTime
+
+	// Convert the head hex path to a decoded byte path
+	headPath := common.FromHex(path)
+
+	// Get Stem nodes
+	err := b.getSliceStem(headPath, t, response, &metaData, storage)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get Head node
+	err = b.getSliceHead(headPath, t, response, &metaData, storage)
+	if err != nil {
+		return nil, err
+	}
+
+	if depth > 0 {
+		// Get Slice nodes
+		err = b.getSliceTrie(headPath, t, response, &metaData, depth, storage)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	response.populateMetaData(metaData)
+
+	return response, nil
+}
+
+func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error {
+	leavesFetchTime := int64(0)
+	totalStemStartTime := makeTimestamp()
+
+	for i := 0; i < len(headPath); i++ {
+		// Create path for each node along the stem
+		nodePath := make([]byte, len(headPath[:i]))
+		copy(nodePath, headPath[:i])
+
+		rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(nodePath))
+		if err != nil {
+			return err
+		}
+
+		// Skip if node not found
+		if rawNode == nil {
+			continue
+		}
+
+		node, nodeElements, err := ResolveNode(nodePath, rawNode, b.StateDatabase.TrieDB())
+		if err != nil {
+			return err
+		}
+
+		leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Stem, response.Leaves, node, nodeElements, storage)
+		if err != nil {
+			return err
+		}
+
+		// Update metadata
+		depthReached := len(node.Path) - len(headPath)
+		if depthReached > metaData.maxDepth {
+			metaData.maxDepth = depthReached
+		}
+		if node.NodeType == sdtypes.Leaf {
+			metaData.leafCount++
+		}
+		leavesFetchTime += leafFetchTime
+	}
+
+	// Update metadata time metrics
+	totalStemTime := makeTimestamp() - totalStemStartTime
+	metaData.sliceNodesFetchTime = totalStemTime - leavesFetchTime
+	metaData.leavesFetchTime += leavesFetchTime
+
+	return nil
+}
+
+func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error {
+	totalHeadStartTime := makeTimestamp()
+
+	rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(headPath))
+	if err != nil {
+		return err
+	}
+
+	// Skip if node not found
+	if rawNode == nil {
+		return nil
+	}
+
+	node, nodeElements, err := ResolveNode(headPath, rawNode, b.StateDatabase.TrieDB())
+	if err != nil {
+		return err
+	}
+
+	leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Head, response.Leaves, node, nodeElements, storage)
+	if err != nil {
+		return err
+	}
+
+	// Update metadata
+	depthReached := len(node.Path) - len(headPath)
+	if depthReached > metaData.maxDepth {
+		metaData.maxDepth = depthReached
+	}
+	if node.NodeType == sdtypes.Leaf {
+		metaData.leafCount++
+	}
+
+	// Update metadata time metrics
+	totalHeadTime := makeTimestamp() - totalHeadStartTime
+	metaData.stemNodesFetchTime = totalHeadTime - leafFetchTime
+	metaData.leavesFetchTime += leafFetchTime
+
+	return nil
+}
+
+func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, depth int, storage bool) error {
+	it, timeTaken := getIteratorAtPath(t, headPath)
+	metaData.trieLoadingTime += timeTaken
+
+	leavesFetchTime := int64(0)
+	totalSliceStartTime := makeTimestamp()
+
+	headPathLen := len(headPath)
+	maxPathLen := headPathLen + depth
+	descend := true
+	for it.Next(descend) {
+		pathLen := len(it.Path())
+
+		// End iteration on coming out of subtrie
+		if pathLen <= headPathLen {
+			break
+		}
+
+		// Avoid descending further if max depth reached
+		if pathLen >= maxPathLen {
+			descend = false
+		} else {
+			descend = true
+		}
+
+		// Skip value nodes
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+
+		node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB())
+		if err != nil {
+			return err
+		}
+
+		leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Slice, response.Leaves, node, nodeElements, storage)
+		if err != nil {
+			return err
+		}
+
+		// Update metadata
+		depthReached := len(node.Path) - len(headPath)
+		if depthReached > metaData.maxDepth {
+			metaData.maxDepth = depthReached
+		}
+		if node.NodeType == sdtypes.Leaf {
+			metaData.leafCount++
+		}
+		leavesFetchTime += leafFetchTime
+	}
+
+	// Update metadata time metrics
+	totalSliceTime := makeTimestamp() - totalSliceStartTime
+	metaData.sliceNodesFetchTime = totalSliceTime - leavesFetchTime
+	metaData.leavesFetchTime += leavesFetchTime
+
+	return nil
+}
+
 // Engine satisfied the ChainContext interface
 func (b *Backend) Engine() consensus.Engine {
 	// TODO: we need to support more than just ethash based engines

pkg/eth/backend_utils.go

@@ -17,20 +17,32 @@
 package eth
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"math/big"
 
+	nodeiter "github.com/cerc-io/go-eth-state-node-iterator"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
+var nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
+var emptyCodeHash = crypto.Keccak256([]byte{})
+
 // RPCMarshalHeader converts the given header to the RPC output.
 // This function is eth/internal so we have to make our own version here...
 func RPCMarshalHeader(head *types.Header) map[string]interface{} {
@@ -300,3 +312,98 @@ func toBlockNumArg(number *big.Int) string {
 	}
 	return hexutil.EncodeBig(number)
 }
+
+func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64) {
+	startTime := makeTimestamp()
+	var it trie.NodeIterator
+
+	if len(startKey)%2 != 0 {
+		// Zero-pad for odd-length keys, required by HexToKeyBytes()
+		startKey = append(startKey, 0)
+		it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
+	} else {
+		it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
+		// Step to the required node (not required if original startKey was odd-length)
+		it.Next(true)
+	}
+
+	return it, makeTimestamp() - startTime
+}
+
+func fillSliceNodeData(
+	ethDB ethdb.KeyValueReader,
+	nodesMap map[string]string,
+	leavesMap map[string]GetSliceResponseAccount,
+	node sdtypes.StateNode,
+	nodeElements []interface{},
+	storage bool,
+) (int64, error) {
+	// Populate the nodes map
+	nodeValHash := crypto.Keccak256Hash(node.NodeValue)
+	nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue)
+
+	// Extract account data if it's a Leaf node
+	leafStartTime := makeTimestamp()
+	if node.NodeType == sdtypes.Leaf && !storage {
+		stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements)
+		if err != nil {
+			return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error())
+		}
+
+		if len(code) > 0 {
+			// Populate the leaves map
+			leavesMap[stateLeafKey] = GetSliceResponseAccount{
+				StorageRoot: storageRoot,
+				EVMCode:     common.Bytes2Hex(code),
+			}
+		}
+	}
+
+	return makeTimestamp() - leafStartTime, nil
+}
+
+func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) {
+	var account types.StateAccount
+	if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+		return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+	}
+
+	if bytes.Equal(account.CodeHash, emptyCodeHash) {
+		return "", "", nil, nil
+	}
+
+	// Extract state leaf key
+	partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+	valueNodePath := append(node.Path, partialPath...)
+	encodedPath := trie.HexToCompact(valueNodePath)
+	leafKey := encodedPath[1:]
+	stateLeafKeyString := common.BytesToHash(leafKey).String()
+
+	storageRootString := account.Root.String()
+
+	// Extract codeHash and get code
+	codeHash := common.BytesToHash(account.CodeHash)
+	codeBytes := rawdb.ReadCode(ethDB, codeHash)
+
+	return stateLeafKeyString, storageRootString, codeBytes, nil
+}
+
+func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
+	nodePath := make([]byte, len(path))
+	copy(nodePath, path)
+
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return sdtypes.StateNode{}, nil, err
+	}
+
+	ty, err := sdtrie.CheckKeyType(nodeElements)
+	if err != nil {
+		return sdtypes.StateNode{}, nil, err
+	}
+	return sdtypes.StateNode{
+		NodeType:  ty,
+		Path:      nodePath,
+		NodeValue: node,
+	}, nodeElements, nil
+}

pkg/eth/cid_retriever_test.go

@@ -300,6 +300,7 @@ var _ = Describe("Retriever", func() {
 				AND header_cids.block_number = $1
 				ORDER BY transaction_cids.index`
 			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
+			Expect(err).ToNot(HaveOccurred())
 			cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())

pkg/eth/eth_state_test.go

@@ -19,6 +19,7 @@ package eth_test
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"math/big"
 	"time"
@@ -26,7 +27,6 @@ import (
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers"
 	"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
-	ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -44,6 +44,40 @@ import (
 
 var (
 	parsedABI abi.ABI
+
+	block1StateRoot    = common.HexToHash("0xa1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21")
+	rootDataHashBlock1 = "a1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21"
+	rootDataBlock1     = "f871a0577652b625b77bdb5bf77bc43f3125cad7464d679d1575565277d3611b8053e780808080a0fe889f10e5db8f2c2bf355928152a17f6e3bb99a9241ac6d84c77e6264509c798080808080808080a011db0cda34a896dabeb6839bb06a38f49514cfa486435984eb013b7df9ee85c58080"
+
+	block5StateRoot    = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43")
+	rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43"
+	rootDataBlock5     = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080"
+
+	account1DataHash     = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4"
+	account1Data         = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659"
+	account1StateLeafKey = "0x6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"
+	account1Code         = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032"
+	account2DataHash     = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35"
+	account2Data         = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account3DataHash     = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2"
+	account3Data         = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account4DataHash     = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951"
+	account4Data         = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account5DataHash     = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91"
+	account5Data         = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+
+	contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0")
+	storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0"
+	storageRootDataBlock5     = "f838a120290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
+
+	contractStorageRootBlock4 = common.HexToHash("0x64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68")
+	storageRootDataHashBlock4 = "64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68"
+	storageRootDataBlock4     = "f8518080a08e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d558080808080808080a07b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af8080808080"
+
+	storageNode1DataHash = "7b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af"
+	storageNode1Data     = "e2a0310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf609"
+	storageNode2DataHash = "8e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d55"
+	storageNode2Data     = "f7a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
 )
 
 func init() {
@@ -81,8 +115,8 @@ var _ = Describe("eth state reading tests", func() {
 			ChainConfig: chainConfig,
 			VMConfig:    vm.Config{},
 			RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
-			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+			GroupCacheConfig: &shared.GroupCacheConfig{
-				StateDB: ethServerShared.GroupConfig{
+				StateDB: shared.GroupConfig{
 					Name:                   "eth_state_test",
 					CacheSizeInMB:          8,
 					CacheExpiryInMins:      60,
@@ -529,4 +563,263 @@ var _ = Describe("eth state reading tests", func() {
 			Expect(header).To(Equal(expectedCanonicalHeader))
 		})
 	})
+
+	Describe("eth_getSlice", func() {
+		It("Retrieves the state slice for root path", func() {
+			path := "0x"
+			depth := 3
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "1",
+						"02-total-trie-nodes":    "6",
+						"03-leaves":              "5",
+						"04-smart-contracts":     "1",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Slice: map[string]string{
+						account1DataHash: account1Data,
+						account2DataHash: account2Data,
+						account3DataHash: account3Data,
+						account4DataHash: account4Data,
+						account5DataHash: account5Data,
+					},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{
+					account1StateLeafKey: {
+						StorageRoot: contractStorageRootBlock5.Hex(),
+						EVMCode:     account1Code,
+					},
+				},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for root path with 0 depth", func() {
+			path := "0x"
+			depth := 0
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for a path to an account", func() {
+			path := "0x06"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "2",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "2",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "1",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Head: map[string]string{
+						account1DataHash: account1Data,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{
+					account1StateLeafKey: {
+						StorageRoot: contractStorageRootBlock5.Hex(),
+						EVMCode:     account1Code,
+					},
+				},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for a path to a non-existing account", func() {
+			path := "0x06"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block1StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block1StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						rootDataHashBlock1: rootDataBlock1,
+					},
+					Head:  map[string]string{},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+
+		It("Retrieves the storage slice for root path", func() {
+			path := "0x"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "1",
+						"02-total-trie-nodes":    "3",
+						"03-leaves":              "2",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Slice: map[string]string{
+						storageNode1DataHash: storageNode1Data,
+						storageNode2DataHash: storageNode2Data,
+					},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for root path with 0 depth", func() {
+			path := "0x"
+			depth := 0
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for root path with deleted nodes", func() {
+			path := "0x"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock5, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock5.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock5: storageRootDataBlock5,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for a path to a storage node", func() {
+			path := "0x0b"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "2",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "2",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Head: map[string]string{
+						storageNode1DataHash: storageNode1Data,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+	})
 })

pkg/eth/filterer.go

@@ -208,7 +208,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa
 	}
 	// If there are no wanted contract addresses, we keep all receipts that match the topic filter
 	if len(wantedAddresses) == 0 {
-		if match := filterMatch(wantedTopics, actualTopics); match == true {
+		if match := filterMatch(wantedTopics, actualTopics); match {
 			return true
 		}
 	}
@@ -218,7 +218,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa
 		for _, actualAddr := range actualAddresses {
 			if wantedAddr == actualAddr {
 				// we keep the receipt if it matches on the topic filter
-				if match := filterMatch(wantedTopics, actualTopics); match == true {
+				if match := filterMatch(wantedTopics, actualTopics); match {
 					return true
 				}
 			}
@@ -240,10 +240,7 @@ func filterMatch(wantedTopics, actualTopics [][]string) bool {
 			matches++
 		}
 	}
-	if matches == 4 {
+	return matches == 4
-		return true
-	}
-	return false
 }
 
 // returns 1 if the two slices have a string in common, 0 if they do not

pkg/eth/helpers.go

@@ -17,6 +17,8 @@
 package eth
 
 import (
+	"time"
+
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 )
 
@@ -34,3 +36,8 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType {
 		return sdtypes.Unknown
 	}
 }
+
+// Timestamp in milliseconds
+func makeTimestamp() int64 {
+	return time.Now().UnixNano() / int64(time.Millisecond)
+}

pkg/eth/ipld_retriever.go

@@ -32,10 +32,6 @@ import (
 )
 
 const (
-	// node type removed value.
-	// https://github.com/cerc-io/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34
-	removedNode = 3
-
 	RetrieveHeadersByHashesPgStr = `SELECT cid, data
 								FROM eth.header_cids
 									INNER JOIN public.blocks ON (
@@ -611,7 +607,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr
 		return "", nil, err
 	}
 
-	if accountResult.NodeType == removedNode {
+	if accountResult.NodeType == sdtypes.Removed.Int() {
 		return "", EmptyNodeValue, nil
 	}
 
@@ -643,7 +639,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad
 		return "", nil, err
 	}
 
-	if accountResult.NodeType == removedNode {
+	if accountResult.NodeType == sdtypes.Removed.Int() {
 		return "", EmptyNodeValue, nil
 	}
 
@@ -671,7 +667,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add
 	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
 		return "", nil, nil, err
 	}
-	if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
+	if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() {
 		return "", EmptyNodeValue, EmptyNodeValue, nil
 	}
 
@@ -704,7 +700,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad
 		return "", nil, err
 	}
 
-	if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
+	if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() {
 		return "", EmptyNodeValue, nil
 	}
 

pkg/eth/test_helpers.go

@@ -18,6 +18,7 @@ package eth
 
 import (
 	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	. "github.com/onsi/gomega"
 )
 
 // TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
@@ -39,3 +40,10 @@ func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
 	}
 	return false
 }
+
+func CheckGetSliceResponse(sliceResponse GetSliceResponse, expectedResponse GetSliceResponse) {
+	Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID))
+	Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats))
+	Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes))
+	Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves))
+}

pkg/eth/types.go

@@ -18,7 +18,9 @@ package eth
 
 import (
 	"errors"
+	"fmt"
 	"math/big"
+	"strconv"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -264,3 +266,63 @@ type LogResult struct {
 	TxnIndex    int64  `db:"txn_index"`
 	TxHash      string `db:"tx_hash"`
 }
+
+// GetSliceResponse holds response for the eth_getSlice method
+type GetSliceResponse struct {
+	SliceID   string                             `json:"sliceId"`
+	MetaData  GetSliceResponseMetadata           `json:"metadata"`
+	TrieNodes GetSliceResponseTrieNodes          `json:"trieNodes"`
+	Leaves    map[string]GetSliceResponseAccount `json:"leaves"` // key: Keccak256Hash(address) in hex (leafKey)
+}
+
+func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) {
+	sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String())
+	sr.MetaData = GetSliceResponseMetadata{
+		NodeStats: make(map[string]string, 0),
+		TimeStats: make(map[string]string, 0),
+	}
+	sr.Leaves = make(map[string]GetSliceResponseAccount)
+	sr.TrieNodes = GetSliceResponseTrieNodes{
+		Stem:  make(map[string]string),
+		Head:  make(map[string]string),
+		Slice: make(map[string]string),
+	}
+}
+
+func (sr *GetSliceResponse) populateMetaData(metaData metaDataFields) {
+	sr.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head))
+	sr.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(metaData.maxDepth)
+	sr.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head) + len(sr.TrieNodes.Slice))
+	sr.MetaData.NodeStats["03-leaves"] = strconv.Itoa(metaData.leafCount)
+	sr.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(sr.Leaves))
+
+	sr.MetaData.TimeStats["00-trie-loading"] = strconv.FormatInt(metaData.trieLoadingTime, 10)
+	sr.MetaData.TimeStats["01-fetch-stem-keys"] = strconv.FormatInt(metaData.stemNodesFetchTime, 10)
+	sr.MetaData.TimeStats["02-fetch-slice-keys"] = strconv.FormatInt(metaData.sliceNodesFetchTime, 10)
+	sr.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.FormatInt(metaData.leavesFetchTime, 10)
+}
+
+type GetSliceResponseMetadata struct {
+	TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one)
+	NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts
+}
+
+type GetSliceResponseTrieNodes struct {
+	Stem  map[string]string `json:"stem"` // key: Keccak256Hash(data) in hex, value: trie node data in hex
+	Head  map[string]string `json:"head"`
+	Slice map[string]string `json:"sliceNodes"`
+}
+
+type GetSliceResponseAccount struct {
+	StorageRoot string `json:"storageRoot"`
+	EVMCode     string `json:"evmCode"`
+}
+
+type metaDataFields struct {
+	maxDepth            int
+	leafCount           int
+	trieLoadingTime     int64
+	stemNodesFetchTime  int64
+	sliceNodesFetchTime int64
+	leavesFetchTime     int64
+}