cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Implement getSlice API #206

Merged
prathamesh0 merged 21 commits from pm-get-slice into v4 2022-12-19 08:42:23 +00:00
Conversation 1 Commits 21 Files Changed 12 +686 -23
12 changed files with 686 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -4,6 +4,7 @@ go 1.18
require (
github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha
github.com/cerc-io/go-eth-state-node-iterator v1.1.9
github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha
github.com/ethereum/go-ethereum v1.10.26
github.com/graph-gophers/graphql-go v1.3.0
@ -41,7 +42,6 @@ require (
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cerc-io/go-eth-state-node-iterator v1.1.9 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/containerd/cgroups v1.0.3 // indirect

5
pkg/eth/api.go
View File

@ -837,6 +837,11 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre
}, state.Error()
}
// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth
func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) {
return pea.B.GetSlice(path, depth, root, storage)
}
// revertError is an API error that encompassas an EVM revertal with JSON error
// code and a binary data blob.
type revertError struct {

5
pkg/eth/api_test.go
View File

@ -24,7 +24,6 @@ import (
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
@ -202,8 +201,8 @@ var _ = Describe("API", func() {
ChainConfig: chainConfig,
VMConfig: vm.Config{},
RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call.
GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: ethServerShared.GroupConfig{
GroupCacheConfig: &shared.GroupCacheConfig{
StateDB: shared.GroupConfig{
Name: "api_test",
CacheSizeInMB: 8,
CacheExpiryInMins: 60,

192
pkg/eth/backend.go
View File

@ -17,6 +17,7 @@
package eth
import (
"bytes"
"context"
"database/sql"
"errors"
@ -42,12 +43,13 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
)
@ -888,6 +890,192 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address,
return storageRlp, err
}
func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) {
response := new(GetSliceResponse)
response.init(path, depth, root)
// Metadata fields
metaData := metaDataFields{}
startTime := makeTimestamp()
t, _ := b.StateDatabase.OpenTrie(root)
metaData.trieLoadingTime = makeTimestamp() - startTime
// Convert the head hex path to a decoded byte path
headPath := common.FromHex(path)
// Get Stem nodes
err := b.getSliceStem(headPath, t, response, &metaData, storage)
if err != nil {
return nil, err
}
// Get Head node
err = b.getSliceHead(headPath, t, response, &metaData, storage)
if err != nil {
return nil, err
}
if depth > 0 {
// Get Slice nodes
err = b.getSliceTrie(headPath, t, response, &metaData, depth, storage)
if err != nil {
return nil, err
}
}
response.populateMetaData(metaData)
return response, nil
}
func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error {
leavesFetchTime := int64(0)
totalStemStartTime := makeTimestamp()
for i := 0; i < len(headPath); i++ {
// Create path for each node along the stem
nodePath := make([]byte, len(headPath[:i]))
copy(nodePath, headPath[:i])
rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(nodePath))
if err != nil {
return err
}
// Skip if node not found
if rawNode == nil {
continue
}
node, nodeElements, err := ResolveNode(nodePath, rawNode, b.StateDatabase.TrieDB())
if err != nil {
return err
}
leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Stem, response.Leaves, node, nodeElements, storage)
if err != nil {
return err
}
// Update metadata
depthReached := len(node.Path) - len(headPath)
if depthReached > metaData.maxDepth {
metaData.maxDepth = depthReached
}
if node.NodeType == sdtypes.Leaf {
metaData.leafCount++
}
leavesFetchTime += leafFetchTime
}
// Update metadata time metrics
totalStemTime := makeTimestamp() - totalStemStartTime
metaData.sliceNodesFetchTime = totalStemTime - leavesFetchTime
metaData.leavesFetchTime += leavesFetchTime
return nil
}
func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error {
totalHeadStartTime := makeTimestamp()
rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(headPath))
if err != nil {
return err
}
// Skip if node not found
if rawNode == nil {
return nil
}
node, nodeElements, err := ResolveNode(headPath, rawNode, b.StateDatabase.TrieDB())
if err != nil {
return err
}
leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Head, response.Leaves, node, nodeElements, storage)
if err != nil {
return err
}
// Update metadata
depthReached := len(node.Path) - len(headPath)
if depthReached > metaData.maxDepth {
metaData.maxDepth = depthReached
}
if node.NodeType == sdtypes.Leaf {
metaData.leafCount++
}
// Update metadata time metrics
totalHeadTime := makeTimestamp() - totalHeadStartTime
metaData.stemNodesFetchTime = totalHeadTime - leafFetchTime
metaData.leavesFetchTime += leafFetchTime
return nil
}
func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, depth int, storage bool) error {
it, timeTaken := getIteratorAtPath(t, headPath)
metaData.trieLoadingTime += timeTaken
leavesFetchTime := int64(0)
totalSliceStartTime := makeTimestamp()
headPathLen := len(headPath)
maxPathLen := headPathLen + depth
descend := true
for it.Next(descend) {
pathLen := len(it.Path())
// End iteration on coming out of subtrie
if pathLen <= headPathLen {
break
}
// Avoid descending further if max depth reached
if pathLen >= maxPathLen {
descend = false
} else {
descend = true
}
// Skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB())
if err != nil {
return err
}
leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Slice, response.Leaves, node, nodeElements, storage)
if err != nil {
return err
}
// Update metadata
depthReached := len(node.Path) - len(headPath)
if depthReached > metaData.maxDepth {
metaData.maxDepth = depthReached
}
if node.NodeType == sdtypes.Leaf {
metaData.leafCount++
}
leavesFetchTime += leafFetchTime
}
// Update metadata time metrics
totalSliceTime := makeTimestamp() - totalSliceStartTime
metaData.sliceNodesFetchTime = totalSliceTime - leavesFetchTime
metaData.leavesFetchTime += leavesFetchTime
return nil
}
// Engine satisfied the ChainContext interface
func (b *Backend) Engine() consensus.Engine {
// TODO: we need to support more than just ethash based engines

107
pkg/eth/backend_utils.go
View File

@ -17,20 +17,32 @@
package eth
import (
"bytes"
"context"
"encoding/json"
"fmt"
"math/big"
nodeiter "github.com/cerc-io/go-eth-state-node-iterator"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
)
var nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
var emptyCodeHash = crypto.Keccak256([]byte{})
// RPCMarshalHeader converts the given header to the RPC output.
// This function is eth/internal so we have to make our own version here...
func RPCMarshalHeader(head *types.Header) map[string]interface{} {
@ -300,3 +312,98 @@ func toBlockNumArg(number *big.Int) string {
}
return hexutil.EncodeBig(number)
}
func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64) {
startTime := makeTimestamp()
var it trie.NodeIterator
if len(startKey)%2 != 0 {
// Zero-pad for odd-length keys, required by HexToKeyBytes()
startKey = append(startKey, 0)
it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
} else {
it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
// Step to the required node (not required if original startKey was odd-length)
it.Next(true)
}
return it, makeTimestamp() - startTime
}
func fillSliceNodeData(
ethDB ethdb.KeyValueReader,
nodesMap map[string]string,
leavesMap map[string]GetSliceResponseAccount,
node sdtypes.StateNode,
nodeElements []interface{},
storage bool,
) (int64, error) {
// Populate the nodes map
nodeValHash := crypto.Keccak256Hash(node.NodeValue)
nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue)
// Extract account data if it's a Leaf node
leafStartTime := makeTimestamp()
if node.NodeType == sdtypes.Leaf && !storage {
stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements)
if err != nil {
return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error())
}
if len(code) > 0 {
// Populate the leaves map
leavesMap[stateLeafKey] = GetSliceResponseAccount{
StorageRoot: storageRoot,
EVMCode: common.Bytes2Hex(code),
}
}
}
return makeTimestamp() - leafStartTime, nil
}
func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) {
var account types.StateAccount
if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
}
if bytes.Equal(account.CodeHash, emptyCodeHash) {
return "", "", nil, nil
}
// Extract state leaf key
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
stateLeafKeyString := common.BytesToHash(leafKey).String()
storageRootString := account.Root.String()
// Extract codeHash and get code
codeHash := common.BytesToHash(account.CodeHash)
codeBytes := rawdb.ReadCode(ethDB, codeHash)
return stateLeafKeyString, storageRootString, codeBytes, nil
}
func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
nodePath := make([]byte, len(path))
copy(nodePath, path)
var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
return sdtypes.StateNode{}, nil, err
}
ty, err := sdtrie.CheckKeyType(nodeElements)
if err != nil {
return sdtypes.StateNode{}, nil, err
}
return sdtypes.StateNode{
NodeType: ty,
Path: nodePath,
NodeValue: node,
}, nodeElements, nil
}

1
pkg/eth/cid_retriever_test.go
View File

@ -300,6 +300,7 @@ var _ = Describe("Retriever", func() {
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
Expect(err).ToNot(HaveOccurred())
cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
Expect(err).ToNot(HaveOccurred())
Expect(empty).ToNot(BeTrue())

299
pkg/eth/eth_state_test.go
View File

@ -19,6 +19,7 @@ package eth_test
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/big"
"time"
@ -26,7 +27,6 @@ import (
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth"
"github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers"
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -44,6 +44,40 @@ import (
var (
parsedABI abi.ABI
block1StateRoot = common.HexToHash("0xa1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21")
rootDataHashBlock1 = "a1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21"
rootDataBlock1 = "f871a0577652b625b77bdb5bf77bc43f3125cad7464d679d1575565277d3611b8053e780808080a0fe889f10e5db8f2c2bf355928152a17f6e3bb99a9241ac6d84c77e6264509c798080808080808080a011db0cda34a896dabeb6839bb06a38f49514cfa486435984eb013b7df9ee85c58080"
block5StateRoot = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43")
rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43"
rootDataBlock5 = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080"
account1DataHash = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4"
account1Data = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659"
account1StateLeafKey = "0x6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"
account1Code = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032"
account2DataHash = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35"
account2Data = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
account3DataHash = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2"
account3Data = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
account4DataHash = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951"
account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91"
account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0")
storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0"
storageRootDataBlock5 = "f838a120290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
contractStorageRootBlock4 = common.HexToHash("0x64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68")
storageRootDataHashBlock4 = "64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68"
storageRootDataBlock4 = "f8518080a08e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d558080808080808080a07b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af8080808080"
storageNode1DataHash = "7b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af"
storageNode1Data = "e2a0310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf609"
storageNode2DataHash = "8e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d55"
storageNode2Data = "f7a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
)
func init() {
@ -81,8 +115,8 @@ var _ = Describe("eth state reading tests", func() {
ChainConfig: chainConfig,
VMConfig: vm.Config{},
RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call.
GroupCacheConfig: &ethServerShared.GroupCacheConfig{
StateDB: ethServerShared.GroupConfig{
GroupCacheConfig: &shared.GroupCacheConfig{
StateDB: shared.GroupConfig{
Name: "eth_state_test",
CacheSizeInMB: 8,
CacheExpiryInMins: 60,
@ -529,4 +563,263 @@ var _ = Describe("eth state reading tests", func() {
Expect(header).To(Equal(expectedCanonicalHeader))
})
})
Describe("eth_getSlice", func() {
It("Retrieves the state slice for root path", func() {
path := "0x"
depth := 3
sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "1",
"01-max-depth": "1",
"02-total-trie-nodes": "6",
"03-leaves": "5",
"04-smart-contracts": "1",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{},
Head: map[string]string{
rootDataHashBlock5: rootDataBlock5,
},
Slice: map[string]string{
account1DataHash: account1Data,
account2DataHash: account2Data,
account3DataHash: account3Data,
account4DataHash: account4Data,
account5DataHash: account5Data,
},
},
Leaves: map[string]eth.GetSliceResponseAccount{
account1StateLeafKey: {
StorageRoot: contractStorageRootBlock5.Hex(),
EVMCode: account1Code,
},
},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the state slice for root path with 0 depth", func() {
path := "0x"
depth := 0
sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "1",
"01-max-depth": "0",
"02-total-trie-nodes": "1",
"03-leaves": "0",
"04-smart-contracts": "0",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{},
Head: map[string]string{
rootDataHashBlock5: rootDataBlock5,
},
Slice: map[string]string{},
},
Leaves: map[string]eth.GetSliceResponseAccount{},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the state slice for a path to an account", func() {
path := "0x06"
depth := 2
sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "2",
"01-max-depth": "0",
"02-total-trie-nodes": "2",
"03-leaves": "1",
"04-smart-contracts": "1",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{
rootDataHashBlock5: rootDataBlock5,
},
Head: map[string]string{
account1DataHash: account1Data,
},
Slice: map[string]string{},
},
Leaves: map[string]eth.GetSliceResponseAccount{
account1StateLeafKey: {
StorageRoot: contractStorageRootBlock5.Hex(),
EVMCode: account1Code,
},
},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the state slice for a path to a non-existing account", func() {
path := "0x06"
depth := 2
sliceResponse, err := api.GetSlice(ctx, path, depth, block1StateRoot, false)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block1StateRoot.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "1",
"01-max-depth": "0",
"02-total-trie-nodes": "1",
"03-leaves": "0",
"04-smart-contracts": "0",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{
rootDataHashBlock1: rootDataBlock1,
},
Head: map[string]string{},
Slice: map[string]string{},
},
Leaves: map[string]eth.GetSliceResponseAccount{},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the storage slice for root path", func() {
path := "0x"
depth := 2
sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "1",
"01-max-depth": "1",
"02-total-trie-nodes": "3",
"03-leaves": "2",
"04-smart-contracts": "0",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{},
Head: map[string]string{
storageRootDataHashBlock4: storageRootDataBlock4,
},
Slice: map[string]string{
storageNode1DataHash: storageNode1Data,
storageNode2DataHash: storageNode2Data,
},
},
Leaves: map[string]eth.GetSliceResponseAccount{},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the storage slice for root path with 0 depth", func() {
path := "0x"
depth := 0
sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "1",
"01-max-depth": "0",
"02-total-trie-nodes": "1",
"03-leaves": "0",
"04-smart-contracts": "0",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{},
Head: map[string]string{
storageRootDataHashBlock4: storageRootDataBlock4,
},
Slice: map[string]string{},
},
Leaves: map[string]eth.GetSliceResponseAccount{},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the storage slice for root path with deleted nodes", func() {
path := "0x"
depth := 2
sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock5, true)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock5.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "1",
"01-max-depth": "0",
"02-total-trie-nodes": "1",
"03-leaves": "1",
"04-smart-contracts": "0",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{},
Head: map[string]string{
storageRootDataHashBlock5: storageRootDataBlock5,
},
Slice: map[string]string{},
},
Leaves: map[string]eth.GetSliceResponseAccount{},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
It("Retrieves the storage slice for a path to a storage node", func() {
path := "0x0b"
depth := 2
sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
Expect(err).ToNot(HaveOccurred())
expectedResponse := eth.GetSliceResponse{
SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
MetaData: eth.GetSliceResponseMetadata{
NodeStats: map[string]string{
"00-stem-and-head-nodes": "2",
"01-max-depth": "0",
"02-total-trie-nodes": "2",
"03-leaves": "1",
"04-smart-contracts": "0",
},
},
TrieNodes: eth.GetSliceResponseTrieNodes{
Stem: map[string]string{
storageRootDataHashBlock4: storageRootDataBlock4,
},
Head: map[string]string{
storageNode1DataHash: storageNode1Data,
},
Slice: map[string]string{},
},
Leaves: map[string]eth.GetSliceResponseAccount{},
}
eth.CheckGetSliceResponse(*sliceResponse, expectedResponse)
})
})
})

9
pkg/eth/filterer.go
View File

@ -208,7 +208,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa
}
// If there are no wanted contract addresses, we keep all receipts that match the topic filter
if len(wantedAddresses) == 0 {
if match := filterMatch(wantedTopics, actualTopics); match == true {
if match := filterMatch(wantedTopics, actualTopics); match {
return true
}
}
@ -218,7 +218,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa
for _, actualAddr := range actualAddresses {
if wantedAddr == actualAddr {
// we keep the receipt if it matches on the topic filter
if match := filterMatch(wantedTopics, actualTopics); match == true {
if match := filterMatch(wantedTopics, actualTopics); match {
return true
}
}
@ -240,10 +240,7 @@ func filterMatch(wantedTopics, actualTopics [][]string) bool {
matches++
}
}
if matches == 4 {
return true
}
return false
return matches == 4
}
// returns 1 if the two slices have a string in common, 0 if they do not

7
pkg/eth/helpers.go
View File

@ -17,6 +17,8 @@
package eth
import (
"time"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
)
@ -34,3 +36,8 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType {
return sdtypes.Unknown
}
}
// Timestamp in milliseconds
func makeTimestamp() int64 {
return time.Now().UnixNano() / int64(time.Millisecond)
}

12
pkg/eth/ipld_retriever.go
View File

@ -32,10 +32,6 @@ import (
)
const (
// node type removed value.
// https://github.com/cerc-io/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34
removedNode = 3
RetrieveHeadersByHashesPgStr = `SELECT cid, data
FROM eth.header_cids
INNER JOIN public.blocks ON (
@ -611,7 +607,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr
return "", nil, err
}
if accountResult.NodeType == removedNode {
if accountResult.NodeType == sdtypes.Removed.Int() {
return "", EmptyNodeValue, nil
}
@ -643,7 +639,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad
return "", nil, err
}
if accountResult.NodeType == removedNode {
if accountResult.NodeType == sdtypes.Removed.Int() {
return "", EmptyNodeValue, nil
}
@ -671,7 +667,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add
if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
return "", nil, nil, err
}
if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() {
return "", EmptyNodeValue, EmptyNodeValue, nil
}
@ -704,7 +700,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad
return "", nil, err
}
if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() {
return "", EmptyNodeValue, nil
}

8
pkg/eth/test_helpers.go
View File

@ -18,6 +18,7 @@ package eth
import (
"github.com/ethereum/go-ethereum/statediff/indexer/models"
. "github.com/onsi/gomega"
)
// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
@ -39,3 +40,10 @@ func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
}
return false
}
func CheckGetSliceResponse(sliceResponse GetSliceResponse, expectedResponse GetSliceResponse) {
Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID))
Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats))
Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes))
Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves))
}

62
pkg/eth/types.go
View File

@ -18,7 +18,9 @@ package eth
import (
"errors"
"fmt"
"math/big"
"strconv"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -264,3 +266,63 @@ type LogResult struct {
TxnIndex int64 `db:"txn_index"`
TxHash string `db:"tx_hash"`
}
// GetSliceResponse holds response for the eth_getSlice method
type GetSliceResponse struct {
SliceID string `json:"sliceId"`
MetaData GetSliceResponseMetadata `json:"metadata"`
TrieNodes GetSliceResponseTrieNodes `json:"trieNodes"`
Leaves map[string]GetSliceResponseAccount `json:"leaves"` // key: Keccak256Hash(address) in hex (leafKey)
}
func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) {
sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String())
sr.MetaData = GetSliceResponseMetadata{
NodeStats: make(map[string]string, 0),
TimeStats: make(map[string]string, 0),
}
sr.Leaves = make(map[string]GetSliceResponseAccount)
sr.TrieNodes = GetSliceResponseTrieNodes{
Stem: make(map[string]string),
Head: make(map[string]string),
Slice: make(map[string]string),
}
}
func (sr *GetSliceResponse) populateMetaData(metaData metaDataFields) {
sr.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head))
sr.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(metaData.maxDepth)
sr.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head) + len(sr.TrieNodes.Slice))
sr.MetaData.NodeStats["03-leaves"] = strconv.Itoa(metaData.leafCount)
sr.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(sr.Leaves))
sr.MetaData.TimeStats["00-trie-loading"] = strconv.FormatInt(metaData.trieLoadingTime, 10)
sr.MetaData.TimeStats["01-fetch-stem-keys"] = strconv.FormatInt(metaData.stemNodesFetchTime, 10)
sr.MetaData.TimeStats["02-fetch-slice-keys"] = strconv.FormatInt(metaData.sliceNodesFetchTime, 10)
sr.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.FormatInt(metaData.leavesFetchTime, 10)
}
type GetSliceResponseMetadata struct {
TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one)
NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts
}
type GetSliceResponseTrieNodes struct {
Stem map[string]string `json:"stem"` // key: Keccak256Hash(data) in hex, value: trie node data in hex
Head map[string]string `json:"head"`
Slice map[string]string `json:"sliceNodes"`
}
type GetSliceResponseAccount struct {
StorageRoot string `json:"storageRoot"`
EVMCode string `json:"evmCode"`
}
type metaDataFields struct {
maxDepth int
leafCount int
trieLoadingTime int64
stemNodesFetchTime int64
sliceNodesFetchTime int64
leavesFetchTime int64
}