cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Use state trie to get stem nodes directly using paths

Browse Source
This commit is contained in:
Prathamesh Musale 2022-12-16 17:21:40 +05:30
parent 865b965669
commit b5bb47e77d
3 changed files with 102 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -4,6 +4,7 @@ go 1.18
require (
github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha
github.com/cerc-io/go-eth-state-node-iterator v1.1.9
github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha
github.com/ethereum/go-ethereum v1.10.26
github.com/graph-gophers/graphql-go v1.3.0
@ -41,7 +42,6 @@ require (
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cerc-io/go-eth-state-node-iterator v1.1.9 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/containerd/cgroups v1.0.3 // indirect

99
pkg/eth/backend.go
View File

@ -44,6 +44,8 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
@ -899,6 +901,7 @@ func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage boo
t, _ := b.StateDatabase.OpenTrie(root)
metaData.trieLoadingTime = makeTimestamp() - startTime
// Convert the head hex path to a decoded byte path
headPath := common.FromHex(path)
// Get Stem nodes
@ -927,65 +930,90 @@ func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage boo
}
func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error {
leavesFetchTime := int64(0)
totalStemStartTime := makeTimestamp()
for i := 0; i < len(headPath); i++ {
// Create path for each node along the stem
startPath := make([]byte, len(headPath[:i]))
copy(startPath, headPath[:i])
nodePath := make([]byte, len(headPath[:i]))
copy(nodePath, headPath[:i])
// Create an iterator initialized at startPath
it, timeTaken := getIteratorAtPath(t, startPath)
metaData.trieLoadingTime += timeTaken
// TODO: verify doesn't return for non-existent node
rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(nodePath))
if err != nil {
return err
}
// Skip if iterator not at required path (might happen if node not present at given path)
if !bytes.Equal(it.Path(), startPath) {
// Skip if node not found
if rawNode == nil {
continue
}
sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Stem, response.Leaves, it, storage)
node, nodeElements, err := ResolveNode(nodePath, rawNode, b.StateDatabase.TrieDB())
if err != nil {
return err
}
leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Stem, response.Leaves, node, nodeElements, storage)
if err != nil {
return err
}
// Update metadata
depthReached := sliceNodeMetrics.pathLen - len(headPath)
depthReached := len(node.Path) - len(headPath)
if depthReached > metaData.maxDepth {
metaData.maxDepth = depthReached
}
if sliceNodeMetrics.isLeaf {
if node.NodeType == sdtypes.Leaf {
metaData.leafCount++
}
metaData.stemNodesFetchTime += sliceNodeMetrics.nodeFetchTime
metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime
leavesFetchTime += leafFetchTime
}
// Update metadata time metrics
totalStemTime := makeTimestamp() - totalStemStartTime
metaData.sliceNodesFetchTime = totalStemTime - leavesFetchTime
metaData.leavesFetchTime += leavesFetchTime
return nil
}
func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error {
// Create an iterator initialized at headPath
it, timeTaken := getIteratorAtPath(t, headPath)
metaData.trieLoadingTime += timeTaken
totalHeadStartTime := makeTimestamp()
// Skip if iterator not at required path (might happen if node not present at given path)
if !bytes.Equal(it.Path(), headPath) {
rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(headPath))
if err != nil {
return err
}
// Skip if node not found
if rawNode == nil {
return nil
}
sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Head, response.Leaves, it, storage)
node, nodeElements, err := ResolveNode(headPath, rawNode, b.StateDatabase.TrieDB())
if err != nil {
return err
}
leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Head, response.Leaves, node, nodeElements, storage)
if err != nil {
return err
}
// Update metadata
depthReached := sliceNodeMetrics.pathLen - len(headPath)
depthReached := len(node.Path) - len(headPath)
if depthReached > metaData.maxDepth {
metaData.maxDepth = depthReached
}
if sliceNodeMetrics.isLeaf {
if node.NodeType == sdtypes.Leaf {
metaData.leafCount++
}
metaData.stemNodesFetchTime += sliceNodeMetrics.nodeFetchTime
metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime
// Update metadata time metrics
totalHeadTime := makeTimestamp() - totalHeadStartTime
metaData.stemNodesFetchTime = totalHeadTime - leafFetchTime
metaData.leavesFetchTime += leafFetchTime
return nil
}
@ -994,6 +1022,9 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice
it, timeTaken := getIteratorAtPath(t, headPath)
metaData.trieLoadingTime += timeTaken
leavesFetchTime := int64(0)
totalSliceStartTime := makeTimestamp()
headPathLen := len(headPath)
maxPathLen := headPathLen + depth
descend := true
@ -1012,23 +1043,37 @@ func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSlice
descend = true
}
sliceNodeMetrics, err := fillSliceNodeData(b.EthDB, b.StateDatabase.TrieDB(), response.TrieNodes.Slice, response.Leaves, it, storage)
// Skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
return nil
}
node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB())
if err != nil {
return err
}
leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Slice, response.Leaves, node, nodeElements, storage)
if err != nil {
return err
}
// Update metadata
depthReached := sliceNodeMetrics.pathLen - len(headPath)
depthReached := len(node.Path) - len(headPath)
if depthReached > metaData.maxDepth {
metaData.maxDepth = depthReached
}
if sliceNodeMetrics.isLeaf {
if node.NodeType == sdtypes.Leaf {
metaData.leafCount++
}
metaData.sliceNodesFetchTime += sliceNodeMetrics.nodeFetchTime
metaData.leavesFetchTime += sliceNodeMetrics.leafFetchTime
leavesFetchTime += leafFetchTime
}
// Update metadata time metrics
totalSliceTime := makeTimestamp() - totalSliceStartTime
metaData.sliceNodesFetchTime = totalSliceTime - leavesFetchTime
metaData.leavesFetchTime += leavesFetchTime
return nil
}

67
pkg/eth/backend_utils.go
View File

@ -330,52 +330,25 @@ func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64)
return it, makeTimestamp() - startTime
}
type SliceNodeMetrics struct {
pathLen int
isLeaf bool
nodeFetchTime int64
leafFetchTime int64
}
func fillSliceNodeData(
ethDB ethdb.KeyValueReader,
trieDB *trie.Database,
nodesMap map[string]string,
leavesMap map[string]GetSliceResponseAccount,
it trie.NodeIterator,
node sdtypes.StateNode,
nodeElements []interface{},
storage bool,
) (SliceNodeMetrics, error) {
// Skip value nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
return SliceNodeMetrics{}, nil
}
nodeStartTime := makeTimestamp()
node, nodeElements, err := sdtrie.ResolveNode(it, trieDB)
if err != nil {
return SliceNodeMetrics{}, err
}
sliceNodeMetrics := SliceNodeMetrics{
pathLen: len(it.Path()),
isLeaf: node.NodeType == sdtypes.Leaf,
}
) (int64, error) {
// Populate the nodes map
nodeVal := node.NodeValue
nodeValHash := crypto.Keccak256Hash(nodeVal)
nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(nodeVal)
sliceNodeMetrics.nodeFetchTime = makeTimestamp() - nodeStartTime
// nodeVal := node.NodeValue
nodeValHash := crypto.Keccak256Hash(node.NodeValue)
nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue)
// Extract account data if it's a Leaf node
leafStartTime := makeTimestamp()
if node.NodeType == sdtypes.Leaf && !storage {
leafStartTime := makeTimestamp()
stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements)
if err != nil {
return SliceNodeMetrics{}, fmt.Errorf("GetSlice account lookup error: %s", err.Error())
return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error())
}
if len(code) > 0 {
@ -385,11 +358,9 @@ func fillSliceNodeData(
EVMCode: common.Bytes2Hex(code),
}
}
sliceNodeMetrics.leafFetchTime = makeTimestamp() - leafStartTime
}
return sliceNodeMetrics, nil
return makeTimestamp() - leafStartTime, nil
}
func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) {
@ -417,3 +388,23 @@ func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNo
return stateLeafKeyString, storageRootString, codeBytes, nil
}
func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
nodePath := make([]byte, len(path))
copy(nodePath, path)
var nodeElements []interface{}
if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
return sdtypes.StateNode{}, nil, err
}
ty, err := sdtrie.CheckKeyType(nodeElements)
if err != nil {
return sdtypes.StateNode{}, nil, err
}
return sdtypes.StateNode{
NodeType: ty,
Path: nodePath,
NodeValue: node,
}, nodeElements, nil
}