From 721a728d4b048519aa2bb80f0248ae48a31a0818 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 19 Dec 2022 02:42:23 -0600 Subject: [PATCH] Implement `getSlice` API (#206) * Implement getSlice API for state nodes * Implement getSlice API for storage nodes * Fix the helper function to create a slice of required paths * Fix query to get state leaf key for given storage root * Add a test to get state slice for root path * Add checks in queries to get canonical data * Add tests to get state slice * Add a todo for using an iterator * Avoid filtering out removed nodes * Add tests to get storage slice * Remove logs * Populate extra contract leaves field in the response * Update tests * Avoid EOAs in additional data in response * Use iterator based approach for getSlice * Skip undesired nodes from stem and head iterators * Update storage slice tests * Fix meta data updates * Use state trie to get stem nodes directly using paths * Bugfix - Continue processing other trie nodes on encountering a leaf * Remove unnecessary TODO --- go.mod | 2 +- pkg/eth/api.go | 5 + pkg/eth/api_test.go | 5 +- pkg/eth/backend.go | 192 +++++++++++++++++++++- pkg/eth/backend_utils.go | 107 ++++++++++++ pkg/eth/cid_retriever_test.go | 1 + pkg/eth/eth_state_test.go | 299 +++++++++++++++++++++++++++++++++- pkg/eth/filterer.go | 9 +- pkg/eth/helpers.go | 7 + pkg/eth/ipld_retriever.go | 12 +- pkg/eth/test_helpers.go | 8 + pkg/eth/types.go | 62 +++++++ 12 files changed, 686 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 28e6e31a..ca0fd5f6 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/cerc-io/eth-ipfs-state-validator/v4 v4.0.10-alpha + github.com/cerc-io/go-eth-state-node-iterator v1.1.9 github.com/cerc-io/ipfs-ethdb/v4 v4.0.10-alpha github.com/ethereum/go-ethereum v1.10.26 github.com/graph-gophers/graphql-go v1.3.0 @@ -41,7 +42,6 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cerc-io/go-eth-state-node-iterator v1.1.9 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cheekybits/genny v1.0.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect diff --git a/pkg/eth/api.go b/pkg/eth/api.go index 5fc64393..a3605a4d 100644 --- a/pkg/eth/api.go +++ b/pkg/eth/api.go @@ -837,6 +837,11 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre }, state.Error() } +// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth +func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) { + return pea.B.GetSlice(path, depth, root, storage) +} + // revertError is an API error that encompassas an EVM revertal with JSON error // code and a binary data blob. type revertError struct { diff --git a/pkg/eth/api_test.go b/pkg/eth/api_test.go index feb500bf..bbbd1a06 100644 --- a/pkg/eth/api_test.go +++ b/pkg/eth/api_test.go @@ -24,7 +24,6 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" - ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -202,8 +201,8 @@ var _ = Describe("API", func() { ChainConfig: chainConfig, VMConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call. - GroupCacheConfig: ðServerShared.GroupCacheConfig{ - StateDB: ethServerShared.GroupConfig{ + GroupCacheConfig: &shared.GroupCacheConfig{ + StateDB: shared.GroupConfig{ Name: "api_test", CacheSizeInMB: 8, CacheExpiryInMins: 60, diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 24d86e54..1123a875 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -17,6 +17,7 @@ package eth import ( + "bytes" "context" "database/sql" "errors" @@ -42,12 +43,13 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" - ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared" - "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" ) @@ -888,6 +890,192 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, return storageRlp, err } +func (b *Backend) GetSlice(path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) { + response := new(GetSliceResponse) + response.init(path, depth, root) + + // Metadata fields + metaData := metaDataFields{} + + startTime := makeTimestamp() + t, _ := b.StateDatabase.OpenTrie(root) + metaData.trieLoadingTime = makeTimestamp() - startTime + + // Convert the head hex path to a decoded byte path + headPath := common.FromHex(path) + + // Get Stem nodes + err := b.getSliceStem(headPath, t, response, &metaData, storage) + if err != nil { + return nil, err + } + + // Get Head node + err = b.getSliceHead(headPath, t, response, &metaData, storage) + if err != nil { + return nil, err + } + + if depth > 0 { + // Get Slice nodes + err = b.getSliceTrie(headPath, t, response, &metaData, depth, storage) + if err != nil { + return nil, err + } + } + + response.populateMetaData(metaData) + + return response, nil +} + +func (b *Backend) getSliceStem(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error { + leavesFetchTime := int64(0) + totalStemStartTime := makeTimestamp() + + for i := 0; i < len(headPath); i++ { + // Create path for each node along the stem + nodePath := make([]byte, len(headPath[:i])) + copy(nodePath, headPath[:i]) + + rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(nodePath)) + if err != nil { + return err + } + + // Skip if node not found + if rawNode == nil { + continue + } + + node, nodeElements, err := ResolveNode(nodePath, rawNode, b.StateDatabase.TrieDB()) + if err != nil { + return err + } + + leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Stem, response.Leaves, node, nodeElements, storage) + if err != nil { + return err + } + + // Update metadata + depthReached := len(node.Path) - len(headPath) + if depthReached > metaData.maxDepth { + metaData.maxDepth = depthReached + } + if node.NodeType == sdtypes.Leaf { + metaData.leafCount++ + } + leavesFetchTime += leafFetchTime + } + + // Update metadata time metrics + totalStemTime := makeTimestamp() - totalStemStartTime + metaData.sliceNodesFetchTime = totalStemTime - leavesFetchTime + metaData.leavesFetchTime += leavesFetchTime + + return nil +} + +func (b *Backend) getSliceHead(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, storage bool) error { + totalHeadStartTime := makeTimestamp() + + rawNode, _, err := t.(*trie.StateTrie).TryGetNode(trie.HexToCompact(headPath)) + if err != nil { + return err + } + + // Skip if node not found + if rawNode == nil { + return nil + } + + node, nodeElements, err := ResolveNode(headPath, rawNode, b.StateDatabase.TrieDB()) + if err != nil { + return err + } + + leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Head, response.Leaves, node, nodeElements, storage) + if err != nil { + return err + } + + // Update metadata + depthReached := len(node.Path) - len(headPath) + if depthReached > metaData.maxDepth { + metaData.maxDepth = depthReached + } + if node.NodeType == sdtypes.Leaf { + metaData.leafCount++ + } + + // Update metadata time metrics + totalHeadTime := makeTimestamp() - totalHeadStartTime + metaData.stemNodesFetchTime = totalHeadTime - leafFetchTime + metaData.leavesFetchTime += leafFetchTime + + return nil +} + +func (b *Backend) getSliceTrie(headPath []byte, t state.Trie, response *GetSliceResponse, metaData *metaDataFields, depth int, storage bool) error { + it, timeTaken := getIteratorAtPath(t, headPath) + metaData.trieLoadingTime += timeTaken + + leavesFetchTime := int64(0) + totalSliceStartTime := makeTimestamp() + + headPathLen := len(headPath) + maxPathLen := headPathLen + depth + descend := true + for it.Next(descend) { + pathLen := len(it.Path()) + + // End iteration on coming out of subtrie + if pathLen <= headPathLen { + break + } + + // Avoid descending further if max depth reached + if pathLen >= maxPathLen { + descend = false + } else { + descend = true + } + + // Skip value nodes + if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { + continue + } + + node, nodeElements, err := sdtrie.ResolveNode(it, b.StateDatabase.TrieDB()) + if err != nil { + return err + } + + leafFetchTime, err := fillSliceNodeData(b.EthDB, response.TrieNodes.Slice, response.Leaves, node, nodeElements, storage) + if err != nil { + return err + } + + // Update metadata + depthReached := len(node.Path) - len(headPath) + if depthReached > metaData.maxDepth { + metaData.maxDepth = depthReached + } + if node.NodeType == sdtypes.Leaf { + metaData.leafCount++ + } + leavesFetchTime += leafFetchTime + } + + // Update metadata time metrics + totalSliceTime := makeTimestamp() - totalSliceStartTime + metaData.sliceNodesFetchTime = totalSliceTime - leavesFetchTime + metaData.leavesFetchTime += leavesFetchTime + + return nil +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/backend_utils.go b/pkg/eth/backend_utils.go index 7adf9792..68655604 100644 --- a/pkg/eth/backend_utils.go +++ b/pkg/eth/backend_utils.go @@ -17,20 +17,32 @@ package eth import ( + "bytes" "context" "encoding/json" "fmt" "math/big" + nodeiter "github.com/cerc-io/go-eth-state-node-iterator" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" + "github.com/ethereum/go-ethereum/trie" ) +var nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") +var emptyCodeHash = crypto.Keccak256([]byte{}) + // RPCMarshalHeader converts the given header to the RPC output. // This function is eth/internal so we have to make our own version here... func RPCMarshalHeader(head *types.Header) map[string]interface{} { @@ -300,3 +312,98 @@ func toBlockNumArg(number *big.Int) string { } return hexutil.EncodeBig(number) } + +func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64) { + startTime := makeTimestamp() + var it trie.NodeIterator + + if len(startKey)%2 != 0 { + // Zero-pad for odd-length keys, required by HexToKeyBytes() + startKey = append(startKey, 0) + it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey)) + } else { + it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey)) + // Step to the required node (not required if original startKey was odd-length) + it.Next(true) + } + + return it, makeTimestamp() - startTime +} + +func fillSliceNodeData( + ethDB ethdb.KeyValueReader, + nodesMap map[string]string, + leavesMap map[string]GetSliceResponseAccount, + node sdtypes.StateNode, + nodeElements []interface{}, + storage bool, +) (int64, error) { + // Populate the nodes map + nodeValHash := crypto.Keccak256Hash(node.NodeValue) + nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue) + + // Extract account data if it's a Leaf node + leafStartTime := makeTimestamp() + if node.NodeType == sdtypes.Leaf && !storage { + stateLeafKey, storageRoot, code, err := extractContractAccountInfo(ethDB, node, nodeElements) + if err != nil { + return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error()) + } + + if len(code) > 0 { + // Populate the leaves map + leavesMap[stateLeafKey] = GetSliceResponseAccount{ + StorageRoot: storageRoot, + EVMCode: common.Bytes2Hex(code), + } + } + } + + return makeTimestamp() - leafStartTime, nil +} + +func extractContractAccountInfo(ethDB ethdb.KeyValueReader, node sdtypes.StateNode, nodeElements []interface{}) (string, string, []byte, error) { + var account types.StateAccount + if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { + return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err) + } + + if bytes.Equal(account.CodeHash, emptyCodeHash) { + return "", "", nil, nil + } + + // Extract state leaf key + partialPath := trie.CompactToHex(nodeElements[0].([]byte)) + valueNodePath := append(node.Path, partialPath...) + encodedPath := trie.HexToCompact(valueNodePath) + leafKey := encodedPath[1:] + stateLeafKeyString := common.BytesToHash(leafKey).String() + + storageRootString := account.Root.String() + + // Extract codeHash and get code + codeHash := common.BytesToHash(account.CodeHash) + codeBytes := rawdb.ReadCode(ethDB, codeHash) + + return stateLeafKeyString, storageRootString, codeBytes, nil +} + +func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) { + nodePath := make([]byte, len(path)) + copy(nodePath, path) + + var nodeElements []interface{} + if err := rlp.DecodeBytes(node, &nodeElements); err != nil { + return sdtypes.StateNode{}, nil, err + } + + ty, err := sdtrie.CheckKeyType(nodeElements) + if err != nil { + return sdtypes.StateNode{}, nil, err + } + return sdtypes.StateNode{ + NodeType: ty, + Path: nodePath, + NodeValue: node, + }, nodeElements, nil +} diff --git a/pkg/eth/cid_retriever_test.go b/pkg/eth/cid_retriever_test.go index 29f9c6c8..5ab41827 100644 --- a/pkg/eth/cid_retriever_test.go +++ b/pkg/eth/cid_retriever_test.go @@ -300,6 +300,7 @@ var _ = Describe("Retriever", func() { AND header_cids.block_number = $1 ORDER BY transaction_cids.index` err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64()) + Expect(err).ToNot(HaveOccurred()) cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1) Expect(err).ToNot(HaveOccurred()) Expect(empty).ToNot(BeTrue()) diff --git a/pkg/eth/eth_state_test.go b/pkg/eth/eth_state_test.go index 2fe62e89..cd793c83 100644 --- a/pkg/eth/eth_state_test.go +++ b/pkg/eth/eth_state_test.go @@ -19,6 +19,7 @@ package eth_test import ( "bytes" "context" + "fmt" "io/ioutil" "math/big" "time" @@ -26,7 +27,6 @@ import ( "github.com/cerc-io/ipld-eth-server/v4/pkg/eth" "github.com/cerc-io/ipld-eth-server/v4/pkg/eth/test_helpers" "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" - ethServerShared "github.com/cerc-io/ipld-eth-server/v4/pkg/shared" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -44,6 +44,40 @@ import ( var ( parsedABI abi.ABI + + block1StateRoot = common.HexToHash("0xa1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21") + rootDataHashBlock1 = "a1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21" + rootDataBlock1 = "f871a0577652b625b77bdb5bf77bc43f3125cad7464d679d1575565277d3611b8053e780808080a0fe889f10e5db8f2c2bf355928152a17f6e3bb99a9241ac6d84c77e6264509c798080808080808080a011db0cda34a896dabeb6839bb06a38f49514cfa486435984eb013b7df9ee85c58080" + + block5StateRoot = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43") + rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43" + rootDataBlock5 = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080" + + account1DataHash = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4" + account1Data = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659" + account1StateLeafKey = "0x6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45" + account1Code = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032" + account2DataHash = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35" + account2Data = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account3DataHash = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2" + account3Data = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account4DataHash = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951" + account4Data = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + account5DataHash = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91" + account5Data = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + + contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0") + storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0" + storageRootDataBlock5 = "f838a120290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7" + + contractStorageRootBlock4 = common.HexToHash("0x64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68") + storageRootDataHashBlock4 = "64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68" + storageRootDataBlock4 = "f8518080a08e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d558080808080808080a07b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af8080808080" + + storageNode1DataHash = "7b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af" + storageNode1Data = "e2a0310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf609" + storageNode2DataHash = "8e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d55" + storageNode2Data = "f7a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7" ) func init() { @@ -81,8 +115,8 @@ var _ = Describe("eth state reading tests", func() { ChainConfig: chainConfig, VMConfig: vm.Config{}, RPCGasCap: big.NewInt(10000000000), // Max gas capacity for a rpc call. - GroupCacheConfig: ðServerShared.GroupCacheConfig{ - StateDB: ethServerShared.GroupConfig{ + GroupCacheConfig: &shared.GroupCacheConfig{ + StateDB: shared.GroupConfig{ Name: "eth_state_test", CacheSizeInMB: 8, CacheExpiryInMins: 60, @@ -529,4 +563,263 @@ var _ = Describe("eth state reading tests", func() { Expect(header).To(Equal(expectedCanonicalHeader)) }) }) + + Describe("eth_getSlice", func() { + It("Retrieves the state slice for root path", func() { + path := "0x" + depth := 3 + sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "1", + "02-total-trie-nodes": "6", + "03-leaves": "5", + "04-smart-contracts": "1", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + rootDataHashBlock5: rootDataBlock5, + }, + Slice: map[string]string{ + account1DataHash: account1Data, + account2DataHash: account2Data, + account3DataHash: account3Data, + account4DataHash: account4Data, + account5DataHash: account5Data, + }, + }, + Leaves: map[string]eth.GetSliceResponseAccount{ + account1StateLeafKey: { + StorageRoot: contractStorageRootBlock5.Hex(), + EVMCode: account1Code, + }, + }, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the state slice for root path with 0 depth", func() { + path := "0x" + depth := 0 + sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "0", + "04-smart-contracts": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + rootDataHashBlock5: rootDataBlock5, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the state slice for a path to an account", func() { + path := "0x06" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "2", + "01-max-depth": "0", + "02-total-trie-nodes": "2", + "03-leaves": "1", + "04-smart-contracts": "1", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{ + rootDataHashBlock5: rootDataBlock5, + }, + Head: map[string]string{ + account1DataHash: account1Data, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{ + account1StateLeafKey: { + StorageRoot: contractStorageRootBlock5.Hex(), + EVMCode: account1Code, + }, + }, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the state slice for a path to a non-existing account", func() { + path := "0x06" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, block1StateRoot, false) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block1StateRoot.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "0", + "04-smart-contracts": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{ + rootDataHashBlock1: rootDataBlock1, + }, + Head: map[string]string{}, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + + It("Retrieves the storage slice for root path", func() { + path := "0x" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "1", + "02-total-trie-nodes": "3", + "03-leaves": "2", + "04-smart-contracts": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + storageRootDataHashBlock4: storageRootDataBlock4, + }, + Slice: map[string]string{ + storageNode1DataHash: storageNode1Data, + storageNode2DataHash: storageNode2Data, + }, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the storage slice for root path with 0 depth", func() { + path := "0x" + depth := 0 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "0", + "04-smart-contracts": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + storageRootDataHashBlock4: storageRootDataBlock4, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the storage slice for root path with deleted nodes", func() { + path := "0x" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock5, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock5.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "1", + "01-max-depth": "0", + "02-total-trie-nodes": "1", + "03-leaves": "1", + "04-smart-contracts": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{}, + Head: map[string]string{ + storageRootDataHashBlock5: storageRootDataBlock5, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + It("Retrieves the storage slice for a path to a storage node", func() { + path := "0x0b" + depth := 2 + sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true) + Expect(err).ToNot(HaveOccurred()) + + expectedResponse := eth.GetSliceResponse{ + SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()), + MetaData: eth.GetSliceResponseMetadata{ + NodeStats: map[string]string{ + "00-stem-and-head-nodes": "2", + "01-max-depth": "0", + "02-total-trie-nodes": "2", + "03-leaves": "1", + "04-smart-contracts": "0", + }, + }, + TrieNodes: eth.GetSliceResponseTrieNodes{ + Stem: map[string]string{ + storageRootDataHashBlock4: storageRootDataBlock4, + }, + Head: map[string]string{ + storageNode1DataHash: storageNode1Data, + }, + Slice: map[string]string{}, + }, + Leaves: map[string]eth.GetSliceResponseAccount{}, + } + + eth.CheckGetSliceResponse(*sliceResponse, expectedResponse) + }) + }) }) diff --git a/pkg/eth/filterer.go b/pkg/eth/filterer.go index 8a17f446..d4cc936a 100644 --- a/pkg/eth/filterer.go +++ b/pkg/eth/filterer.go @@ -208,7 +208,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa } // If there are no wanted contract addresses, we keep all receipts that match the topic filter if len(wantedAddresses) == 0 { - if match := filterMatch(wantedTopics, actualTopics); match == true { + if match := filterMatch(wantedTopics, actualTopics); match { return true } } @@ -218,7 +218,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wa for _, actualAddr := range actualAddresses { if wantedAddr == actualAddr { // we keep the receipt if it matches on the topic filter - if match := filterMatch(wantedTopics, actualTopics); match == true { + if match := filterMatch(wantedTopics, actualTopics); match { return true } } @@ -240,10 +240,7 @@ func filterMatch(wantedTopics, actualTopics [][]string) bool { matches++ } } - if matches == 4 { - return true - } - return false + return matches == 4 } // returns 1 if the two slices have a string in common, 0 if they do not diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index f9a995fc..cce23579 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -17,6 +17,8 @@ package eth import ( + "time" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) @@ -34,3 +36,8 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType { return sdtypes.Unknown } } + +// Timestamp in milliseconds +func makeTimestamp() int64 { + return time.Now().UnixNano() / int64(time.Millisecond) +} diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index 2cbdcaea..3cfb6530 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -32,10 +32,6 @@ import ( ) const ( - // node type removed value. - // https://github.com/cerc-io/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34 - removedNode = 3 - RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids INNER JOIN public.blocks ON ( @@ -611,7 +607,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Addr return "", nil, err } - if accountResult.NodeType == removedNode { + if accountResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, nil } @@ -643,7 +639,7 @@ func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Ad return "", nil, err } - if accountResult.NodeType == removedNode { + if accountResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, nil } @@ -671,7 +667,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(add if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil { return "", nil, nil, err } - if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { + if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, EmptyNodeValue, nil } @@ -704,7 +700,7 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad return "", nil, err } - if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode { + if storageResult.StateLeafRemoved || storageResult.NodeType == sdtypes.Removed.Int() { return "", EmptyNodeValue, nil } diff --git a/pkg/eth/test_helpers.go b/pkg/eth/test_helpers.go index f59f4b43..f2eb1d76 100644 --- a/pkg/eth/test_helpers.go +++ b/pkg/eth/test_helpers.go @@ -18,6 +18,7 @@ package eth import ( "github.com/ethereum/go-ethereum/statediff/indexer/models" + . "github.com/onsi/gomega" ) // TxModelsContainsCID used to check if a list of TxModels contains a specific cid string @@ -39,3 +40,10 @@ func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool { } return false } + +func CheckGetSliceResponse(sliceResponse GetSliceResponse, expectedResponse GetSliceResponse) { + Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID)) + Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats)) + Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes)) + Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves)) +} diff --git a/pkg/eth/types.go b/pkg/eth/types.go index 5118eda2..a1f1e9c8 100644 --- a/pkg/eth/types.go +++ b/pkg/eth/types.go @@ -18,7 +18,9 @@ package eth import ( "errors" + "fmt" "math/big" + "strconv" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -264,3 +266,63 @@ type LogResult struct { TxnIndex int64 `db:"txn_index"` TxHash string `db:"tx_hash"` } + +// GetSliceResponse holds response for the eth_getSlice method +type GetSliceResponse struct { + SliceID string `json:"sliceId"` + MetaData GetSliceResponseMetadata `json:"metadata"` + TrieNodes GetSliceResponseTrieNodes `json:"trieNodes"` + Leaves map[string]GetSliceResponseAccount `json:"leaves"` // key: Keccak256Hash(address) in hex (leafKey) +} + +func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) { + sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String()) + sr.MetaData = GetSliceResponseMetadata{ + NodeStats: make(map[string]string, 0), + TimeStats: make(map[string]string, 0), + } + sr.Leaves = make(map[string]GetSliceResponseAccount) + sr.TrieNodes = GetSliceResponseTrieNodes{ + Stem: make(map[string]string), + Head: make(map[string]string), + Slice: make(map[string]string), + } +} + +func (sr *GetSliceResponse) populateMetaData(metaData metaDataFields) { + sr.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head)) + sr.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(metaData.maxDepth) + sr.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head) + len(sr.TrieNodes.Slice)) + sr.MetaData.NodeStats["03-leaves"] = strconv.Itoa(metaData.leafCount) + sr.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(sr.Leaves)) + + sr.MetaData.TimeStats["00-trie-loading"] = strconv.FormatInt(metaData.trieLoadingTime, 10) + sr.MetaData.TimeStats["01-fetch-stem-keys"] = strconv.FormatInt(metaData.stemNodesFetchTime, 10) + sr.MetaData.TimeStats["02-fetch-slice-keys"] = strconv.FormatInt(metaData.sliceNodesFetchTime, 10) + sr.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.FormatInt(metaData.leavesFetchTime, 10) +} + +type GetSliceResponseMetadata struct { + TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one) + NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts +} + +type GetSliceResponseTrieNodes struct { + Stem map[string]string `json:"stem"` // key: Keccak256Hash(data) in hex, value: trie node data in hex + Head map[string]string `json:"head"` + Slice map[string]string `json:"sliceNodes"` +} + +type GetSliceResponseAccount struct { + StorageRoot string `json:"storageRoot"` + EVMCode string `json:"evmCode"` +} + +type metaDataFields struct { + maxDepth int + leafCount int + trieLoadingTime int64 + stemNodesFetchTime int64 + sliceNodesFetchTime int64 + leavesFetchTime int64 +}