cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Implement getSlice API #206

Merged
prathamesh0 merged 21 commits from pm-get-slice into v4 2022-12-19 08:42:23 +00:00
Conversation 1 Commits 21 Files Changed 12 +308 -1
6 changed files with 308 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a2a391c459 - Show all commits

8
pkg/eth/api.go
View File

@ -837,6 +837,14 @@ func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Addre
}, state.Error()
}
// GetSlice returns a slice of state or storage nodes from a provided root to a provided path and past it to a certain depth
func (pea *PublicEthAPI) GetSlice(ctx context.Context, path string, depth int, root common.Hash, storage bool) (*GetSliceResponse, error) {
if storage {
return pea.B.GetStorageSlice(path, depth, root)
}
return pea.B.GetStateSlice(path, depth, root)
}
// revertError is an API error that encompassas an EVM revertal with JSON error
// code and a binary data blob.
type revertError struct {

4
pkg/eth/api_test.go
View File

@ -1145,4 +1145,8 @@ var _ = Describe("API", func() {
Expect(code).To(BeEmpty())
})
})
Describe("eth_getSlice", func() {
// TODO Implement
})
})

125
pkg/eth/backend.go
View File

@ -43,6 +43,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-cid"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
@ -95,7 +96,12 @@ const (
AND header_cids.block_hash = (SELECT canonical_header_hash(header_cids.block_number))
ORDER BY header_cids.block_number DESC
LIMIT 1`
RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
RetrieveBlockNumberForStateRoot = `SELECT block_number
FROM eth.header_cids
WHERE state_root = $1
ORDER BY block_number DESC
LIMIT 1`
)
const (
@ -888,6 +894,123 @@ func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address,
return storageRlp, err
}
func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) {
response := new(GetSliceResponse)
response.init(path, depth, root)
// Start a timer
trieLoadingStart := makeTimestamp()
// Get the block height for the input state root
blockHeight, err := b.getBlockHeightForStateRoot(root)
if err != nil {
return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error())
}
headPath, stemPaths, slicePaths, err := getPaths(path, depth)
if err != nil {
return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error())
}
response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart))
// Begin tx
tx, err := b.DB.Beginx()
if err != nil {
return nil, err
}
defer func() {
if p := recover(); p != nil {
shared.Rollback(tx)
panic(p)
} else if err != nil {
shared.Rollback(tx)
} else {
err = tx.Commit()
}
}()
// Fetch stem nodes
// some of the "stem" nodes can be leaf nodes (but not value nodes)
stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, stemPaths)
if err != nil {
return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error())
}
response.TrieNodes.Stem = stemNodes
response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent
// Fetch slice nodes
sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, slicePaths)
if err != nil {
return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error())
}
response.TrieNodes.Slice = sliceNodes
response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent
// Fetch head node
headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, [][]byte{headPath})
if err != nil {
return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error())
}
response.TrieNodes.Head = headNode
// Fetch leaf contract data and fill in remaining metadata
leafFetchStart := makeTimestamp()
leafNodes := make([]cid.Cid, 0, len(stemLeafCIDs)+len(sliceLeafCIDs)+len(headLeafCID))
leafNodes = append(leafNodes, stemLeafCIDs...)
leafNodes = append(leafNodes, sliceLeafCIDs...)
leafNodes = append(leafNodes, headLeafCID...)
// TODO: fill in contract data `response.Leaves`
maxDepth := deepestPath - len(headPath)
if maxDepth < 0 {
maxDepth = 0
}
response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth)
response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1)
response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes))
response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts
response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart))
response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1)
return response, nil
}
func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) {
// TODO Implement
return nil, nil
}
func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) {
var blockHeight uint64
return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String())
}
func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight uint64, paths [][]byte) (map[string]string, []cid.Cid, int, string, error) {
nodes := make(map[string]string)
fetchStart := makeTimestamp()
// Get CIDs for all nodes at the provided paths
leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStatesByPathsAndBlockNumber(tx, paths, blockHeight)
if err != nil {
return nil, nil, 0, "", err
}
// Populate the nodes map for leaf nodes
err = populateNodesMap(nodes, leafCIDs, leafIPLDs)
if err != nil {
return nil, nil, 0, "", err
}
// Populate the nodes map for intermediate nodes
err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs)
if err != nil {
return nil, nil, 0, "", err
}
return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil
}
// Engine satisfied the ChainContext interface
func (b *Backend) Engine() consensus.Engine {
// TODO: we need to support more than just ethash based engines

82
pkg/eth/helpers.go
View File

@ -17,7 +17,16 @@
package eth
import (
"bytes"
"fmt"
"math"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
)
func ResolveToNodeType(nodeType int) sdtypes.NodeType {
@ -34,3 +43,76 @@ func ResolveToNodeType(nodeType int) sdtypes.NodeType {
return sdtypes.Unknown
}
}
var pathSteps = []byte{'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f'}
// Return head, stem, and slice byte paths for the given head path and depth
func getPaths(path string, depth int) ([]byte, [][]byte, [][]byte, error) {
// Convert the head hex path to a decoded byte path
headPath := common.FromHex(path)
pathLen := len(headPath)
if pathLen > 64 { // max path len is 64
return nil, nil, nil, fmt.Errorf("path length cannot exceed 64; got %d", pathLen)
}
maxDepth := 64 - pathLen
if depth > maxDepth {
return nil, nil, nil, fmt.Errorf("max depth for path %s is %d; got %d", path, maxDepth, depth)
}
// Collect all of the stem paths
stemPaths := make([][]byte, 0, pathLen)
for i := 0; i < pathLen; i++ {
stemPaths = append(stemPaths, headPath[:i])
}
// Generate all of the slice paths
slicePaths := make([][]byte, 0, int(math.Pow(16, float64(depth))))
makeSlicePaths(headPath, depth, &slicePaths)
return headPath, stemPaths, slicePaths, nil
}
// iterative function to generate the set of slice paths
func makeSlicePaths(path []byte, depth int, slicePaths *[][]byte) {
depth-- // decrement the depth
nextPaths := make([][]byte, 16) // slice to hold the next 16 paths
for i, step := range pathSteps { // iterate through steps
nextPath := append(path, step) // create next paths by adding steps to current path
nextPaths[i] = nextPath
newSlicePaths := append(*slicePaths, nextPath) // add next paths to the collection of all slice paths
slicePaths = &newSlicePaths
}
if depth == 0 { // if depth has reach 0, return
return
}
for _, nextPath := range nextPaths { // if not, then we iterate over the next paths
makeSlicePaths(nextPath, depth, slicePaths) // and repeat the process for each one
}
}
// use to return timestamp in milliseconds
func makeTimestamp() int64 {
return time.Now().UnixNano() / int64(time.Millisecond)
}
func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) error {
for i, cid := range cids {
decodedMh, err := multihash.Decode(cid.Hash())
if err != nil {
return err
}
data := iplds[i]
hash := crypto.Keccak256Hash(data)
if !bytes.Equal(hash.Bytes(), decodedMh.Digest) {
panic("multihash digest should equal keccak of raw data")
}
nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data)
}
return nil
}

51
pkg/eth/ipld_retriever.go
View File

@ -23,6 +23,7 @@ import (
"github.com/cerc-io/ipld-eth-server/v4/pkg/shared"
"github.com/ethereum/go-ethereum/statediff/trie_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ipfs/go-cid"
"github.com/jmoiron/sqlx"
"github.com/ethereum/go-ethereum/common"
@ -238,6 +239,17 @@ const (
)
WHERE tx_hash = $1
AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))`
RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.mh_key, state_cids.node_type
FROM eth.state_cids
INNER JOIN public.blocks ON (
state_cids.mh_key = blocks.key
AND state_cids.block_number = blocks.block_number
)
WHERE state_path = $1
AND state_cids.block_number <= $2
AND node_type != 3
ORDER BY state_cids.block_number DESC
LIMIT 1`
RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type
FROM eth.state_cids
INNER JOIN eth.header_cids ON (
@ -723,3 +735,42 @@ func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(ad
}
return storageResult.CID, i[1].([]byte), nil
}
// RetrieveStatesByPathsAndBlockNumber returns the cid and rlp bytes for the state nodes corresponding to the provided state paths and block number
func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) {
deepestPath := 0
leafNodeCIDs := make([]cid.Cid, 0)
intermediateNodeCIDs := make([]cid.Cid, 0)
leafNodeIPLDs := make([][]byte, 0)
intermediateNodeIPLDs := make([][]byte, 0)
for _, path := range paths {
// Create a result object, select: cid, blockNumber and nodeType
res := new(nodeInfo)
if err := r.db.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil {
return nil, nil, nil, nil, 0, err
}
pathLen := len(path)
if pathLen > deepestPath {
deepestPath = pathLen
}
cid, err := cid.Decode(res.CID)
if err != nil {
return nil, nil, nil, nil, 0, err
}
if res.NodeType == sdtypes.Leaf.Int() {
leafNodeCIDs = append(leafNodeCIDs, cid)
leafNodeIPLDs = append(leafNodeIPLDs, res.Data)
} else {
intermediateNodeCIDs = append(intermediateNodeCIDs, cid)
intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data)
}
}
return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil
}

39
pkg/eth/types.go
View File

@ -18,6 +18,7 @@ package eth
import (
"errors"
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common"
@ -264,3 +265,41 @@ type LogResult struct {
TxnIndex int64 `db:"txn_index"`
TxHash string `db:"tx_hash"`
}
// GetSliceResponse holds response for the eth_getSlice method
type GetSliceResponse struct {
SliceID string `json:"sliceId"`
MetaData GetSliceResponseMetadata `json:"metadata"`
TrieNodes GetSliceResponseTrieNodes `json:"trieNodes"`
Leaves map[string]GetSliceResponseAccount `json:"leaves"` // we won't be using addresses, but keccak256(address) // TODO: address comment
}
func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) {
sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String())
sr.MetaData = GetSliceResponseMetadata{
NodeStats: make(map[string]string, 0),
TimeStats: make(map[string]string, 0),
}
sr.Leaves = make(map[string]GetSliceResponseAccount)
sr.TrieNodes = GetSliceResponseTrieNodes{
Stem: make(map[string]string),
Head: make(map[string]string),
Slice: make(map[string]string),
}
}
type GetSliceResponseMetadata struct {
TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one)
NodeStats map[string]string `json:"trieNodes"` // total, leaves, smart contracts
}
type GetSliceResponseTrieNodes struct {
Stem map[string]string `json:"stem"`
Head map[string]string `json:"head"`
Slice map[string]string `json:"sliceNodes"`
}
type GetSliceResponseAccount struct {
StorageRoot string `json:"storageRoot"`
EVMCode string `json:"evmCode"`
}