diff --git a/pkg/eth/backend.go b/pkg/eth/backend.go index 8d1ed56c..817b677e 100644 --- a/pkg/eth/backend.go +++ b/pkg/eth/backend.go @@ -102,6 +102,14 @@ const ( WHERE state_root = $1 ORDER BY block_number DESC LIMIT 1` + RetrieveBlockNumberAndStateLeafKeyForStorageRoot = `SELECT block_number, state_leaf_key + FROM eth.state_cids, eth.state_accounts + WHERE state_accounts.storage_root = $1 + AND state_cids.state_id = state_accounts.state_path + AND state_cids.storage_root = state_accounts.header_id + AND state_cids.storage_root = state_accounts.block_number + ORDER BY block_number DESC + LIMIT 1` ) const ( @@ -907,6 +915,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS return nil, fmt.Errorf("GetStateSlice blockheight lookup error: %s", err.Error()) } + // Get all the paths headPath, stemPaths, slicePaths, err := getPaths(path, depth) if err != nil { return nil, fmt.Errorf("GetStateSlice path generation error: %s", err.Error()) @@ -931,7 +940,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS // Fetch stem nodes // some of the "stem" nodes can be leaf nodes (but not value nodes) - stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, stemPaths) + stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, stemPaths, blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice stem node lookup error: %s", err.Error()) } @@ -939,7 +948,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent // Fetch slice nodes - sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, slicePaths) + sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStateNodesByPathsAndBlockNumber(tx, slicePaths, blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice slice node lookup error: %s", err.Error()) } @@ -947,7 +956,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent // Fetch head node - headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, blockHeight, [][]byte{headPath}) + headNode, headLeafCID, _, _, err := b.getStateNodesByPathsAndBlockNumber(tx, [][]byte{headPath}, blockHeight) if err != nil { return nil, fmt.Errorf("GetStateSlice head node lookup error: %s", err.Error()) } @@ -960,6 +969,7 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS leafNodes = append(leafNodes, sliceLeafCIDs...) leafNodes = append(leafNodes, headLeafCID...) // TODO: fill in contract data `response.Leaves` + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) maxDepth := deepestPath - len(headPath) if maxDepth < 0 { @@ -970,15 +980,84 @@ func (b *Backend) GetStateSlice(path string, depth int, root common.Hash) (*GetS response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(leafNodes)) response.MetaData.NodeStats["04-smart-contracts"] = "" // TODO: count # of contracts - response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(int(makeTimestamp() - leafFetchStart)) response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) return response, nil } func (b *Backend) GetStorageSlice(path string, depth int, root common.Hash) (*GetSliceResponse, error) { - // TODO Implement - return nil, nil + response := new(GetSliceResponse) + response.init(path, depth, root) + + // Start a timer + trieLoadingStart := makeTimestamp() + + // Get the block height and state leaf key for the input storage root + blockHeight, stateLeafKey, err := b.getBlockHeightAndStateLeafKeyForStorageRoot(root) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice blockheight and state key lookup error: %s", err.Error()) + } + + // Get all the paths + headPath, stemPaths, slicePaths, err := getPaths(path, depth) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice path generation error: %s", err.Error()) + } + response.MetaData.TimeStats["00-trie-loading"] = strconv.Itoa(int(makeTimestamp() - trieLoadingStart)) + + // Begin tx + tx, err := b.DB.Beginx() + if err != nil { + return nil, err + } + defer func() { + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else if err != nil { + shared.Rollback(tx) + } else { + err = tx.Commit() + } + }() + + // Fetch stem nodes + // some of the "stem" nodes can be leaf nodes (but not value nodes) + stemNodes, stemLeafCIDs, _, timeSpent, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, stemPaths, blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice stem node lookup error: %s", err.Error()) + } + response.TrieNodes.Stem = stemNodes + response.MetaData.TimeStats["01-fetch-stem-keys"] = timeSpent + + // Fetch slice nodes + sliceNodes, sliceLeafCIDs, deepestPath, timeSpent, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, slicePaths, blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice slice node lookup error: %s", err.Error()) + } + response.TrieNodes.Slice = sliceNodes + response.MetaData.TimeStats["02-fetch-slice-keys"] = timeSpent + + // Fetch head node + headNode, headLeafCID, _, _, err := b.getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, [][]byte{headPath}, blockHeight) + if err != nil { + return nil, fmt.Errorf("GetStorageSlice head node lookup error: %s", err.Error()) + } + response.TrieNodes.Head = headNode + + // Fill in metadata + maxDepth := deepestPath - len(headPath) + if maxDepth < 0 { + maxDepth = 0 + } + response.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(maxDepth) + + response.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + len(response.TrieNodes.Slice) + 1) + response.MetaData.NodeStats["03-leaves"] = strconv.Itoa(len(stemLeafCIDs) + len(sliceLeafCIDs) + len(headLeafCID)) + response.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(response.TrieNodes.Stem) + 1) + response.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.Itoa(0) + + return response, nil } func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { @@ -986,7 +1065,16 @@ func (b *Backend) getBlockHeightForStateRoot(root common.Hash) (uint64, error) { return blockHeight, b.DB.Get(&blockHeight, RetrieveBlockNumberForStateRoot, root.String()) } -func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight uint64, paths [][]byte) (map[string]string, []cid.Cid, int, string, error) { +func (b *Backend) getBlockHeightAndStateLeafKeyForStorageRoot(root common.Hash) (uint64, string, error) { + var res struct { + BlockNumber uint64 `db:"block_number"` + StateLeafKey string `db:"state_leaf_key"` + } + + return res.BlockNumber, res.StateLeafKey, b.DB.Get(&res, RetrieveBlockNumberAndStateLeafKeyForStorageRoot, root.String()) +} + +func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { nodes := make(map[string]string) fetchStart := makeTimestamp() @@ -1011,6 +1099,31 @@ func (b *Backend) getStateNodesByPathsAndBlockNumber(tx *sqlx.Tx, blockHeight ui return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil } +func (b *Backend) getStorageNodesByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, blockHeight uint64) (map[string]string, []cid.Cid, int, string, error) { + nodes := make(map[string]string) + fetchStart := makeTimestamp() + + // Get CIDs for all nodes at the provided paths + leafCIDs, leafIPLDs, intermediateCIDs, intermediateIPLDs, deepestPath, err := b.IPLDRetriever.RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx, stateLeafKey, paths, blockHeight) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for leaf nodes + err = populateNodesMap(nodes, leafCIDs, leafIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + // Populate the nodes map for intermediate nodes + err = populateNodesMap(nodes, intermediateCIDs, intermediateIPLDs) + if err != nil { + return nil, nil, 0, "", err + } + + return nodes, leafCIDs, deepestPath, strconv.Itoa(int(makeTimestamp() - fetchStart)), nil +} + // Engine satisfied the ChainContext interface func (b *Backend) Engine() consensus.Engine { // TODO: we need to support more than just ethash based engines diff --git a/pkg/eth/helpers.go b/pkg/eth/helpers.go index 6caded31..31783bb6 100644 --- a/pkg/eth/helpers.go +++ b/pkg/eth/helpers.go @@ -108,7 +108,7 @@ func populateNodesMap(nodes map[string]string, cids []cid.Cid, iplds [][]byte) e data := iplds[i] hash := crypto.Keccak256Hash(data) if !bytes.Equal(hash.Bytes(), decodedMh.Digest) { - panic("multihash digest should equal keccak of raw data") + return fmt.Errorf("multihash digest should equal keccak of raw data") } nodes[common.Bytes2Hex(decodedMh.Digest)] = common.Bytes2Hex(data) diff --git a/pkg/eth/ipld_retriever.go b/pkg/eth/ipld_retriever.go index fe970a82..24a628c3 100644 --- a/pkg/eth/ipld_retriever.go +++ b/pkg/eth/ipld_retriever.go @@ -239,7 +239,7 @@ const ( ) WHERE tx_hash = $1 AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))` - RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.mh_key, state_cids.node_type + RetrieveStateByPathAndBlockNumberPgStr = `SELECT state_cids.cid, data FROM eth.state_cids INNER JOIN public.blocks ON ( state_cids.mh_key = blocks.key @@ -250,6 +250,23 @@ const ( AND node_type != 3 ORDER BY state_cids.block_number DESC LIMIT 1` + RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr = `SELECT storage_cids.cid, data + FROM eth.storage_cids + INNER JOIN eth.state_cids ON ( + storage_cids.state_path = state_cids.state_path + AND storage_cids.header_id = state_cids.header_id + AND storage_cids.block_number = state_cids.block_number + ) + INNER JOIN public.blocks ON ( + storage_cids.mh_key = blocks.key + AND storage_cids.block_number = blocks.block_number + ) + WHERE state_leaf_key = $1 + AND storage_path = $2 + AND storage_cids.block_number <= $3 + AND node_type != 3 + ORDER BY storage_cids.block_number DESC + LIMIT 1` RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, state_cids.mh_key, state_cids.block_number, state_cids.node_type FROM eth.state_cids INNER JOIN eth.header_cids ON ( @@ -746,10 +763,51 @@ func (r *IPLDRetriever) RetrieveStatesByPathsAndBlockNumber(tx *sqlx.Tx, paths [ leafNodeIPLDs := make([][]byte, 0) intermediateNodeIPLDs := make([][]byte, 0) + // TODO: fetch all nodes in a single query for _, path := range paths { - // Create a result object, select: cid, blockNumber and nodeType + // Create a result object, select: cid, data res := new(nodeInfo) - if err := r.db.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + if err := tx.Get(res, RetrieveStateByPathAndBlockNumberPgStr, path, number); err != nil { + return nil, nil, nil, nil, 0, err + } + + pathLen := len(path) + if pathLen > deepestPath { + deepestPath = pathLen + } + + cid, err := cid.Decode(res.CID) + if err != nil { + return nil, nil, nil, nil, 0, err + } + + if res.NodeType == sdtypes.Leaf.Int() { + leafNodeCIDs = append(leafNodeCIDs, cid) + leafNodeIPLDs = append(leafNodeIPLDs, res.Data) + } else { + intermediateNodeCIDs = append(intermediateNodeCIDs, cid) + intermediateNodeIPLDs = append(intermediateNodeIPLDs, res.Data) + } + } + + return leafNodeCIDs, leafNodeIPLDs, intermediateNodeCIDs, intermediateNodeIPLDs, deepestPath, nil +} + +// RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber returns the cid and rlp bytes for the storage nodes corresponding to the provided state leaf key, storage paths and block number +func (r *IPLDRetriever) RetrieveStorageByStateLeafKeyAndPathsAndBlockNumber(tx *sqlx.Tx, stateLeafKey string, paths [][]byte, number uint64) ([]cid.Cid, [][]byte, []cid.Cid, [][]byte, int, error) { + deepestPath := 0 + + leafNodeCIDs := make([]cid.Cid, 0) + intermediateNodeCIDs := make([]cid.Cid, 0) + + leafNodeIPLDs := make([][]byte, 0) + intermediateNodeIPLDs := make([][]byte, 0) + + // TODO: fetch all nodes in a single query + for _, path := range paths { + // Create a result object, select: cid, data + res := new(nodeInfo) + if err := tx.Get(res, RetrieveStorageByStateLeafKeyAndPathAndBlockNumberPgStr, stateLeafKey, path, number); err != nil { return nil, nil, nil, nil, 0, err }