cerc-io/go-ethereum
14
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

update statediff builder to process internalized leaf node values (storage trie)

Browse Source
This commit is contained in:
i-norden 2023-02-16 15:49:18 -06:00
parent 1e574d4df8
commit ae5ab3d9b0
1 changed files with 217 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

657
statediff/builder.go
View File

@ -23,6 +23,8 @@ import (
"bytes" "bytes"
"fmt" "fmt"
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
@ -44,7 +46,7 @@ var (
// Builder interface exposes the method for building a state diff between two blocks // Builder interface exposes the method for building a state diff between two blocks
type Builder interface { type Builder interface {
BuildStateDiffObject(args Args, params Params) (types2.StateObject, error) BuildStateDiffObject(args Args, params Params) (types2.StateObject, error)
WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink, ipldOutput types2.IPLDSink) error
} }
type StateDiffBuilder struct { type StateDiffBuilder struct {
@ -55,21 +57,20 @@ type IterPair struct {
Older, Newer trie.NodeIterator Older, Newer trie.NodeIterator
} }
// convenience func StateNodeAppender(nodes *[]types2.StateLeafNode) types2.StateNodeSink {
func StateNodeAppender(nodes *[]types2.StateNode) types2.StateNodeSink { return func(node types2.StateLeafNode) error {
return func(node types2.StateNode) error {
*nodes = append(*nodes, node) *nodes = append(*nodes, node)
return nil return nil
} }
} }
func StorageNodeAppender(nodes *[]types2.StorageNode) types2.StorageNodeSink { func StorageNodeAppender(nodes *[]types2.StorageLeafNode) types2.StorageNodeSink {
return func(node types2.StorageNode) error { return func(node types2.StorageLeafNode) error {
*nodes = append(*nodes, node) *nodes = append(*nodes, node)
return nil return nil
} }
} }
func CodeMappingAppender(codeAndCodeHashes *[]types2.CodeAndCodeHash) types2.CodeSink { func IPLDMappingAppender(codeAndCodeHashes *[]types2.IPLD) types2.IPLDSink {
return func(c types2.CodeAndCodeHash) error { return func(c types2.IPLD) error {
*codeAndCodeHashes = append(*codeAndCodeHashes, c) *codeAndCodeHashes = append(*codeAndCodeHashes, c)
return nil return nil
} }
@ -84,24 +85,25 @@ func NewBuilder(stateCache state.Database) Builder {
// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters // BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (types2.StateObject, error) { func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (types2.StateObject, error) {
var stateNodes []types2.StateNode var stateNodes []types2.StateLeafNode
var codeAndCodeHashes []types2.CodeAndCodeHash var iplds []types2.IPLD
err := sdb.WriteStateDiffObject( err := sdb.WriteStateDiffObject(
types2.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot}, types2.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
params, StateNodeAppender(&stateNodes), CodeMappingAppender(&codeAndCodeHashes)) params, StateNodeAppender(&stateNodes), IPLDMappingAppender(&iplds))
if err != nil { if err != nil {
return types2.StateObject{}, err return types2.StateObject{}, err
} }
return types2.StateObject{ return types2.StateObject{
BlockHash: args.BlockHash, BlockHash: args.BlockHash,
BlockNumber: args.BlockNumber, BlockNumber: args.BlockNumber,
Nodes: stateNodes, Nodes: stateNodes,
CodeAndCodeHashes: codeAndCodeHashes, IPLDs: iplds,
}, nil }, nil
} }
// WriteStateDiffObject writes a statediff object to output callback // WriteStateDiffObject writes a statediff object to output callback
func (sdb *StateDiffBuilder) WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error { func (sdb *StateDiffBuilder) WriteStateDiffObject(args types2.StateRoots, params Params, output types2.StateNodeSink,
ipldOutput types2.IPLDSink) error {
// Load tries for old and new states // Load tries for old and new states
oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot) oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
if err != nil { if err != nil {
@ -127,19 +129,16 @@ func (sdb *StateDiffBuilder) WriteStateDiffObject(args types2.StateRoots, params
}, },
} }
if !params.IntermediateStateNodes { return sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs, params, output, ipldOutput)
return sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs, params, output, codeOutput)
} else {
return sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs, params, output, codeOutput)
}
} }
func (sdb *StateDiffBuilder) BuildStateDiffWithIntermediateStateNodes(iterPairs []IterPair, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error { func (sdb *StateDiffBuilder) BuildStateDiffWithIntermediateStateNodes(iterPairs []IterPair, params Params,
output types2.StateNodeSink, ipldOutput types2.IPLDSink) error {
// collect a slice of all the nodes that were touched and exist at B (B-A) // collect a slice of all the nodes that were touched and exist at B (B-A)
// a map of their leafkey to all the accounts that were touched and exist at B // a map of their leafkey to all the accounts that were touched and exist at B
// and a slice of all the paths for the nodes in both of the above sets // and a slice of all the paths for the nodes in both of the above sets
diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes( diffAccountsAtB, err := sdb.createdAndUpdatedState(
iterPairs[0].Older, iterPairs[0].Newer, params.watchedAddressesLeafPaths, output) iterPairs[0].Older, iterPairs[0].Newer, params.watchedAddressesLeafPaths, ipldOutput)
if err != nil { if err != nil {
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
} }
@ -147,9 +146,8 @@ func (sdb *StateDiffBuilder) BuildStateDiffWithIntermediateStateNodes(iterPairs
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B // collect a slice of all the nodes that existed at a path in A that doesn't exist in B
// a map of their leafkey to all the accounts that were touched and exist at A // a map of their leafkey to all the accounts that were touched and exist at A
diffAccountsAtA, err := sdb.deletedOrUpdatedState( diffAccountsAtA, err := sdb.deletedOrUpdatedState(
iterPairs[1].Older, iterPairs[1].Newer, iterPairs[1].Older, iterPairs[1].Newer, diffAccountsAtB,
diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafPaths, params.watchedAddressesLeafPaths, output)
params.IntermediateStateNodes, params.IntermediateStorageNodes, output)
if err != nil { if err != nil {
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
} }
@ -165,59 +163,12 @@ func (sdb *StateDiffBuilder) BuildStateDiffWithIntermediateStateNodes(iterPairs
updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys) updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys)
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
err = sdb.buildAccountUpdates( err = sdb.buildAccountUpdates(diffAccountsAtB, diffAccountsAtA, updatedKeys, output, ipldOutput)
diffAccountsAtB, diffAccountsAtA, updatedKeys,
params.IntermediateStorageNodes, output)
if err != nil { if err != nil {
return fmt.Errorf("error building diff for updated accounts: %v", err) return fmt.Errorf("error building diff for updated accounts: %v", err)
} }
// build the diff nodes for created accounts // build the diff nodes for created accounts
err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput) err = sdb.buildAccountCreations(diffAccountsAtB, output, ipldOutput)
if err != nil {
return fmt.Errorf("error building diff for created accounts: %v", err)
}
return nil
}
func (sdb *StateDiffBuilder) BuildStateDiffWithoutIntermediateStateNodes(iterPairs []IterPair, params Params, output types2.StateNodeSink, codeOutput types2.CodeSink) error {
// collect a map of their leafkey to all the accounts that were touched and exist at B
// and a slice of all the paths for the nodes in both of the above sets
diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState(
iterPairs[0].Older, iterPairs[0].Newer,
params.watchedAddressesLeafPaths)
if err != nil {
return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
}
// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
// a map of their leafkey to all the accounts that were touched and exist at A
diffAccountsAtA, err := sdb.deletedOrUpdatedState(
iterPairs[1].Older, iterPairs[1].Newer,
diffAccountsAtB, diffPathsAtB, params.watchedAddressesLeafPaths,
params.IntermediateStateNodes, params.IntermediateStorageNodes, output)
if err != nil {
return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
}
// collect and sort the leafkeys for both account mappings into a slice
createKeys := trie_helpers.SortKeys(diffAccountsAtB)
deleteKeys := trie_helpers.SortKeys(diffAccountsAtA)
// and then find the intersection of these keys
// these are the leafkeys for the accounts which exist at both A and B but are different
// this also mutates the passed in createKeys and deleteKeys, removing in intersection keys
// and leaving the truly created or deleted keys in place
updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys)
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
err = sdb.buildAccountUpdates(
diffAccountsAtB, diffAccountsAtA, updatedKeys,
params.IntermediateStorageNodes, output)
if err != nil {
return fmt.Errorf("error building diff for updated accounts: %v", err)
}
// build the diff nodes for created accounts
err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput)
if err != nil { if err != nil {
return fmt.Errorf("error building diff for created accounts: %v", err) return fmt.Errorf("error building diff for created accounts: %v", err)
} }
@ -225,73 +176,11 @@ func (sdb *StateDiffBuilder) BuildStateDiffWithoutIntermediateStateNodes(iterPai
} }
// createdAndUpdatedState returns // createdAndUpdatedState returns
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
// and a slice of the paths for all of the nodes included in both
func (sdb *StateDiffBuilder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddressesLeafPaths [][]byte) (types2.AccountMap, map[string]bool, error) {
diffPathsAtB := make(map[string]bool)
diffAccountsAtB := make(types2.AccountMap)
watchingAddresses := len(watchedAddressesLeafPaths) > 0
it, _ := trie.NewDifferenceIterator(a, b)
for it.Next(true) {
// ignore node if it is not along paths of interest
if watchingAddresses && !isValidPrefixPath(watchedAddressesLeafPaths, it.Path()) {
continue
}
// skip null nodes
if bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue
}
nodePath := make([]byte, len(it.Path()))
copy(nodePath, it.Path())
// if it is a value node, we index the value by leaf key
if it.Leaf() {
// ignore leaf node if it is not a watched address
if !isWatchedAddress(watchedAddressesLeafPaths, nodePath) {
continue
}
// created vs updated is important for leaf nodes since we need to diff their storage
// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
var account types.StateAccount
accountRLP := make([]byte, 0)
copy(accountRLP, it.LeafBlob())
if err := rlp.DecodeBytes(accountRLP, &account); err != nil {
return nil, nil, fmt.Errorf("error decoding account for leaf value at leaf key %x\nerror: %v", it.LeafKey(), err)
}
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
parentNodeRLP, err := sdb.StateCache.TrieDB().Node(it.Parent())
if err != nil {
return nil, nil, err
}
leafNodeHash := crypto.Keccak256(parentNodeRLP)
diffAccountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{
Removed: false,
Path: nodePath,
LeafKey: leafKey,
Account: &account,
LeafNodeHash: leafNodeHash,
}
} else {
// add non-value-node paths to the list of diffPathsAtB
diffPathsAtB[common.Bytes2Hex(nodePath)] = true
}
}
return diffAccountsAtB, diffPathsAtB, it.Error()
}
// createdAndUpdatedStateWithIntermediateNodes returns
// a slice of all the intermediate nodes that exist in a different state at B than A // a slice of all the intermediate nodes that exist in a different state at B than A
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A // a mapping of their leafkeys to all the accounts that exist in a different state at B than A
// and a slice of the paths for all of the nodes included in both // and a slice of the paths for all of the nodes included in both
func (sdb *StateDiffBuilder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, watchedAddressesLeafPaths [][]byte, output types2.StateNodeSink) (types2.AccountMap, map[string]bool, error) { func (sdb *StateDiffBuilder) createdAndUpdatedState(a, b trie.NodeIterator,
diffPathsAtB := make(map[string]bool) watchedAddressesLeafPaths [][]byte, output types2.IPLDSink) (types2.AccountMap, error) {
diffAccountsAtB := make(types2.AccountMap) diffAccountsAtB := make(types2.AccountMap)
watchingAddresses := len(watchedAddressesLeafPaths) > 0 watchingAddresses := len(watchedAddressesLeafPaths) > 0
@ -307,60 +196,72 @@ func (sdb *StateDiffBuilder) createdAndUpdatedStateWithIntermediateNodes(a, b tr
continue continue
} }
nodePath := make([]byte, len(it.Path())) // index values by leaf key
copy(nodePath, it.Path())
// index value nodes by leaf key
if it.Leaf() { if it.Leaf() {
// ignore leaf node if it is not a watched address // if it is a "value" node, we will index the value by leaf key
if !isWatchedAddress(watchedAddressesLeafPaths, nodePath) { accountW, err := sdb.processStateValueNode(it, watchedAddressesLeafPaths)
continue
}
// created vs updated is important for leaf nodes since we need to diff their storage
// so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
var account types.StateAccount
accountRLP := make([]byte, 0)
copy(accountRLP, it.LeafBlob())
if err := rlp.DecodeBytes(accountRLP, &account); err != nil {
return nil, nil, fmt.Errorf("error decoding account for leaf node at key %x\nerror: %v", it.LeafKey(), err)
}
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
parentNodeRLP, err := sdb.StateCache.TrieDB().Node(it.Parent())
if err != nil { if err != nil {
return nil, nil, err return nil, err
}
leafNodeHash := crypto.Keccak256(parentNodeRLP)
diffAccountsAtB[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{
Removed: false,
Path: nodePath,
LeafKey: leafKey,
Account: &account,
LeafNodeHash: leafNodeHash,
} }
// for now, just add it to diffAccountsAtB
// we will compare to diffAccountsAtA to determine which diffAccountsAtB
// were creations and which were updates and also identify accounts that were removed going A->B
diffAccountsAtB[common.Bytes2Hex(accountW.LeafKey)] = accountW
} else { // trie nodes will be written to blockstore only } else { // trie nodes will be written to blockstore only
// reminder that this includes leaf nodes, since the geth iteratio.Leaf() actually signifies a "value" node
nodeVal := make([]byte, len(it.NodeBlob())) nodeVal := make([]byte, len(it.NodeBlob()))
copy(nodeVal, it.NodeBlob()) copy(nodeVal, it.NodeBlob())
if err := output(types2.StateNode{ nodeHash := make([]byte, len(it.Hash().Bytes()))
Removed: false, copy(nodeHash, it.Hash().Bytes())
Path: nodePath, if err := output(types2.IPLD{
NodeValue: nodeVal, // TODO: add Hash field so we dont have to recompute hash to insert into blockstore CID: ipld2.Keccak256ToCid(ipld2.MEthStateTrie, nodeHash).String(),
Content: nodeVal,
}); err != nil { }); err != nil {
return nil, nil, err return nil, err
} }
// add non-value-node paths to the list of diffPathsAtB
diffPathsAtB[common.Bytes2Hex(nodePath)] = true
} }
} }
return diffAccountsAtB, diffPathsAtB, it.Error() return diffAccountsAtB, it.Error()
}
// reminder: it.Leaf() == true when the iterator is positioned at a "value node" which is not something that actually exists in an MMPT
func (sdb *StateDiffBuilder) processStateValueNode(it trie.NodeIterator, watchedAddressesLeafPaths [][]byte) (types2.AccountWrapper, error) {
// skip if it is not a watched address
nodePath := make([]byte, len(it.Path()))
copy(nodePath, it.Path())
if !isWatchedAddress(watchedAddressesLeafPaths, nodePath) {
return types2.AccountWrapper{}, nil
}
// created vs updated is important for leaf nodes since we need to diff their storage
// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
var account types.StateAccount
accountRLP := make([]byte, 0)
copy(accountRLP, it.LeafBlob())
if err := rlp.DecodeBytes(accountRLP, &account); err != nil {
return types2.AccountWrapper{}, fmt.Errorf("error decoding account for leaf value at leaf key %x\nerror: %v", it.LeafKey(), err)
}
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
// since this is a "value node", we need to move up to the "parent" node which is the actual leaf node
// it should be in the fastcache since it necessarily was recently accessed to reach the current node
parentNodeRLP, err := sdb.StateCache.TrieDB().Node(it.Parent())
if err != nil {
return types2.AccountWrapper{}, err
}
return types2.AccountWrapper{
LeafKey: leafKey,
Account: &account,
NodeHash: crypto.Keccak256(parentNodeRLP),
}, nil
} }
// deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B
// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
func (sdb *StateDiffBuilder) deletedOrUpdatedState(a, b trie.NodeIterator, diffAccountsAtB types2.AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafPaths [][]byte, intermediateStateNodes, intermediateStorageNodes bool, output types2.StateNodeSink) (types2.AccountMap, error) { func (sdb *StateDiffBuilder) deletedOrUpdatedState(a, b trie.NodeIterator, diffAccountsAtB types2.AccountMap,
watchedAddressesLeafPaths [][]byte, output types2.StateNodeSink) (types2.AccountMap, error) {
diffAccountAtA := make(types2.AccountMap) diffAccountAtA := make(types2.AccountMap)
watchingAddresses := len(watchedAddressesLeafPaths) > 0 watchingAddresses := len(watchedAddressesLeafPaths) > 0
@ -376,90 +277,33 @@ func (sdb *StateDiffBuilder) deletedOrUpdatedState(a, b trie.NodeIterator, diffA
continue continue
} }
nodePath := make([]byte, len(it.Path()))
copy(nodePath, it.Path())
if it.Leaf() { if it.Leaf() {
// ignore leaf node if it is not a watched address accountW, err := sdb.processStateValueNode(it, watchedAddressesLeafPaths)
if !isWatchedAddress(watchedAddressesLeafPaths, nodePath) {
continue
}
// map all different accounts at A to their leafkey
var account types.StateAccount
accountRLP := make([]byte, 0)
copy(accountRLP, it.LeafBlob())
if err := rlp.DecodeBytes(accountRLP, &account); err != nil {
return nil, fmt.Errorf("error decoding account for leaf node at key %x\n nerror: %v", it.LeafKey(), err)
}
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
parentNodeRLP, err := sdb.StateCache.TrieDB().Node(it.Parent())
if err != nil { if err != nil {
return nil, err return nil, err
} }
leafKey := common.Bytes2Hex(accountW.LeafKey)
leafNodeHash := crypto.Keccak256(parentNodeRLP) diffAccountAtA[leafKey] = accountW
// if this node's leaf key did not show up in diffAccountsAtB
diffAccountAtA[common.Bytes2Hex(leafKey)] = types2.AccountWrapper{ // that means the account was deleted
Removed: false, // in that case, emit an empty "removed" diff state node
Path: nodePath, // include empty "removed" diff storage nodes for all the storage slots
LeafKey: leafKey, if _, ok := diffAccountsAtB[leafKey]; !ok {
Account: &account, diff := types2.StateLeafNode{
LeafNodeHash: leafNodeHash, AccountWrapper: accountW,
} Removed: true,
// if this node's path did not show up in diffPathsAtB
// that means the node at this path was deleted (or moved) in B
if _, ok := diffPathsAtB[common.Bytes2Hex(nodePath)]; !ok {
// TODO: REMOVE THIS CONDITION
// value nodes dont insert path in diffPathsAtB, this will always be !ok
var diff types2.StateNode
// if this node's leaf key also did not show up in diffAccountsAtB
// that means the node was deleted
// in that case, emit an empty "removed" diff state node
// include empty "removed" diff storage nodes for all the storage slots
if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok {
diff = types2.StateNode{
Removed: true,
Path: nodePath,
LeafKey: leafKey,
NodeValue: []byte{},
}
var storageDiffs []types2.StorageNode
err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, StorageNodeAppender(&storageDiffs))
if err != nil {
return nil, fmt.Errorf("failed building storage diffs for removed state account with key %x\r\nerror: %v", leafKey, err)
}
diff.StorageNodes = storageDiffs
} else {
// emit an empty "removed" diff with empty leaf key if the account was moved
diff = types2.StateNode{
Removed: true,
Path: nodePath,
NodeValue: []byte{},
}
} }
var storageDiff []types2.StorageLeafNode
err := sdb.buildRemovedAccountStorageNodes(accountW.Account.Root, StorageNodeAppender(&storageDiff))
if err != nil {
return nil, fmt.Errorf("failed building storage diffs for removed state account with key %x\r\nerror: %v", leafKey, err)
}
diff.StorageDiff = storageDiff
if err := output(diff); err != nil { if err := output(diff); err != nil {
return nil, err return nil, err
} }
} }
} else {
// if this node's path did not show up in diffPathsAtB
// that means the node at this path was deleted (or moved) in B
// emit an empty "removed" diff to signify as such
if intermediateStateNodes {
if _, ok := diffPathsAtB[common.Bytes2Hex(nodePath)]; !ok {
if err := output(types2.StateNode{
Path: nodePath,
NodeValue: []byte{},
Removed: true,
}); err != nil {
return nil, err
}
}
}
// fall through, we did everything we need to do with these node types
} }
} }
return diffAccountAtA, it.Error() return diffAccountAtA, it.Error()
@ -469,29 +313,26 @@ func (sdb *StateDiffBuilder) deletedOrUpdatedState(a, b trie.NodeIterator, diffA
// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states // to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
// needs to be called before building account creations and deletions as this mutates // needs to be called before building account creations and deletions as this mutates
// those account maps to remove the accounts which were updated // those account maps to remove the accounts which were updated
func (sdb *StateDiffBuilder) buildAccountUpdates(creations, deletions types2.AccountMap, updatedKeys []string, intermediateStorageNodes bool, output types2.StateNodeSink) error { func (sdb *StateDiffBuilder) buildAccountUpdates(creations, deletions types2.AccountMap, updatedKeys []string,
output types2.StateNodeSink, ipldOutput types2.IPLDSink) error {
var err error var err error
for _, key := range updatedKeys { for _, key := range updatedKeys {
createdAcc := creations[key] createdAcc := creations[key]
deletedAcc := deletions[key] deletedAcc := deletions[key]
var storageDiffs []types2.StorageNode var storageDiff []types2.StorageLeafNode
if deletedAcc.Account != nil && createdAcc.Account != nil { if deletedAcc.Account != nil && createdAcc.Account != nil {
oldSR := deletedAcc.Account.Root oldSR := deletedAcc.Account.Root
newSR := createdAcc.Account.Root newSR := createdAcc.Account.Root
err = sdb.buildStorageNodesIncremental( err = sdb.buildStorageNodesIncremental(
oldSR, newSR, intermediateStorageNodes, oldSR, newSR, StorageNodeAppender(&storageDiff), ipldOutput)
StorageNodeAppender(&storageDiffs))
if err != nil { if err != nil {
return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
} }
} }
if err = output(types2.StateNode{ if err = output(types2.StateLeafNode{
Removed: createdAcc.Removed, AccountWrapper: createdAcc,
Path: createdAcc.Path, Removed: false,
NodeValue: createdAcc.NodeValue, StorageDiff: storageDiff,
LeafKey: createdAcc.LeafKey,
NodeHash: createdAcc.LeafNodeHash,
StorageNodes: storageDiffs,
}); err != nil { }); err != nil {
return err return err
} }
@ -504,32 +345,29 @@ func (sdb *StateDiffBuilder) buildAccountUpdates(creations, deletions types2.Acc
// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A // buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
// it also returns the code and codehash for created contract accounts // it also returns the code and codehash for created contract accounts
func (sdb *StateDiffBuilder) buildAccountCreations(accounts types2.AccountMap, intermediateStorageNodes bool, output types2.StateNodeSink, codeOutput types2.CodeSink) error { func (sdb *StateDiffBuilder) buildAccountCreations(accounts types2.AccountMap, output types2.StateNodeSink, ipldOutput types2.IPLDSink) error {
for _, val := range accounts { for _, val := range accounts {
diff := types2.StateNode{ diff := types2.StateLeafNode{
Removed: val.Removed, AccountWrapper: val,
Path: val.Path, Removed: false,
LeafKey: val.LeafKey,
NodeValue: val.NodeValue,
NodeHash: val.LeafNodeHash,
} }
if !bytes.Equal(val.Account.CodeHash, nullCodeHash) { if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
// For contract creations, any storage node contained is a diff // For contract creations, any storage node contained is a diff
var storageDiffs []types2.StorageNode var storageDiff []types2.StorageLeafNode
err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, StorageNodeAppender(&storageDiffs)) err := sdb.buildStorageNodesEventual(val.Account.Root, StorageNodeAppender(&storageDiff), ipldOutput)
if err != nil { if err != nil {
return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) return fmt.Errorf("failed building eventual storage diffs for node with leaf key %x\r\nerror: %v", val.LeafKey, err)
} }
diff.StorageNodes = storageDiffs diff.StorageDiff = storageDiff
// emit codehash => code mappings for cod // emit codehash => code mappings for cod
codeHash := common.BytesToHash(val.Account.CodeHash) codeHash := common.BytesToHash(val.Account.CodeHash)
code, err := sdb.StateCache.ContractCode(common.Hash{}, codeHash) code, err := sdb.StateCache.ContractCode(common.Hash{}, codeHash)
if err != nil { if err != nil {
return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
} }
if err := codeOutput(types2.CodeAndCodeHash{ if err := ipldOutput(types2.IPLD{
Hash: codeHash, CID: ipld2.Keccak256ToCid(ipld2.RawBinary, codeHash.Bytes()).String(),
Code: code, Content: code,
}); err != nil { }); err != nil {
return err return err
} }
@ -544,7 +382,8 @@ func (sdb *StateDiffBuilder) buildAccountCreations(accounts types2.AccountMap, i
// buildStorageNodesEventual builds the storage diff node objects for a created account // buildStorageNodesEventual builds the storage diff node objects for a created account
// i.e. it returns all the storage nodes at this state, since there is no previous state // i.e. it returns all the storage nodes at this state, since there is no previous state
func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, output types2.StorageNodeSink,
ipldOutput types2.IPLDSink) error {
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
return nil return nil
} }
@ -555,7 +394,7 @@ func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, intermedi
return err return err
} }
it := sTrie.NodeIterator(make([]byte, 0)) it := sTrie.NodeIterator(make([]byte, 0))
err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output) err = sdb.buildStorageNodesFromTrie(it, output, ipldOutput)
if err != nil { if err != nil {
return err return err
} }
@ -564,49 +403,62 @@ func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, intermedi
// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator
// including intermediate nodes can be turned on or off // including intermediate nodes can be turned on or off
func (sdb *StateDiffBuilder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error { func (sdb *StateDiffBuilder) buildStorageNodesFromTrie(it trie.NodeIterator, output types2.StorageNodeSink,
ipldOutput types2.IPLDSink) error {
for it.Next(true) { for it.Next(true) {
// skip value nodes // skip null nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { if bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue continue
} }
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB())
if err != nil { if it.Leaf() {
return err storageLeafNode, err := sdb.processStorageValueNode(it)
} if err != nil {
switch node.NodeType { return err
case types2.Leaf: }
partialPath := trie.CompactToHex(nodeElements[0].([]byte)) if err := output(storageLeafNode); err != nil {
valueNodePath := append(node.Path, partialPath...) return err
encodedPath := trie.HexToCompact(valueNodePath) }
leafKey := encodedPath[1:] } else {
if err := output(types2.StorageNode{ nodeVal := make([]byte, len(it.NodeBlob()))
NodeType: node.NodeType, copy(nodeVal, it.NodeBlob())
Path: node.Path, nodeHash := make([]byte, len(it.Hash().Bytes()))
NodeValue: node.NodeValue, copy(nodeHash, it.Hash().Bytes())
LeafKey: leafKey, if err := ipldOutput(types2.IPLD{
CID: ipld2.Keccak256ToCid(ipld2.MEthStorageTrie, nodeHash).String(),
Content: nodeVal,
}); err != nil { }); err != nil {
return err return err
} }
case types2.Extension, types2.Branch:
if intermediateNodes {
if err := output(types2.StorageNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
}); err != nil {
return err
}
}
default:
return fmt.Errorf("unexpected node type %s", node.NodeType)
} }
} }
return it.Error() return it.Error()
} }
// reminder: it.Leaf() == true when the iterator is positioned at a "value node" which is not something that actually exists in an MMPT
func (sdb *StateDiffBuilder) processStorageValueNode(it trie.NodeIterator) (types2.StorageLeafNode, error) {
// skip if it is not a watched address
leafKey := make([]byte, len(it.LeafKey()))
copy(leafKey, it.LeafKey())
value := make([]byte, len(it.LeafBlob()))
copy(value, it.LeafBlob())
// since this is a "value node", we need to move up to the "parent" node which is the actual leaf node
// it should be in the fastcache since it necessarily was recently accessed to reach the current node
parentNodeRLP, err := sdb.StateCache.TrieDB().Node(it.Parent())
if err != nil {
return types2.StorageLeafNode{}, err
}
return types2.StorageLeafNode{
LeafKey: leafKey,
Value: value,
NodeHash: crypto.Keccak256(parentNodeRLP),
}, nil
}
// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account // buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account
func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, output types2.StorageNodeSink) error {
if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
return nil return nil
} }
@ -617,7 +469,7 @@ func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, int
return err return err
} }
it := sTrie.NodeIterator(make([]byte, 0)) it := sTrie.NodeIterator(make([]byte, 0))
err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output) err = sdb.buildRemovedStorageNodesFromTrie(it, output)
if err != nil { if err != nil {
return err return err
} }
@ -625,50 +477,29 @@ func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, int
} }
// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator // buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator
// including intermediate nodes can be turned on or off func (sdb *StateDiffBuilder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, output types2.StorageNodeSink) error {
func (sdb *StateDiffBuilder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) error {
for it.Next(true) { for it.Next(true) {
// skip value nodes // skip null nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { if bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue continue
} }
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) if it.Leaf() { // only leaf values are indexed, don't need to demarcate removed intermediate nodes
if err != nil { leafKey := make([]byte, len(it.LeafKey()))
return err copy(leafKey, it.LeafKey())
} if err := output(types2.StorageLeafNode{
switch node.NodeType { Removed: true,
case types2.Leaf: LeafKey: leafKey,
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
if err := output(types2.StorageNode{
NodeType: types2.Removed,
Path: node.Path,
NodeValue: []byte{},
LeafKey: leafKey,
}); err != nil { }); err != nil {
return err return err
} }
case types2.Extension, types2.Branch:
if intermediateNodes {
if err := output(types2.StorageNode{
NodeType: types2.Removed,
Path: node.Path,
NodeValue: []byte{},
}); err != nil {
return err
}
}
default:
return fmt.Errorf("unexpected node type %s", node.NodeType)
} }
} }
return it.Error() return it.Error()
} }
// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output types2.StorageNodeSink) error { func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, output types2.StorageNodeSink,
ipldOutput types2.IPLDSink) error {
if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
return nil return nil
} }
@ -682,128 +513,74 @@ func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldSR common.Hash, new
return err return err
} }
diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage( diffSlotsAtB, err := sdb.createdAndUpdatedStorage(
oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), output, ipldOutput)
intermediateNodes, output)
if err != nil { if err != nil {
return err return err
} }
err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}),
diffSlotsAtB, diffPathsAtB, intermediateNodes, output) diffSlotsAtB, output)
if err != nil { if err != nil {
return err return err
} }
return nil return nil
} }
func (sdb *StateDiffBuilder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output types2.StorageNodeSink) (map[string]bool, map[string]bool, error) { func (sdb *StateDiffBuilder) createdAndUpdatedStorage(a, b trie.NodeIterator, output types2.StorageNodeSink,
diffPathsAtB := make(map[string]bool) ipldOutput types2.IPLDSink) (map[string]bool, error) {
diffSlotsAtB := make(map[string]bool) diffSlotsAtB := make(map[string]bool)
it, _ := trie.NewDifferenceIterator(a, b) it, _ := trie.NewDifferenceIterator(a, b)
for it.Next(true) { for it.Next(true) {
// skip value nodes // skip null nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { if bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue continue
} }
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) if it.Leaf() {
if err != nil { storageLeafNode, err := sdb.processStorageValueNode(it)
return nil, nil, err if err != nil {
} return nil, err
switch node.NodeType { }
case types2.Leaf: if err := output(storageLeafNode); err != nil {
partialPath := trie.CompactToHex(nodeElements[0].([]byte)) return nil, err
valueNodePath := append(node.Path, partialPath...) }
encodedPath := trie.HexToCompact(valueNodePath) diffSlotsAtB[common.Bytes2Hex(storageLeafNode.LeafKey)] = true
leafKey := encodedPath[1:] } else {
diffSlotsAtB[common.Bytes2Hex(leafKey)] = true nodeVal := make([]byte, len(it.NodeBlob()))
if err := output(types2.StorageNode{ copy(nodeVal, it.NodeBlob())
NodeType: node.NodeType, nodeHash := make([]byte, len(it.Hash().Bytes()))
Path: node.Path, copy(nodeHash, it.Hash().Bytes())
NodeValue: node.NodeValue, if err := ipldOutput(types2.IPLD{
LeafKey: leafKey, CID: ipld2.Keccak256ToCid(ipld2.MEthStorageTrie, nodeHash).String(),
Content: nodeVal,
}); err != nil { }); err != nil {
return nil, nil, err return nil, err
} }
case types2.Extension, types2.Branch:
if intermediateNodes {
if err := output(types2.StorageNode{
NodeType: node.NodeType,
Path: node.Path,
NodeValue: node.NodeValue,
}); err != nil {
return nil, nil, err
}
}
default:
return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
} }
diffPathsAtB[common.Bytes2Hex(node.Path)] = true
} }
return diffSlotsAtB, diffPathsAtB, it.Error() return diffSlotsAtB, it.Error()
} }
func (sdb *StateDiffBuilder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output types2.StorageNodeSink) error { func (sdb *StateDiffBuilder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB map[string]bool, output types2.StorageNodeSink) error {
it, _ := trie.NewDifferenceIterator(b, a) it, _ := trie.NewDifferenceIterator(b, a)
for it.Next(true) { for it.Next(true) {
// skip value nodes // skip null nodes
if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) { if bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
continue continue
} }
node, nodeElements, err := trie_helpers.ResolveNode(it, sdb.StateCache.TrieDB()) if it.Leaf() {
if err != nil { leafKey := make([]byte, len(it.LeafKey()))
return err copy(leafKey, it.LeafKey())
} // if this node's leaf key did not show up in diffSlotsAtB
// that means the storage slot was vacated
switch node.NodeType {
case types2.Leaf:
partialPath := trie.CompactToHex(nodeElements[0].([]byte))
valueNodePath := append(node.Path, partialPath...)
encodedPath := trie.HexToCompact(valueNodePath)
leafKey := encodedPath[1:]
// if this node's path did not show up in diffPathsAtB
// that means the node at this path was deleted (or moved) in B
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
// if this node's leaf key also did not show up in diffSlotsAtB
// that means the node was deleted
// in that case, emit an empty "removed" diff storage node
if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok {
if err := output(types2.StorageNode{
NodeType: types2.Removed,
Path: node.Path,
NodeValue: []byte{},
LeafKey: leafKey,
}); err != nil {
return err
}
} else {
// emit an empty "removed" diff with empty leaf key if the account was moved
if err := output(types2.StorageNode{
NodeType: types2.Removed,
Path: node.Path,
NodeValue: []byte{},
}); err != nil {
return err
}
}
}
case types2.Extension, types2.Branch:
// if this node's path did not show up in diffPathsAtB
// that means the node at this path was deleted in B
// in that case, emit an empty "removed" diff storage node // in that case, emit an empty "removed" diff storage node
if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok { if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok {
if intermediateNodes { if err := output(types2.StorageLeafNode{
if err := output(types2.StorageNode{ Removed: true,
NodeType: types2.Removed, LeafKey: leafKey,
Path: node.Path, }); err != nil {
NodeValue: []byte{}, return err
}); err != nil {
return err
}
} }
} }
default:
return fmt.Errorf("unexpected node type %s", node.NodeType)
} }
} }
return it.Error() return it.Error()