From 619f147b4ed75375a55df277c8df18ce8b5472f1 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sat, 24 Oct 2020 16:58:16 +0800 Subject: [PATCH] output code & codehash iteratively had to rf some types for this --- statediff/api.go | 1 + statediff/builder.go | 168 +++++++++++++------------ statediff/builder_test.go | 16 +-- statediff/indexer/indexer.go | 14 +++ statediff/indexer/shared/functions.go | 17 +++ statediff/service.go | 12 +- statediff/testhelpers/mocks/builder.go | 2 +- statediff/testhelpers/mocks/service.go | 3 +- statediff/types.go | 15 +-- statediff/types/types.go | 10 ++ 10 files changed, 152 insertions(+), 106 deletions(-) diff --git a/statediff/api.go b/statediff/api.go index 1fffd90a1..33ecedd67 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + . "github.com/ethereum/go-ethereum/statediff/types" ) // APIName is the namespace used for the state diffing service API diff --git a/statediff/builder.go b/statediff/builder.go index d9ccc9f3b..354204408 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - sdtypes "github.com/ethereum/go-ethereum/statediff/types" + . "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" ) @@ -44,7 +44,7 @@ var ( type Builder interface { BuildStateDiffObject(args Args, params Params) (StateObject, error) BuildStateTrieObject(current *types.Block) (StateObject, error) - WriteStateDiffObject(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) + WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error } type builder struct { @@ -52,18 +52,24 @@ type builder struct { } // convenience -func stateNodeAppender(nodes *[]sdtypes.StateNode) sdtypes.StateNodeSink { - return func(node sdtypes.StateNode) error { +func stateNodeAppender(nodes *[]StateNode) StateNodeSink { + return func(node StateNode) error { *nodes = append(*nodes, node) return nil } } -func storageNodeAppender(nodes *[]sdtypes.StorageNode) sdtypes.StorageNodeSink { - return func(node sdtypes.StorageNode) error { +func storageNodeAppender(nodes *[]StorageNode) StorageNodeSink { + return func(node StorageNode) error { *nodes = append(*nodes, node) return nil } } +func codeMappingAppender(codeAndCodeHashes *[]CodeAndCodeHash) CodeSink { + return func(c CodeAndCodeHash) error { + *codeAndCodeHashes = append(*codeAndCodeHashes, c) + return nil + } +} // NewBuilder is used to create a statediff builder func NewBuilder(stateCache state.Database) Builder { @@ -91,8 +97,8 @@ func (sdb *builder) BuildStateTrieObject(current *types.Block) (StateObject, err }, nil } -func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, []CodeAndCodeHash, error) { - stateNodes := make([]sdtypes.StateNode, 0) +func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAndCodeHash, error) { + stateNodes := make([]StateNode, 0) codeAndCodeHashes := make([]CodeAndCodeHash, 0) for it.Next(true) { // skip value nodes @@ -117,7 +123,7 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, [ return nil, nil, err } switch ty { - case sdtypes.Leaf: + case Leaf: var account state.Account if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) @@ -126,14 +132,14 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, [ valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - node := sdtypes.StateNode{ + node := StateNode{ NodeType: ty, Path: nodePath, LeafKey: leafKey, NodeValue: node, } if !bytes.Equal(account.CodeHash, nullCodeHash) { - var storageNodes []sdtypes.StorageNode + var storageNodes []StorageNode err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes)) if err != nil { return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err) @@ -151,8 +157,8 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, [ }) } stateNodes = append(stateNodes, node) - case sdtypes.Extension, sdtypes.Branch: - stateNodes = append(stateNodes, sdtypes.StateNode{ + case Extension, Branch: + stateNodes = append(stateNodes, StateNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -166,10 +172,11 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, [ // BuildStateDiffObject builds a statediff object from two blocks and the provided parameters func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, error) { - var stateNodes []sdtypes.StateNode - codeAndCodeHashes, err := sdb.WriteStateDiffObject(StateRoots{ - OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot, - }, params, stateNodeAppender(&stateNodes)) + var stateNodes []StateNode + var codeAndCodeHashes []CodeAndCodeHash + err := sdb.WriteStateDiffObject( + StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot}, + params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes)) if err != nil { return StateObject{}, err } @@ -182,24 +189,24 @@ func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, } // Writes a statediff object to output callback -func (sdb *builder) WriteStateDiffObject(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { +func (sdb *builder) WriteStateDiffObject(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error { if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 { // if we are watching only specific accounts then we are only diffing leaf nodes - return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params, output) + return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params, output, codeOutput) } else { - return sdb.buildStateDiffWithIntermediateStateNodes(args, params, output) + return sdb.buildStateDiffWithIntermediateStateNodes(args, params, output, codeOutput) } } -func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { +func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error { // Load tries for old and new states oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot) if err != nil { - return nil, fmt.Errorf("error creating trie for oldStateRoot: %v", err) + return fmt.Errorf("error creating trie for oldStateRoot: %v", err) } newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot) if err != nil { - return nil, fmt.Errorf("error creating trie for newStateRoot: %v", err) + return fmt.Errorf("error creating trie for newStateRoot: %v", err) } // collect a slice of all the intermediate nodes that were touched and exist at B @@ -209,7 +216,7 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, pa oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), output) if err != nil { - return nil, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) } // collect a slice of all the nodes that existed at a path in A that doesn't exist in B @@ -218,7 +225,7 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, pa oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB, output) if err != nil { - return nil, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } // collect and sort the leafkey keys for both account mappings into a slice @@ -236,25 +243,25 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, pa diffAccountsAtB, diffAccountsAtA, updatedKeys, params.WatchedStorageSlots, params.IntermediateStorageNodes, output) if err != nil { - return nil, fmt.Errorf("error building diff for updated accounts: %v", err) + return fmt.Errorf("error building diff for updated accounts: %v", err) } // build the diff nodes for created accounts - codeAndCodeHashes, err := sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output) + err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput) if err != nil { - return nil, fmt.Errorf("error building diff for created accounts: %v", err) + return fmt.Errorf("error building diff for created accounts: %v", err) } - return codeAndCodeHashes, nil + return nil } -func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { +func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, params Params, output StateNodeSink, codeOutput CodeSink) error { // Load tries for old (A) and new (B) states oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot) if err != nil { - return nil, fmt.Errorf("error creating trie for oldStateRoot: %v", err) + return fmt.Errorf("error creating trie for oldStateRoot: %v", err) } newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot) if err != nil { - return nil, fmt.Errorf("error creating trie for newStateRoot: %v", err) + return fmt.Errorf("error creating trie for newStateRoot: %v", err) } // collect a map of their leafkey to all the accounts that were touched and exist at B @@ -263,7 +270,7 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), params.WatchedAddresses) if err != nil { - return nil, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) } // collect a slice of all the nodes that existed at a path in A that doesn't exist in B @@ -272,7 +279,7 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB, output) if err != nil { - return nil, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } // collect and sort the leafkeys for both account mappings into a slice @@ -290,14 +297,14 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, diffAccountsAtB, diffAccountsAtA, updatedKeys, params.WatchedStorageSlots, params.IntermediateStorageNodes, output) if err != nil { - return nil, fmt.Errorf("error building diff for updated accounts: %v", err) + return fmt.Errorf("error building diff for updated accounts: %v", err) } // build the diff nodes for created accounts - codeAndCodeHashes, err := sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output) + err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput) if err != nil { - return nil, fmt.Errorf("error building diff for created accounts: %v", err) + return fmt.Errorf("error building diff for created accounts: %v", err) } - return codeAndCodeHashes, nil + return nil } // createdAndUpdatedState returns @@ -329,7 +336,7 @@ func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddres if err != nil { return nil, nil, err } - if ty == sdtypes.Leaf { + if ty == Leaf { // created vs updated is important for leaf nodes since we need to diff their storage // so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey var account state.Account @@ -360,7 +367,7 @@ func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddres // a slice of all the intermediate nodes that exist in a different state at B than A // a mapping of their leafkeys to all the accounts that exist in a different state at B than A // and a slice of the paths for all of the nodes included in both -func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, output sdtypes.StateNodeSink) (AccountMap, map[string]bool, error) { +func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, output StateNodeSink) (AccountMap, map[string]bool, error) { diffPathsAtB := make(map[string]bool) diffAcountsAtB := make(AccountMap) it, _ := trie.NewDifferenceIterator(a, b) @@ -387,7 +394,7 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt return nil, nil, err } switch ty { - case sdtypes.Leaf: + case Leaf: // created vs updated is important for leaf nodes since we need to diff their storage // so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey var account state.Account @@ -405,10 +412,10 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt LeafKey: leafKey, Account: &account, } - case sdtypes.Extension, sdtypes.Branch: + case Extension, Branch: // create a diff for any intermediate node that has changed at b // created vs updated makes no difference for intermediate nodes since we do not need to diff storage - if err := output(sdtypes.StateNode{ + if err := output(StateNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -426,7 +433,7 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B -func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output sdtypes.StateNodeSink) (AccountMap, error) { +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output StateNodeSink) (AccountMap, error) { diffAccountAtA := make(AccountMap) it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { @@ -443,10 +450,10 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // that means the node at this path was deleted (or moved) in B // emit an empty "removed" diff to signify as such if _, ok := diffPathsAtB[common.Bytes2Hex(nodePath)]; !ok { - if err := output(sdtypes.StateNode{ + if err := output(StateNode{ Path: nodePath, NodeValue: []byte{}, - NodeType: sdtypes.Removed, + NodeType: Removed, }); err != nil { return nil, err } @@ -464,7 +471,7 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m return nil, err } switch ty { - case sdtypes.Leaf: + case Leaf: // map all different accounts at A to their leafkey var account state.Account if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { @@ -481,7 +488,7 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m LeafKey: leafKey, Account: &account, } - case sdtypes.Extension, sdtypes.Branch: + case Extension, Branch: // fall through, we did everything we need to do with these node types default: return nil, fmt.Errorf("unexpected node type %s", ty) @@ -495,12 +502,12 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // needs to be called before building account creations and deletions as this mutates // those account maps to remove the accounts which were updated func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, - watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error { + watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink) error { var err error for _, key := range updatedKeys { createdAcc := creations[key] deletedAcc := deletions[key] - var storageDiffs []sdtypes.StorageNode + var storageDiffs []StorageNode if deletedAcc.Account != nil && createdAcc.Account != nil { oldSR := deletedAcc.Account.Root newSR := createdAcc.Account.Root @@ -511,7 +518,7 @@ func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updated return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) } } - if err = output(sdtypes.StateNode{ + if err = output(StateNode{ NodeType: createdAcc.NodeType, Path: createdAcc.Path, NodeValue: createdAcc.NodeValue, @@ -529,10 +536,9 @@ func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updated // buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A // it also returns the code and codehash for created contract accounts -func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { - codeAndCodeHashes := make([]CodeAndCodeHash, 0) +func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output StateNodeSink, codeOutput CodeSink) error { for _, val := range accounts { - diff := sdtypes.StateNode{ + diff := StateNode{ NodeType: val.NodeType, Path: val.Path, LeafKey: val.LeafKey, @@ -540,34 +546,36 @@ func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKey } if !bytes.Equal(val.Account.CodeHash, nullCodeHash) { // For contract creations, any storage node contained is a diff - var storageDiffs []sdtypes.StorageNode + var storageDiffs []StorageNode err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) if err != nil { - return nil, fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) + return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) } diff.StorageNodes = storageDiffs // emit codehash => code mappings for cod codeHash := common.BytesToHash(val.Account.CodeHash) code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash) if err != nil { - return nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) + return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) } - codeAndCodeHashes = append(codeAndCodeHashes, CodeAndCodeHash{ + if err := codeOutput(CodeAndCodeHash{ Hash: codeHash, Code: code, - }) + }); err != nil { + return err + } } if err := output(diff); err != nil { - return nil, err + return err } } - return codeAndCodeHashes, nil + return nil } // buildStorageNodesEventual builds the storage diff node objects for a created account // i.e. it returns all the storage nodes at this state, since there is no previous state -func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { +func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { return nil } @@ -588,7 +596,7 @@ func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator // if any storage keys are provided it will only return those leaf nodes // including intermediate nodes can be turned on or off -func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { +func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { for it.Next(true) { // skip value nodes if it.Leaf() { @@ -612,13 +620,13 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora return err } switch ty { - case sdtypes.Leaf: + case Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] if isWatchedStorageKey(watchedStorageKeys, leafKey) { - if err := output(sdtypes.StorageNode{ + if err := output(StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -627,9 +635,9 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora return err } } - case sdtypes.Extension, sdtypes.Branch: + case Extension, Branch: if intermediateNodes { - if err := output(sdtypes.StorageNode{ + if err := output(StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -645,7 +653,7 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora } // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A -func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { +func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { return nil } @@ -673,7 +681,7 @@ func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common return nil } -func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, error) { +func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) (map[string]bool, error) { diffPathsAtB := make(map[string]bool) it, _ := trie.NewDifferenceIterator(a, b) for it.Next(true) { @@ -699,13 +707,13 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys return nil, err } switch ty { - case sdtypes.Leaf: + case Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] if isWatchedStorageKey(watchedKeys, leafKey) { - if err := output(sdtypes.StorageNode{ + if err := output(StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -714,9 +722,9 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys return nil, err } } - case sdtypes.Extension, sdtypes.Branch: + case Extension, Branch: if intermediateNodes { - if err := output(sdtypes.StorageNode{ + if err := output(StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -732,7 +740,7 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys return diffPathsAtB, it.Error() } -func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { +func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output StorageNodeSink) error { it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { // skip value nodes @@ -763,24 +771,24 @@ func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB return err } switch ty { - case sdtypes.Leaf: + case Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] if isWatchedStorageKey(watchedKeys, leafKey) { - if err := output(sdtypes.StorageNode{ - NodeType: sdtypes.Removed, + if err := output(StorageNode{ + NodeType: Removed, Path: nodePath, NodeValue: []byte{}, }); err != nil { return err } } - case sdtypes.Extension, sdtypes.Branch: + case Extension, Branch: if intermediateNodes { - if err := output(sdtypes.StorageNode{ - NodeType: sdtypes.Removed, + if err := output(StorageNode{ + NodeType: Removed, Path: nodePath, NodeValue: []byte{}, }); err != nil { diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 2e6c14948..8ad0d9822 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -618,7 +618,7 @@ func TestBuilder(t *testing.T) { StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -870,7 +870,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -1084,7 +1084,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -1254,7 +1254,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -1851,7 +1851,7 @@ func TestBuilderWithMovedAccount(t *testing.T) { }, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -1973,7 +1973,7 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { }, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -2155,7 +2155,7 @@ func TestBuildStateTrie(t *testing.T) { StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, @@ -2236,7 +2236,7 @@ func TestBuildStateTrie(t *testing.T) { StorageNodes: emptyStorage, }, }, - CodeAndCodeHashes: []statediff.CodeAndCodeHash{ + CodeAndCodeHashes: []sdtypes.CodeAndCodeHash{ { Hash: testhelpers.CodeHash, Code: testhelpers.ByteCodeAfterDeployment, diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index bdd0cd527..1dafe6fde 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -44,6 +44,7 @@ import ( type Indexer interface { PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error + PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error } // StateDiffIndexer satisfies the Indexer interface for ethereum statediff objects @@ -364,3 +365,16 @@ func (sdi *StateDiffIndexer) PushStateNode(tx *BlockTx, stateNode sdtypes.StateN return nil } + +// Publishes code and codehash pairs to the ipld database +func (sdi *StateDiffIndexer) PushCodeAndCodeHash(tx *BlockTx, codeAndCodeHash sdtypes.CodeAndCodeHash) error { + // codec doesn't matter since db key is multihash-based + mhKey, err := shared.MultihashKeyFromKeccak256(codeAndCodeHash.Hash) + if err != nil { + return err + } + if err := shared.PublishDirect(tx.dbtx, mhKey, codeAndCodeHash.Code); err != nil { + return err + } + return nil +} diff --git a/statediff/indexer/shared/functions.go b/statediff/indexer/shared/functions.go index 26391decb..1627cec53 100644 --- a/statediff/indexer/shared/functions.go +++ b/statediff/indexer/shared/functions.go @@ -25,6 +25,7 @@ import ( "github.com/ipfs/go-ipfs-ds-help" node "github.com/ipfs/go-ipld-format" "github.com/jmoiron/sqlx" + "github.com/multiformats/go-multihash" "github.com/sirupsen/logrus" ) @@ -105,3 +106,19 @@ func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) { _, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) return c.String(), err } + +// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string +func MultihashKeyFromKeccak256(hash common.Hash) (string, error) { + mh, err := multihash.Encode(hash.Bytes(), multihash.KECCAK_256) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(mh) + return blockstore.BlockPrefix.String() + dbKey.String(), nil +} + +// PublishDirect diretly writes a previously derived mhkey => value pair to the ipld database +func PublishDirect(tx *sqlx.Tx, key string, value []byte) error { + _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, key, value) + return err +} diff --git a/statediff/service.go b/statediff/service.go index e65f27807..00475688e 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -41,7 +41,7 @@ import ( ind "github.com/ethereum/go-ethereum/statediff/indexer" nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" "github.com/ethereum/go-ethereum/statediff/indexer/postgres" - sdtypes "github.com/ethereum/go-ethereum/statediff/types" + . "github.com/ethereum/go-ethereum/statediff/types" ) const chainEventChanSize = 20000 @@ -465,14 +465,16 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p tx, err := sds.indexer.PushBlock(block, receipts, totalDifficulty) // defer handling of commit/rollback for any return case defer tx.Close() - output := func(node sdtypes.StateNode) error { + output := func(node StateNode) error { return sds.indexer.PushStateNode(tx, node) - return nil } - _, err = sds.Builder.WriteStateDiffObject(StateRoots{ + codeOutput := func(c CodeAndCodeHash) error { + return sds.indexer.PushCodeAndCodeHash(tx, c) + } + err = sds.Builder.WriteStateDiffObject(StateRoots{ NewStateRoot: block.Root(), OldStateRoot: parentRoot, - }, params, output) + }, params, output, codeOutput) // allow dereferencing of parent, keep current locked as it should be the next parent sds.BlockChain.UnlockTrie(parentRoot) diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index 8abc622b4..ff9faf3ec 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -42,7 +42,7 @@ func (builder *Builder) BuildStateDiffObject(args statediff.Args, params statedi } // BuildStateDiffObject mock method -func (builder *Builder) WriteStateDiffObject(args statediff.StateRoots, params statediff.Params, output sdtypes.StateNodeSink) error { +func (builder *Builder) WriteStateDiffObject(args statediff.StateRoots, params statediff.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error { builder.StateRoots = args builder.Params = params diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index 3805e3d9f..965c4fcf2 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) // MockStateDiffService is a mock state diff service @@ -289,7 +290,7 @@ func (sds *MockStateDiffService) closeType(subType common.Hash) { delete(sds.SubscriptionTypes, subType) } -func (sds *MockStateDiffService) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- statediff.CodeAndCodeHash, quitChan chan<- bool) { +func (sds *MockStateDiffService) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- sdtypes.CodeAndCodeHash, quitChan chan<- bool) { panic("implement me") } diff --git a/statediff/types.go b/statediff/types.go index 63491ccd8..5c798a6f9 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -87,17 +87,10 @@ func (sd *Payload) Encode() ([]byte, error) { // StateObject is the final output structure from the builder type StateObject struct { - BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Nodes []types.StateNode `json:"nodes" gencodec:"required"` - CodeAndCodeHashes []CodeAndCodeHash `json:"codeMapping"` -} - -// CodeAndCodeHash struct for holding codehash => code mappings -// we can't use an actual map because they are not rlp serializable -type CodeAndCodeHash struct { - Hash common.Hash `json:"codeHash"` - Code []byte `json:"code"` + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Nodes []types.StateNode `json:"nodes" gencodec:"required"` + CodeAndCodeHashes []types.CodeAndCodeHash `json:"codeMapping"` } // AccountMap is a mapping of hex encoded path => account wrapper diff --git a/statediff/types/types.go b/statediff/types/types.go index 3d19ca50b..08e2124fa 100644 --- a/statediff/types/types.go +++ b/statediff/types/types.go @@ -19,6 +19,8 @@ package types +import "github.com/ethereum/go-ethereum/common" + // NodeType for explicitly setting type of node type NodeType string @@ -47,5 +49,13 @@ type StorageNode struct { LeafKey []byte `json:"leafKey"` } +// CodeAndCodeHash struct for holding codehash => code mappings +// we can't use an actual map because they are not rlp serializable +type CodeAndCodeHash struct { + Hash common.Hash `json:"codeHash"` + Code []byte `json:"code"` +} + type StateNodeSink func(StateNode) error type StorageNodeSink func(StorageNode) error +type CodeSink func(CodeAndCodeHash) error