Merge pull request #6 from vulcanize/multithread

Concurrent statediff

cmd/serve.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -124,7 +123,7 @@ func startServers(serv sd.IService) error {
 	}
 	if httpPath != "" {
 		logWithCommand.Info("starting up HTTP server")
-		if _, _, err := rpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{statediff.APIName}, nil, nil, rpc.HTTPTimeouts{}); err != nil {
+		if _, _, err := rpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{sd.APIName}, nil, nil, rpc.HTTPTimeouts{}); err != nil {
 			return err
 		}
 	}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.7.1
+	github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha
 )
 
 replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5

go.sum

@@ -289,6 +289,8 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZ
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha h1:obQGU35NWJjxR9UTB2LEyjpO1B3XTRst0pBSYk/Z++g=
+github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha/go.mod h1:tjtesuYyYOHhG1Xq7BExdsHhRy6pi92/gc1NkZ+ojSY=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5 h1:U+BqhjRLR22e9OEm8cgWC3Eq3bh8G6azjNpXeenfCG4=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=

pkg/api.go

@@ -17,10 +17,14 @@ package statediff
 
 import (
 	"context"
-
-	"github.com/ethereum/go-ethereum/statediff"
 )
 
+// APIName is the namespace used for the state diffing service API
+const APIName = "statediff"
+
+// APIVersion is the version of the state diffing service API
+const APIVersion = "0.0.1"
+
 // PublicStateDiffAPI provides an RPC interface
 // that can be used to fetch historical diffs from leveldb directly
 type PublicStateDiffAPI struct {
@@ -35,11 +39,11 @@ func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI {
 }
 
 // StateDiffAt returns a state diff payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
 	return api.sds.StateDiffAt(blockNumber, params)
 }
 
 // StateTrieAt returns a state trie payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params Params) (*Payload, error) {
 	return api.sds.StateTrieAt(blockNumber, params)
 }

pkg/builder.go

@@ -0,0 +1,645 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+
+	iter "github.com/vulcanize/go-eth-state-node-iterator"
+)
+
+var (
+	nullHashBytes     = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
+	emptyNode, _      = rlp.EncodeToBytes([]byte{})
+	emptyContractRoot = crypto.Keccak256Hash(emptyNode)
+)
+
+// Builder interface exposes the method for building a state diff between two blocks
+type Builder interface {
+	BuildStateDiffObject(args Args, params Params) (StateObject, error)
+	BuildStateTrieObject(current *types.Block) (StateObject, error)
+}
+
+type builder struct {
+	stateCache state.Database
+}
+
+type iterPair struct {
+	older, newer trie.NodeIterator
+}
+
+// NewBuilder is used to create a statediff builder
+func NewBuilder(stateCache state.Database) Builder {
+	return &builder{
+		stateCache: stateCache, // state cache is safe for concurrent reads
+	}
+}
+
+// BuildStateTrieObject builds a state trie object from the provided block
+func (sdb *builder) BuildStateTrieObject(current *types.Block) (StateObject, error) {
+	currentTrie, err := sdb.stateCache.OpenTrie(current.Root())
+	if err != nil {
+		return StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
+	}
+	it := currentTrie.NodeIterator([]byte{})
+	stateNodes, err := sdb.buildStateTrie(it)
+	if err != nil {
+		return StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
+	}
+	return StateObject{
+		BlockNumber: current.Number(),
+		BlockHash:   current.Hash(),
+		Nodes:       stateNodes,
+	}, nil
+}
+
+func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (StateNode, []interface{}, error) {
+	nodePath := make([]byte, len(it.Path()))
+	copy(nodePath, it.Path())
+	node, err := trieDB.Node(it.Hash())
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return StateNode{}, nil, err
+	}
+	ty, err := CheckKeyType(nodeElements)
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	return StateNode{
+		NodeType:  ty,
+		Path:      nodePath,
+		NodeValue: node,
+	}, nodeElements, nil
+}
+
+func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, error) {
+	stateNodes := make([]StateNode, 0)
+	for it.Next(true) {
+		// skip value nodes
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return nil, err
+		}
+		switch node.NodeType {
+		case Leaf:
+			var account state.Account
+			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+				return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+			}
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			storageNodes, err := sdb.buildStorageNodesEventual(account.Root, nil, true)
+			if err != nil {
+				return nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
+			}
+			stateNodes = append(stateNodes, StateNode{
+				NodeType:     node.NodeType,
+				Path:         node.Path,
+				LeafKey:      leafKey,
+				NodeValue:    node.NodeValue,
+				StorageNodes: storageNodes,
+			})
+		case Extension, Branch:
+			stateNodes = append(stateNodes, StateNode{
+				NodeType:  node.NodeType,
+				Path:      node.Path,
+				NodeValue: node.NodeValue,
+			})
+		default:
+			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+	}
+	return stateNodes, it.Error()
+}
+
+// BuildStateDiff builds a statediff object from two blocks and the provided parameters
+func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, error) {
+	if len(params.WatchedAddresses) > 0 {
+		// if we are watching only specific accounts then we are only diffing leaf nodes
+		log.Info("Ignoring intermediate state nodes because WatchedAddresses was passed")
+		params.IntermediateStateNodes = false
+	}
+
+	// Load tries for old and new states
+	oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
+	if err != nil {
+		return StateObject{}, fmt.Errorf("error creating trie for old state root: %v", err)
+	}
+	newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
+	if err != nil {
+		return StateObject{}, fmt.Errorf("error creating trie for new state root: %v", err)
+	}
+	nWorkers := params.Workers
+	if nWorkers == 0 {
+		nWorkers = 1
+	}
+
+	// Split old and new tries into corresponding subtrie iterators
+	oldIterFac := iter.NewSubtrieIteratorFactory(oldTrie, nWorkers)
+	newIterFac := iter.NewSubtrieIteratorFactory(newTrie, nWorkers)
+	iterChan := make(chan []iterPair, nWorkers)
+
+	// Create iterators ahead of time to avoid race condition in state.Trie access
+	for i := uint(0); i < nWorkers; i++ {
+		// two state iterations per diff build
+		iterChan <- []iterPair{
+			iterPair{
+				older: oldIterFac.IteratorAt(i),
+				newer: newIterFac.IteratorAt(i),
+			},
+			iterPair{
+				older: oldIterFac.IteratorAt(i),
+				newer: newIterFac.IteratorAt(i),
+			},
+		}
+	}
+
+	nodeChan := make(chan []StateNode)
+	var wg sync.WaitGroup
+
+	for w := uint(0); w < nWorkers; w++ {
+		wg.Add(1)
+		go func(iterChan <-chan []iterPair) error {
+			defer wg.Done()
+			if iters, more := <-iterChan; more {
+				subtrieNodes, err := sdb.buildStateDiff(iters, params)
+				if err != nil {
+					return err
+				}
+				nodeChan <- subtrieNodes
+			}
+			return nil
+		}(iterChan)
+	}
+
+	go func() {
+		defer close(nodeChan)
+		defer close(iterChan)
+		wg.Wait()
+	}()
+
+	stateNodes := make([]StateNode, 0)
+	for subtrieNodes := range nodeChan {
+		stateNodes = append(stateNodes, subtrieNodes...)
+	}
+
+	return StateObject{
+		BlockHash:   args.BlockHash,
+		BlockNumber: args.BlockNumber,
+		Nodes:       stateNodes,
+	}, nil
+}
+
+func (sdb *builder) buildStateDiff(args []iterPair, params Params) ([]StateNode, error) {
+	// collect a slice of all the intermediate nodes that were touched and exist at B
+	// a map of their leafkey to all the accounts that were touched and exist at B
+	// and a slice of all the paths for the nodes in both of the above sets
+	createdOrUpdatedIntermediateNodes, diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState(
+		args[0], params.WatchedAddresses, params.IntermediateStateNodes)
+	if err != nil {
+		return nil, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
+	}
+
+	// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
+	// a map of their leafkey to all the accounts that were touched and exist at A
+	emptiedPaths, diffAccountsAtA, err := sdb.deletedOrUpdatedState(args[1], diffPathsAtB)
+	if err != nil {
+		return nil, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
+	}
+
+	// collect and sort the leafkey keys for both account mappings into a slice
+	createKeys := sortKeys(diffAccountsAtB)
+	deleteKeys := sortKeys(diffAccountsAtA)
+
+	// and then find the intersection of these keys
+	// these are the leafkeys for the accounts which exist at both A and B but are different
+	// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
+	// and leaving the truly created or deleted keys in place
+	updatedKeys := findIntersection(createKeys, deleteKeys)
+
+	// build the diff nodes for the updated accounts using the mappings at both A and B
+	// as directed by the keys found as the intersection of the two
+	updatedAccounts, err := sdb.buildAccountUpdates(
+		diffAccountsAtB, diffAccountsAtA, updatedKeys,
+		params.WatchedStorageSlots, params.IntermediateStorageNodes)
+	if err != nil {
+		return nil, fmt.Errorf("error building diff for updated accounts: %v", err)
+	}
+	// build the diff nodes for created accounts
+	createdAccounts, err := sdb.buildAccountCreations(
+		diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes)
+	if err != nil {
+		return nil, fmt.Errorf("error building diff for created accounts: %v", err)
+	}
+
+	// assemble all of the nodes into the statediff object, including the intermediate nodes
+	res := append(
+		append(
+			append(updatedAccounts, createdAccounts...),
+			createdOrUpdatedIntermediateNodes...,
+		),
+		emptiedPaths...)
+
+	var paths [][]byte
+	for _, n := range res {
+		paths = append(paths, n.Path)
+	}
+	return res, nil
+}
+
+// createdAndUpdatedState returns
+// a slice of all the intermediate nodes that exist in a different state at B than A
+// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
+// and a slice of the paths for all of the nodes included in both
+func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddresses []common.Address, intermediates bool) ([]StateNode, AccountMap, map[string]bool, error) {
+	createdOrUpdatedIntermediateNodes := make([]StateNode, 0)
+	diffPathsAtB := make(map[string]bool)
+	diffAcountsAtB := make(AccountMap)
+	it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
+	for it.Next(true) {
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		switch node.NodeType {
+		case Leaf:
+			// created vs updated is important for leaf nodes since we need to diff their storage
+			// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
+			var account state.Account
+			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+				return nil, nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+			}
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			if isWatchedAddress(watchedAddresses, leafKey) {
+				diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
+					NodeType:  node.NodeType,
+					Path:      node.Path,
+					NodeValue: node.NodeValue,
+					LeafKey:   leafKey,
+					Account:   &account,
+				}
+			}
+		case Extension, Branch:
+			// create a diff for any intermediate node that has changed at b
+			// created vs updated makes no difference for intermediate nodes since we do not need to diff storage
+			if intermediates {
+				createdOrUpdatedIntermediateNodes = append(createdOrUpdatedIntermediateNodes, StateNode{
+					NodeType:  node.NodeType,
+					Path:      node.Path,
+					NodeValue: node.NodeValue,
+				})
+			}
+		default:
+			return nil, nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+		// add both intermediate and leaf node paths to the list of diffPathsAtB
+		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
+	}
+	return createdOrUpdatedIntermediateNodes, diffAcountsAtB, diffPathsAtB, it.Error()
+}
+
+// deletedOrUpdatedState returns a slice of all the paths that are emptied at B
+// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
+func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffPathsAtB map[string]bool) ([]StateNode, AccountMap, error) {
+	emptiedPaths := make([]StateNode, 0)
+	diffAccountAtA := make(AccountMap)
+	it, _ := trie.NewDifferenceIterator(iters.newer, iters.older)
+	for it.Next(true) {
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return nil, nil, err
+		}
+		// if this nodePath did not show up in diffPathsAtB
+		// that means the node at this path was deleted (or moved) in B
+		// emit an empty "removed" diff to signify as such
+		if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
+			emptiedPaths = append(emptiedPaths, StateNode{
+				Path:      node.Path,
+				NodeValue: []byte{},
+				NodeType:  Removed,
+			})
+		}
+		switch node.NodeType {
+		case Leaf:
+			// map all different accounts at A to their leafkey
+			var account state.Account
+			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+			}
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
+				NodeType:  node.NodeType,
+				Path:      node.Path,
+				NodeValue: node.NodeValue,
+				LeafKey:   leafKey,
+				Account:   &account,
+			}
+		case Extension, Branch:
+			// fall through, we did everything we need to do with these node types
+		default:
+			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+	}
+	return emptiedPaths, diffAccountAtA, it.Error()
+}
+
+// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
+// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
+// needs to be called before building account creations and deletions as this mutates
+// those account maps to remove the accounts which were updated
+func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, watchedStorageKeys []common.Hash, intermediateStorageNodes bool) ([]StateNode, error) {
+	updatedAccounts := make([]StateNode, 0, len(updatedKeys))
+	var err error
+	for _, key := range updatedKeys {
+		createdAcc := creations[key]
+		deletedAcc := deletions[key]
+		var storageDiffs []StorageNode
+		if deletedAcc.Account != nil && createdAcc.Account != nil {
+			oldSR := deletedAcc.Account.Root
+			newSR := createdAcc.Account.Root
+			storageDiffs, err = sdb.buildStorageNodesIncremental(oldSR, newSR, watchedStorageKeys, intermediateStorageNodes)
+			if err != nil {
+				return nil, fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
+			}
+		}
+		updatedAccounts = append(updatedAccounts, StateNode{
+			NodeType:     createdAcc.NodeType,
+			Path:         createdAcc.Path,
+			NodeValue:    createdAcc.NodeValue,
+			LeafKey:      createdAcc.LeafKey,
+			StorageNodes: storageDiffs,
+		})
+		delete(creations, key)
+		delete(deletions, key)
+	}
+
+	return updatedAccounts, nil
+}
+
+// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
+func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool) ([]StateNode, error) {
+	accountDiffs := make([]StateNode, 0, len(accounts))
+	for _, val := range accounts {
+		// For account creations, any storage node contained is a diff
+		storageDiffs, err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes)
+		if err != nil {
+			return nil, fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
+		}
+		accountDiffs = append(accountDiffs, StateNode{
+			NodeType:     val.NodeType,
+			Path:         val.Path,
+			LeafKey:      val.LeafKey,
+			NodeValue:    val.NodeValue,
+			StorageNodes: storageDiffs,
+		})
+	}
+
+	return accountDiffs, nil
+}
+
+// buildStorageNodesEventual builds the storage diff node objects for a created account
+// i.e. it returns all the storage nodes at this state, since there is no previous state
+func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) {
+	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
+		return nil, nil
+	}
+	log.Debug("Storage Root For Eventual Diff", "root", sr, sr.Hex())
+	sTrie, err := sdb.stateCache.OpenTrie(sr)
+	if err != nil {
+		log.Info("error in build storage diff eventual", "error", err)
+		return nil, err
+	}
+	it := sTrie.NodeIterator(make([]byte, 0))
+	return sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes)
+}
+
+// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node iterator
+// if any storage keys are provided it will only return those leaf nodes
+// including intermediate nodes can be turned on or off
+func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) {
+	storageDiffs := make([]StorageNode, 0)
+	for it.Next(true) {
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return nil, err
+		}
+		switch node.NodeType {
+		case Leaf:
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			if isWatchedStorageKey(watchedStorageKeys, leafKey) {
+				storageDiffs = append(storageDiffs, StorageNode{
+					NodeType:  node.NodeType,
+					Path:      node.Path,
+					NodeValue: node.NodeValue,
+					LeafKey:   leafKey,
+				})
+			}
+		case Extension, Branch:
+			if intermediateNodes {
+				storageDiffs = append(storageDiffs, StorageNode{
+					NodeType:  node.NodeType,
+					Path:      node.Path,
+					NodeValue: node.NodeValue,
+				})
+			}
+		default:
+			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+	}
+	return storageDiffs, it.Error()
+}
+
+// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
+func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) {
+	if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
+		return nil, nil
+	}
+	log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
+	oldTrie, err := sdb.stateCache.OpenTrie(oldSR)
+	if err != nil {
+		return nil, err
+	}
+	newTrie, err := sdb.stateCache.OpenTrie(newSR)
+	if err != nil {
+		return nil, err
+	}
+
+	createdOrUpdatedStorage, diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), watchedStorageKeys, intermediateNodes)
+	if err != nil {
+		return nil, err
+	}
+	deletedStorage, err := sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB, watchedStorageKeys, intermediateNodes)
+	if err != nil {
+		return nil, err
+	}
+	return append(createdOrUpdatedStorage, deletedStorage...), nil
+}
+
+func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool) ([]StorageNode, map[string]bool, error) {
+	createdOrUpdatedStorage := make([]StorageNode, 0)
+	diffPathsAtB := make(map[string]bool)
+	it, _ := trie.NewDifferenceIterator(a, b)
+	for it.Next(true) {
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return nil, nil, err
+		}
+		switch node.NodeType {
+		case Leaf:
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			if isWatchedStorageKey(watchedKeys, leafKey) {
+				createdOrUpdatedStorage = append(createdOrUpdatedStorage, StorageNode{
+					NodeType:  node.NodeType,
+					Path:      node.Path,
+					NodeValue: node.NodeValue,
+					LeafKey:   leafKey,
+				})
+			}
+		case Extension, Branch:
+			if intermediateNodes {
+				createdOrUpdatedStorage = append(createdOrUpdatedStorage, StorageNode{
+					NodeType:  node.NodeType,
+					Path:      node.Path,
+					NodeValue: node.NodeValue,
+				})
+			}
+		default:
+			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
+	}
+	return createdOrUpdatedStorage, diffPathsAtB, it.Error()
+}
+
+func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) {
+	deletedStorage := make([]StorageNode, 0)
+	it, _ := trie.NewDifferenceIterator(b, a)
+	for it.Next(true) {
+		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
+			continue
+		}
+		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
+		if err != nil {
+			return nil, err
+		}
+		// if this node path showed up in diffPathsAtB
+		// that means this node was updated at B and we already have the updated diff for it
+		// otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event
+		if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok {
+			continue
+		}
+		switch node.NodeType {
+		case Leaf:
+			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+			valueNodePath := append(node.Path, partialPath...)
+			encodedPath := trie.HexToCompact(valueNodePath)
+			leafKey := encodedPath[1:]
+			if isWatchedStorageKey(watchedKeys, leafKey) {
+				deletedStorage = append(deletedStorage, StorageNode{
+					NodeType:  Removed,
+					Path:      node.Path,
+					NodeValue: []byte{},
+				})
+			}
+		case Extension, Branch:
+			if intermediateNodes {
+				deletedStorage = append(deletedStorage, StorageNode{
+					NodeType:  Removed,
+					Path:      node.Path,
+					NodeValue: []byte{},
+				})
+			}
+		default:
+			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
+		}
+	}
+	return deletedStorage, it.Error()
+}
+
+// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
+func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool {
+	// If we aren't watching any specific addresses, we are watching everything
+	if len(watchedAddresses) == 0 {
+		return true
+	}
+	for _, addr := range watchedAddresses {
+		addrHashKey := crypto.Keccak256(addr.Bytes())
+		if bytes.Equal(addrHashKey, stateLeafKey) {
+			return true
+		}
+	}
+	return false
+}
+
+// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch
+func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool {
+	// If we aren't watching any specific addresses, we are watching everything
+	if len(watchedKeys) == 0 {
+		return true
+	}
+	for _, hashKey := range watchedKeys {
+		if bytes.Equal(hashKey.Bytes(), storageLeafKey) {
+			return true
+		}
+	}
+	return false
+}

pkg/helpers.go

@@ -0,0 +1,96 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+func sortKeys(data AccountMap) []string {
+	keys := make([]string, 0, len(data))
+	for key := range data {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	return keys
+}
+
+// findIntersection finds the set of strings from both arrays that are equivalent
+// a and b must first be sorted
+// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
+func findIntersection(a, b []string) []string {
+	lenA := len(a)
+	lenB := len(b)
+	iOfA, iOfB := 0, 0
+	updates := make([]string, 0)
+	if iOfA >= lenA || iOfB >= lenB {
+		return updates
+	}
+	for {
+		switch strings.Compare(a[iOfA], b[iOfB]) {
+		// -1 when a[iOfA] < b[iOfB]
+		case -1:
+			iOfA++
+			if iOfA >= lenA {
+				return updates
+			}
+			// 0 when a[iOfA] == b[iOfB]
+		case 0:
+			updates = append(updates, a[iOfA])
+			iOfA++
+			iOfB++
+			if iOfA >= lenA || iOfB >= lenB {
+				return updates
+			}
+			// 1 when a[iOfA] > b[iOfB]
+		case 1:
+			iOfB++
+			if iOfB >= lenB {
+				return updates
+			}
+		}
+	}
+
+}
+
+// CheckKeyType checks what type of key we have
+func CheckKeyType(elements []interface{}) (NodeType, error) {
+	if len(elements) > 2 {
+		return Branch, nil
+	}
+	if len(elements) < 2 {
+		return Unknown, fmt.Errorf("node cannot be less than two elements in length")
+	}
+	switch elements[0].([]byte)[0] / 16 {
+	case '\x00':
+		return Extension, nil
+	case '\x01':
+		return Extension, nil
+	case '\x02':
+		return Leaf, nil
+	case '\x03':
+		return Leaf, nil
+	default:
+		return Unknown, fmt.Errorf("unknown hex prefix")
+	}
+}

pkg/service.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
 )
 
@@ -48,26 +47,26 @@ type IService interface {
 	// Main event loop for processing state diffs
 	Loop(wg *sync.WaitGroup)
 	// Method to get state diff object at specific block
-	StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error)
+	StateDiffAt(blockNumber uint64, params Params) (*Payload, error)
 	// Method to get state trie object at specific block
-	StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error)
+	StateTrieAt(blockNumber uint64, params Params) (*Payload, error)
 }
 
 // Service is the underlying struct for the state diffing service
 type Service struct {
 	// Used to build the state diff objects
-	Builder statediff.Builder
+	Builder Builder
 	// Used to read data from leveldb
 	lvlDBReader lvlDBReader
 	// Used to signal shutdown of the service
 	QuitChan chan bool
 }
 
-// NewStateDiffService creates a new statediff.Service
+// NewStateDiffService creates a new Service
 func NewStateDiffService(lvlDBReader lvlDBReader) (*Service, error) {
 	return &Service{
 		lvlDBReader: lvlDBReader,
-		Builder:     statediff.NewBuilder(lvlDBReader.StateDB()),
+		Builder:     NewBuilder(lvlDBReader.StateDB()),
 		QuitChan:    make(chan bool),
 	}, nil
 }
@@ -77,12 +76,12 @@ func (sds *Service) Protocols() []p2p.Protocol {
 	return []p2p.Protocol{}
 }
 
-// APIs returns the RPC descriptors the statediff.Service offers
+// APIs returns the RPC descriptors the Service offers
 func (sds *Service) APIs() []rpc.API {
 	return []rpc.API{
 		{
-			Namespace: statediff.APIName,
+			Namespace: APIName,
-			Version:   statediff.APIVersion,
+			Version:   APIVersion,
 			Service:   NewPublicStateDiffAPI(sds),
 			Public:    true,
 		},
@@ -104,7 +103,7 @@ func (sds *Service) Loop(wg *sync.WaitGroup) {
 
 // StateDiffAt returns a state diff object payload at the specific blockheight
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) StateDiffAt(blockNumber uint64, params Params) (*Payload, error) {
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return nil, err
@@ -121,12 +120,12 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*s
 }
 
 // processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
-func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params Params) (*Payload, error) {
-	stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
+	stateDiff, err := sds.Builder.BuildStateDiffObject(Args{
-		NewStateRoot: currentBlock.Root(),
-		OldStateRoot: parentRoot,
 		BlockHash:    currentBlock.Hash(),
 		BlockNumber:  currentBlock.Number(),
+		OldStateRoot: parentRoot,
+		NewStateRoot: currentBlock.Root(),
 	}, params)
 	if err != nil {
 		return nil, err
@@ -139,8 +138,8 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
 	return sds.newPayload(stateDiffRlp, currentBlock, params)
 }
 
-func (sds *Service) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) newPayload(stateObject []byte, block *types.Block, params Params) (*Payload, error) {
-	payload := &statediff.Payload{
+	payload := &Payload{
 		StateObjectRlp: stateObject,
 	}
 	if params.IncludeBlock {
@@ -173,7 +172,7 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params st
 
 // StateTrieAt returns a state trie object payload at the specified blockheight
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) StateTrieAt(blockNumber uint64, params Params) (*Payload, error) {
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return nil, err
@@ -182,7 +181,7 @@ func (sds *Service) StateTrieAt(blockNumber uint64, params statediff.Params) (*s
 	return sds.processStateTrie(currentBlock, params)
 }
 
-func (sds *Service) processStateTrie(block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) processStateTrie(block *types.Block, params Params) (*Payload, error) {
 	stateNodes, err := sds.Builder.BuildStateTrieObject(block)
 	if err != nil {
 		return nil, err

pkg/state_types.go

@@ -0,0 +1,47 @@
+package statediff
+
+// Wrapper types for state trie
+
+import (
+	"github.com/ethereum/go-ethereum/core/state"
+)
+
+// StateNode holds the data for a single state diff node
+type StateNode struct {
+	NodeType     NodeType      `json:"nodeType"        gencodec:"required"`
+	Path         []byte        `json:"path"            gencodec:"required"`
+	NodeValue    []byte        `json:"value"           gencodec:"required"`
+	StorageNodes []StorageNode `json:"storage"`
+	LeafKey      []byte        `json:"leafKey"`
+}
+
+// StorageNode holds the data for a single storage diff node
+type StorageNode struct {
+	NodeType  NodeType `json:"nodeType"        gencodec:"required"`
+	Path      []byte   `json:"path"            gencodec:"required"`
+	NodeValue []byte   `json:"value"           gencodec:"required"`
+	LeafKey   []byte   `json:"leafKey"`
+}
+
+// AccountMap is a mapping of hex encoded path => account wrapper
+type AccountMap map[string]accountWrapper
+
+// accountWrapper is used to temporary associate the unpacked node with its raw values
+type accountWrapper struct {
+	Account   *state.Account
+	NodeType  NodeType
+	Path      []byte
+	NodeValue []byte
+	LeafKey   []byte
+}
+
+// NodeType for explicitly setting type of node
+type NodeType string
+
+const (
+	Unknown   NodeType = "Unknown"
+	Leaf      NodeType = "Leaf"
+	Extension NodeType = "Extension"
+	Branch    NodeType = "Branch"
+	Removed   NodeType = "Removed" // used to represent pathes which have been emptied
+)

pkg/testhelpers/helpers.go

@@ -0,0 +1,124 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package testhelpers
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// MakeChain creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent.
+func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
+	config := params.TestChainConfig
+	blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
+	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
+	return blocks, chain
+}
+
+func TestSelfDestructChainGen(i int, block *core.BlockGen) {
+	signer := types.HomesteadSigner{}
+	switch i {
+	case 0:
+		// Block 1 is mined by Account1Addr
+		// Account1Addr creates a new contract
+		block.SetCoinbase(TestBankAddress)
+		tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, TestBankKey)
+		ContractAddr = crypto.CreateAddress(TestBankAddress, 0)
+		block.AddTx(tx)
+	case 1:
+		// Block 2 is mined by Account1Addr
+		// Account1Addr self-destructs the contract
+		block.SetCoinbase(TestBankAddress)
+		data := common.Hex2Bytes("43D726D6")
+		tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	}
+}
+
+func TestChainGen(i int, block *core.BlockGen) {
+	signer := types.HomesteadSigner{}
+	switch i {
+	case 0:
+		// In block 1, the test bank sends account #1 some ether.
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
+		block.AddTx(tx)
+	case 1:
+		// In block 2, the test bank sends some more ether to account #1.
+		// Account1Addr passes it on to account #2.
+		// Account1Addr creates a test contract.
+		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
+		nonce := block.TxNonce(Account1Addr)
+		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
+		nonce++
+		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key)
+		ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
+		block.AddTx(tx1)
+		block.AddTx(tx2)
+		block.AddTx(tx3)
+	case 2:
+		// Block 3 has a single tx from the bankAccount to the contract, that transfers no value
+		// Block 3 is mined by Account2Addr
+		block.SetCoinbase(Account2Addr)
+		//put function: c16431b9
+		//close function: 43d726d6
+		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	case 3:
+		// Block 4 has three txs from bankAccount to the contract, that transfer no value
+		// Two set the two original slot positions to 0 and one sets another position to a new value
+		// Block 4 is mined by Account2Addr
+		block.SetCoinbase(Account2Addr)
+		data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
+		data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000")
+		data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009")
+
+		nonce := block.TxNonce(TestBankAddress)
+		tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data1), signer, TestBankKey)
+		nonce++
+		tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data2), signer, TestBankKey)
+		nonce++
+		tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data3), signer, TestBankKey)
+		block.AddTx(tx1)
+		block.AddTx(tx2)
+		block.AddTx(tx3)
+	case 4:
+		// Block 5 has one tx from bankAccount to the contract, that transfers no value
+		// It sets the remaining storage value to zero
+		// Block 5 is mined by Account1Addr
+		block.SetCoinbase(Account1Addr)
+		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000")
+		nonce := block.TxNonce(TestBankAddress)
+		tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	case 5:
+		// Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value
+		// Block 6 is mined by Account2Addr
+		block.SetCoinbase(Account2Addr)
+		data := common.Hex2Bytes("43D726D6")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, nil, data), signer, Account1Key)
+		block.AddTx(tx)
+	}
+}

pkg/testhelpers/mocks/blockchain.go

@@ -0,0 +1,128 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// BlockChain is a mock blockchain for testing
+type BlockChain struct {
+	HashesLookedUp         []common.Hash
+	blocksToReturnByHash   map[common.Hash]*types.Block
+	blocksToReturnByNumber map[uint64]*types.Block
+	callCount              int
+	ChainEvents            []core.ChainEvent
+	Receipts               map[common.Hash]types.Receipts
+	TDByHash               map[common.Hash]*big.Int
+}
+
+// SetBlocksForHashes mock method
+func (blockChain *BlockChain) SetBlocksForHashes(blocks map[common.Hash]*types.Block) {
+	if blockChain.blocksToReturnByHash == nil {
+		blockChain.blocksToReturnByHash = make(map[common.Hash]*types.Block)
+	}
+	blockChain.blocksToReturnByHash = blocks
+}
+
+// GetBlockByHash mock method
+func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
+	blockChain.HashesLookedUp = append(blockChain.HashesLookedUp, hash)
+
+	var block *types.Block
+	if len(blockChain.blocksToReturnByHash) > 0 {
+		block = blockChain.blocksToReturnByHash[hash]
+	}
+
+	return block
+}
+
+// SetChainEvents mock method
+func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
+	blockChain.ChainEvents = chainEvents
+}
+
+// SubscribeChainEvent mock method
+func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+	subErr := errors.New("subscription error")
+
+	var eventCounter int
+	subscription := event.NewSubscription(func(quit <-chan struct{}) error {
+		for _, chainEvent := range blockChain.ChainEvents {
+			if eventCounter > 1 {
+				time.Sleep(250 * time.Millisecond)
+				return subErr
+			}
+			select {
+			case ch <- chainEvent:
+			case <-quit:
+				return nil
+			}
+			eventCounter++
+		}
+		return nil
+	})
+
+	return subscription
+}
+
+// SetReceiptsForHash test method
+func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) {
+	if blockChain.Receipts == nil {
+		blockChain.Receipts = make(map[common.Hash]types.Receipts)
+	}
+	blockChain.Receipts[hash] = receipts
+}
+
+// GetReceiptsByHash mock method
+func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+	return blockChain.Receipts[hash]
+}
+
+// SetBlockForNumber test method
+func (blockChain *BlockChain) SetBlockForNumber(block *types.Block, number uint64) {
+	if blockChain.blocksToReturnByNumber == nil {
+		blockChain.blocksToReturnByNumber = make(map[uint64]*types.Block)
+	}
+	blockChain.blocksToReturnByNumber[number] = block
+}
+
+// GetBlockByNumber mock method
+func (blockChain *BlockChain) GetBlockByNumber(number uint64) *types.Block {
+	return blockChain.blocksToReturnByNumber[number]
+}
+
+// GetTdByHash mock method
+func (blockChain *BlockChain) GetTdByHash(hash common.Hash) *big.Int {
+	return blockChain.TDByHash[hash]
+}
+
+func (blockChain *BlockChain) SetTdByHash(hash common.Hash, td *big.Int) {
+	if blockChain.TDByHash == nil {
+		blockChain.TDByHash = make(map[common.Hash]*big.Int)
+	}
+	blockChain.TDByHash[hash] = td
+}
+
+func (blockChain *BlockChain) UnlockTrie(root common.Hash) {}

pkg/testhelpers/mocks/builder.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	statediff "github.com/vulcanize/eth-statediff-service/pkg"
+)
+
+// Builder is a mock state diff builder
+type Builder struct {
+	Args         statediff.Args
+	Params       statediff.Params
+	stateDiff    statediff.StateObject
+	block        *types.Block
+	stateTrie    statediff.StateObject
+	builderError error
+}
+
+// BuildStateDiffObject mock method
+func (builder *Builder) BuildStateDiffObject(args statediff.Args, params statediff.Params) (statediff.StateObject, error) {
+	builder.Args = args
+	builder.Params = params
+
+	return builder.stateDiff, builder.builderError
+}
+
+// BuildStateTrieObject mock method
+func (builder *Builder) BuildStateTrieObject(block *types.Block) (statediff.StateObject, error) {
+	builder.block = block
+
+	return builder.stateTrie, builder.builderError
+}
+
+// SetStateDiffToBuild mock method
+func (builder *Builder) SetStateDiffToBuild(stateDiff statediff.StateObject) {
+	builder.stateDiff = stateDiff
+}
+
+// SetBuilderError mock method
+func (builder *Builder) SetBuilderError(err error) {
+	builder.builderError = err
+}

pkg/testhelpers/test_data.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package testhelpers
+
+import (
+	"math/big"
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// AddressToLeafKey hashes an returns an address
+func AddressToLeafKey(address common.Address) []byte {
+	return crypto.Keccak256(address[:])
+}
+
+// AddressToEncodedPath hashes an address and appends the even-number leaf flag to it
+func AddressToEncodedPath(address common.Address) []byte {
+	addrHash := crypto.Keccak256(address[:])
+	decodedPath := append(EvenLeafFlag, addrHash...)
+	return decodedPath
+}
+
+// Test variables
+var (
+	EvenLeafFlag = []byte{byte(2) << 4}
+	BlockNumber  = big.NewInt(rand.Int63())
+	BlockHash    = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
+	NullCodeHash = crypto.Keccak256Hash([]byte{})
+	StoragePath  = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+	StorageKey   = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+	StorageValue = common.Hex2Bytes("0x03")
+	NullHash     = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+
+	Testdb          = rawdb.NewMemoryDatabase()
+	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
+	BankLeafKey     = AddressToLeafKey(TestBankAddress)
+	TestBankFunds   = big.NewInt(100000000)
+	Genesis         = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
+
+	Account1Key, _  = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+	Account2Key, _  = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
+	Account1Addr    = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
+	Account2Addr    = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
+	Account1LeafKey = AddressToLeafKey(Account1Addr)
+	Account2LeafKey = AddressToLeafKey(Account2Addr)
+	ContractCode    = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506040518060200160405280600160ff16815250600190600161007492919061007a565b506100e4565b82606481019282156100ae579160200282015b828111156100ad578251829060ff1690559160200191906001019061008d565b5b5090506100bb91906100bf565b5090565b6100e191905b808211156100dd5760008160009055506001016100c5565b5090565b90565b6101ca806100f36000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
+	ContractAddr    common.Address
+
+	EmptyRootNode, _  = rlp.EncodeToBytes([]byte{})
+	EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode)
+)

pkg/types.go

@@ -0,0 +1,87 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// Subscription struct holds our subscription channels
+type Subscription struct {
+	PayloadChan chan<- Payload
+	QuitChan    chan<- bool
+}
+
+// Params is used to carry in parameters from subscribing/requesting clients configuration
+type Params struct {
+	IntermediateStateNodes   bool
+	IntermediateStorageNodes bool
+	IncludeBlock             bool
+	IncludeReceipts          bool
+	IncludeTD                bool
+	WatchedAddresses         []common.Address
+	WatchedStorageSlots      []common.Hash
+	Workers                  uint
+}
+
+// Args bundles the arguments for the state diff builder
+type Args struct {
+	OldStateRoot, NewStateRoot, BlockHash common.Hash
+	BlockNumber                           *big.Int
+}
+
+// Payload packages the data to send to statediff subscriptions
+type Payload struct {
+	BlockRlp        []byte   `json:"blockRlp"`
+	TotalDifficulty *big.Int `json:"totalDifficulty"`
+	ReceiptsRlp     []byte   `json:"receiptsRlp"`
+	StateObjectRlp  []byte   `json:"stateObjectRlp"    gencodec:"required"`
+
+	encoded []byte
+	err     error
+}
+
+func (sd *Payload) ensureEncoded() {
+	if sd.encoded == nil && sd.err == nil {
+		sd.encoded, sd.err = json.Marshal(sd)
+	}
+}
+
+// Length to implement Encoder interface for Payload
+func (sd *Payload) Length() int {
+	sd.ensureEncoded()
+	return len(sd.encoded)
+}
+
+// Encode to implement Encoder interface for Payload
+func (sd *Payload) Encode() ([]byte, error) {
+	sd.ensureEncoded()
+	return sd.encoded, sd.err
+}
+
+// StateObject is the final output structure from the builder
+type StateObject struct {
+	BlockHash   common.Hash `json:"blockHash"       gencodec:"required"`
+	BlockNumber *big.Int    `json:"blockNumber"     gencodec:"required"`
+	Nodes       []StateNode `json:"nodes"           gencodec:"required"`
+}