cmd/geth/main.go

@@ -148,8 +148,10 @@ var (
 		utils.EWASMInterpreterFlag,
 		utils.EVMInterpreterFlag,
 		utils.StateDiffFlag,
-		utils.StateDiffModeFlag,
-		utils.StateDiffPathFlag,
+		utils.StateDiffPathsAndProofs,
+		utils.StateDiffIntermediateNodes,
+		utils.StateDiffStreamBlock,
+		utils.StateDiffWatchedAddresses,
 		configFileFlag,
 	}
 

cmd/geth/usage.go

@@ -266,8 +266,10 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "STATE DIFF",
 		Flags: []cli.Flag{
 			utils.StateDiffFlag,
-			utils.StateDiffModeFlag,
-			utils.StateDiffPathFlag,
+			utils.StateDiffPathsAndProofs,
+			utils.StateDiffIntermediateNodes,
+			utils.StateDiffStreamBlock,
+			utils.StateDiffWatchedAddresses,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -29,6 +29,8 @@ import (
 	"strings"
 	"time"
 
+	cli "gopkg.in/urfave/cli.v1"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
@@ -59,11 +61,10 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
-	pcsclite "github.com/gballet/go-libpcsclite"
-	cli "gopkg.in/urfave/cli.v1"
-	"github.com/ethereum/go-ethereum/statediff/service"
 	"github.com/ethereum/go-ethereum/statediff"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
+
+	pcsclite "github.com/gballet/go-libpcsclite"
 )
 
 var (
@@ -742,19 +743,23 @@ var (
 
 	StateDiffFlag = cli.BoolFlag{
 		Name:  "statediff",
-		Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.",
+		Usage: "Enables the processing of state diffs between each block",
 	}
-
-	StateDiffModeFlag = cli.StringFlag{
-		Name:  "statediff.mode",
-		Usage: "Enables the user to determine which persistence mode they'd like to store the state diffs in.",
-		Value: "csv",
+	StateDiffPathsAndProofs = cli.BoolFlag{
+		Name:  "statediff.pathsandproofs",
+		Usage: "Set to true to generate paths and proof sets for diffed state and storage trie leaf nodes",
 	}
-
-	StateDiffPathFlag = cli.StringFlag{
-		Name:  "statediff.path",
-		Usage: "Enables the user to determine where to persist the state diffs.",
-		Value: ".",
+	StateDiffIntermediateNodes = cli.BoolFlag{
+		Name:  "statediff.intermediatenodes",
+		Usage: "Set to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only",
+	}
+	StateDiffStreamBlock = cli.BoolFlag{
+		Name:  "statediff.streamblock",
+		Usage: "Set to true to stream the block data alongside state diff data in the same subscription payload",
+	}
+	StateDiffWatchedAddresses = cli.StringSliceFlag{
+		Name:  "statediff.watchedaddresses",
+		Usage: "If provided, state diffing process is restricted to these addresses",
 	}
 )
 
@@ -965,6 +970,9 @@ func setWS(ctx *cli.Context, cfg *node.Config) {
 	if ctx.GlobalIsSet(WSApiFlag.Name) {
 		cfg.WSModules = splitAndTrim(ctx.GlobalString(WSApiFlag.Name))
 	}
+	if ctx.GlobalBool(StateDiffFlag.Name) {
+		cfg.WSModules = append(cfg.WSModules, "statediff")
+	}
 }
 
 // setIPC creates an IPC path configuration from the set command line flags,
@@ -1606,29 +1614,20 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st
 	}
 }
 
+// RegisterStateDiffService configures and registers a service to stream state diff data over RPC
 func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) {
-	//based on the context, if path and mode are set, update the config here
-	//otherwise pass in an empty config
-
-	modeFlag := ctx.GlobalString(StateDiffModeFlag.Name)
-	mode, err := statediff.NewMode(modeFlag)
-	if err != nil {
-		Fatalf("Failed to register State Diff Service", err)
-	}
-
-	path := ctx.GlobalString(StateDiffPathFlag.Name)
-
 	config := statediff.Config{
-		Mode: mode,
-		Path: path,
+		PathsAndProofs:    ctx.GlobalBool(StateDiffPathsAndProofs.Name),
+		IntermediateNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name),
+		StreamBlock:       ctx.GlobalBool(StateDiffStreamBlock.Name),
+		WatchedAddresses:  ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name),
 	}
-
 	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
 		var ethServ *eth.Ethereum
 		ctx.Service(&ethServ)
 		chainDb := ethServ.ChainDb()
 		blockChain := ethServ.BlockChain()
-		return service.NewStateDiffService(chainDb, blockChain, config)
+		return statediff.NewStateDiffService(chainDb, blockChain, config)
 	}); err != nil {
 		Fatalf("Failed to register State Diff Service", err)
 	}

core/blockchain.go

@@ -106,11 +106,11 @@ const (
 // CacheConfig contains the configuration values for the trie caching/pruning
 // that's resident in a blockchain.
 type CacheConfig struct {
-	TrieCleanLimit      int           // Memory allowance (MB) to use for caching trie nodes in memory
-	TrieCleanNoPrefetch bool          // Whether to disable heuristic state prefetching for followup blocks
-	TrieDirtyLimit      int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
-	TrieDirtyDisabled   bool          // Whether to disable trie write caching and GC altogether (archive node)
-	TrieTimeLimit       time.Duration // Time limit after which to flush the current in-memory trie to disk
+	TrieCleanLimit       int           // Memory allowance (MB) to use for caching trie nodes in memory
+	TrieCleanNoPrefetch  bool          // Whether to disable heuristic state prefetching for followup blocks
+	TrieDirtyLimit       int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
+	TrieDirtyDisabled    bool          // Whether to disable trie write caching and GC altogether (archive node)
+	TrieTimeLimit        time.Duration // Time limit after which to flush the current in-memory trie to disk
 	ProcessingStateDiffs bool          // Whether statediffs processing should be taken into a account before a trie is pruned
 }
 
@@ -1336,16 +1336,19 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 					bc.triegc.Push(root, number)
 					break
 				}
-
 				if bc.cacheConfig.ProcessingStateDiffs {
-					if !bc.allowedRootToBeDereferenced(root.(common.Hash)) {
+					if !bc.rootAllowedToBeDereferenced(root.(common.Hash)) {
 						bc.triegc.Push(root, number)
 						break
 					} else {
+						log.Debug("Current root found in stateDiffsProcessed collection with a count of 2, okay to dereference",
+							"root", root.(common.Hash).Hex(),
+							"blockNumber", uint64(-number),
+							"size of stateDiffsProcessed", len(bc.stateDiffsProcessed))
 						delete(bc.stateDiffsProcessed, root.(common.Hash))
 					}
 				}
-
+				log.Debug("Dereferencing", "root", root.(common.Hash).Hex())
 				triedb.Dereference(root.(common.Hash))
 			}
 		}
@@ -1403,7 +1406,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 // since we need the state tries of the current block and its parent in-memory
 // in order to process statediffs, we should avoid dereferencing roots until
 // its statediff and its child have been processed
-func (bc *BlockChain) allowedRootToBeDereferenced(root common.Hash) bool {
+func (bc *BlockChain) rootAllowedToBeDereferenced(root common.Hash) bool {
 	diffProcessedForSelfAndChildCount := 2
 	count := bc.stateDiffsProcessed[root]
 	return count >= diffProcessedForSelfAndChildCount

core/blockchain_test.go

@@ -2227,8 +2227,8 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) {
 
 func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
 	var (
-		numTxs= 1000
-		numBlocks= 1
+		numTxs    = 1000
+		numBlocks = 1
 	)
 	b.StopTimer()
 	b.ResetTimer()

eth/backend.go

@@ -179,11 +179,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 			EVMInterpreter:          config.EVMInterpreter,
 		}
 		cacheConfig = &core.CacheConfig{
-			TrieCleanLimit:      config.TrieCleanCache,
-			TrieCleanNoPrefetch: config.NoPrefetch,
-			TrieDirtyLimit:      config.TrieDirtyCache,
-			TrieDirtyDisabled:   config.NoPruning,
-			TrieTimeLimit:       config.TrieTimeout,
+			TrieCleanLimit:       config.TrieCleanCache,
+			TrieCleanNoPrefetch:  config.NoPrefetch,
+			TrieDirtyLimit:       config.TrieDirtyCache,
+			TrieDirtyDisabled:    config.NoPruning,
+			TrieTimeLimit:        config.TrieTimeout,
 			ProcessingStateDiffs: config.StateDiff,
 		}
 	)

eth/config.go

@@ -151,11 +151,11 @@ type Config struct {
 	// RPCGasCap is the global gas cap for eth-call variants.
 	RPCGasCap *big.Int `toml:",omitempty"`
 
-	StateDiff bool
-
 	// Checkpoint is a hardcoded checkpoint which can be nil.
 	Checkpoint *params.TrustedCheckpoint `toml:",omitempty"`
 
 	// CheckpointOracle is the configuration for checkpoint oracle.
 	CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"`
+
+	StateDiff bool
 }

statediff/api.go

@@ -0,0 +1,91 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package statediff
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// APIName is the namespace used for the state diffing service API
+const APIName = "statediff"
+
+// APIVersion is the version of the state diffing service API
+const APIVersion = "0.0.1"
+
+// PublicStateDiffAPI provides the a websocket service
+// that can be used to stream out state diffs as they
+// are produced by a full node
+type PublicStateDiffAPI struct {
+	sds IService
+}
+
+// NewPublicStateDiffAPI create a new state diff websocket streaming service.
+func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI {
+	return &PublicStateDiffAPI{
+		sds: sds,
+	}
+}
+
+// Stream is the public method to setup a subscription that fires off state-diff payloads as they are created
+func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, error) {
+	// ensure that the RPC connection supports subscriptions
+	notifier, supported := rpc.NotifierFromContext(ctx)
+	if !supported {
+		return nil, rpc.ErrNotificationsUnsupported
+	}
+
+	// create subscription and start waiting for statediff events
+	rpcSub := notifier.CreateSubscription()
+
+	go func() {
+		// subscribe to events from the state diff service
+		payloadChannel := make(chan Payload, chainEventChanSize)
+		quitChan := make(chan bool, 1)
+		api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan)
+		// loop and await state diff payloads and relay them to the subscriber with the notifier
+		for {
+			select {
+			case packet := <-payloadChannel:
+				if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil {
+					log.Error("Failed to send state diff packet; error: " + notifyErr.Error())
+					unSubErr := api.sds.Unsubscribe(rpcSub.ID)
+					if unSubErr != nil {
+						log.Error("Failed to unsubscribe from the state diff service; error: " + unSubErr.Error())
+					}
+					return
+				}
+			case err := <-rpcSub.Err():
+				if err != nil {
+					log.Error("State diff service rpcSub error: " + err.Error())
+					err = api.sds.Unsubscribe(rpcSub.ID)
+					if err != nil {
+						log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error())
+					}
+					return
+				}
+			case <-quitChan:
+				// don't need to unsubscribe, statediff service does so before sending the quit signal
+				return
+			}
+		}
+	}()
+
+	return rpcSub, nil
+}

statediff/builder.go

@@ -0,0 +1,319 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+var nullNode = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
+
+// Builder interface exposes the method for building a state diff between two blocks
+type Builder interface {
+	BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error)
+}
+
+type builder struct {
+	chainDB    ethdb.Database
+	config     Config
+	blockChain *core.BlockChain
+	stateCache state.Database
+}
+
+// NewBuilder is used to create a state diff builder
+func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) Builder {
+	return &builder{
+		chainDB:    db,
+		config:     config,
+		blockChain: blockChain,
+	}
+}
+
+// BuildStateDiff builds a StateDiff object from two blocks
+func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) {
+	// Generate tries for old and new states
+	sdb.stateCache = sdb.blockChain.StateCache()
+	oldTrie, err := sdb.stateCache.OpenTrie(oldStateRoot)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err)
+	}
+	newTrie, err := sdb.stateCache.OpenTrie(newStateRoot)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error creating trie for newStateRoot: %v", err)
+	}
+
+	// Find created accounts
+	oldIt := oldTrie.NodeIterator([]byte{})
+	newIt := newTrie.NodeIterator([]byte{})
+	creations, err := sdb.collectDiffNodes(oldIt, newIt)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error collecting creation diff nodes: %v", err)
+	}
+
+	// Find deleted accounts
+	oldIt = oldTrie.NodeIterator([]byte{})
+	newIt = newTrie.NodeIterator([]byte{})
+	deletions, err := sdb.collectDiffNodes(newIt, oldIt)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error collecting deletion diff nodes: %v", err)
+	}
+
+	// Find all the diffed keys
+	createKeys := sortKeys(creations)
+	deleteKeys := sortKeys(deletions)
+	updatedKeys := findIntersection(createKeys, deleteKeys)
+
+	// Build and return the statediff
+	updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error building diff for updated accounts: %v", err)
+	}
+	createdAccounts, err := sdb.buildDiffEventual(creations)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error building diff for created accounts: %v", err)
+	}
+	deletedAccounts, err := sdb.buildDiffEventual(deletions)
+	if err != nil {
+		return StateDiff{}, fmt.Errorf("error building diff for deleted accounts: %v", err)
+	}
+
+	return StateDiff{
+		BlockNumber:     blockNumber,
+		BlockHash:       blockHash,
+		CreatedAccounts: createdAccounts,
+		DeletedAccounts: deletedAccounts,
+		UpdatedAccounts: updatedAccounts,
+	}, nil
+}
+
+func (sdb *builder) isWatchedAddress(hashKey []byte) bool {
+	// If we aren't watching any addresses, we are watching everything
+	if len(sdb.config.WatchedAddresses) == 0 {
+		return true
+	}
+	for _, addrStr := range sdb.config.WatchedAddresses {
+		addr := common.HexToAddress(addrStr)
+		addrHashKey := crypto.Keccak256(addr[:])
+		if bytes.Equal(addrHashKey, hashKey) {
+			return true
+		}
+	}
+	return false
+}
+
+func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error) {
+	var diffAccounts = make(AccountsMap)
+	it, _ := trie.NewDifferenceIterator(a, b)
+	for {
+		log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash())
+		if it.Leaf() && sdb.isWatchedAddress(it.LeafKey()) {
+			leafKey := make([]byte, len(it.LeafKey()))
+			copy(leafKey, it.LeafKey())
+			leafKeyHash := common.BytesToHash(leafKey)
+			leafValue := make([]byte, len(it.LeafBlob()))
+			copy(leafValue, it.LeafBlob())
+			// lookup account state
+			var account state.Account
+			if err := rlp.DecodeBytes(leafValue, &account); err != nil {
+				return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err)
+			}
+			aw := accountWrapper{
+				Leaf:     true,
+				Account:  &account,
+				RawKey:   leafKey,
+				RawValue: leafValue,
+			}
+			if sdb.config.PathsAndProofs {
+				leafProof := make([][]byte, len(it.LeafProof()))
+				copy(leafProof, it.LeafProof())
+				leafPath := make([]byte, len(it.Path()))
+				copy(leafPath, it.Path())
+				aw.Proof = leafProof
+				aw.Path = leafPath
+			}
+			// record account to diffs (creation if we are looking at new - old; deletion if old - new)
+			log.Debug("Account lookup successful", "address", leafKeyHash, "account", account)
+			diffAccounts[leafKeyHash] = aw
+		} else if sdb.config.IntermediateNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) {
+			nodeKey := it.Hash()
+			node, err := sdb.stateCache.TrieDB().Node(nodeKey)
+			if err != nil {
+				return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err)
+			}
+			aw := accountWrapper{
+				Leaf:     false,
+				RawKey:   nodeKey.Bytes(),
+				RawValue: node,
+			}
+			log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node)
+			diffAccounts[nodeKey] = aw
+		}
+		cont := it.Next(true)
+		if !cont {
+			break
+		}
+	}
+
+	return diffAccounts, nil
+}
+
+func (sdb *builder) buildDiffEventual(accounts AccountsMap) ([]AccountDiff, error) {
+	accountDiffs := make([]AccountDiff, 0)
+	var err error
+	for _, val := range accounts {
+		// If account is not nil, we need to process storage diffs
+		var storageDiffs []StorageDiff
+		if val.Account != nil {
+			storageDiffs, err = sdb.buildStorageDiffsEventual(val.Account.Root)
+			if err != nil {
+				return nil, fmt.Errorf("failed building eventual storage diffs for %s\r\nerror: %v", common.BytesToHash(val.RawKey), err)
+			}
+		}
+		accountDiffs = append(accountDiffs, AccountDiff{
+			Leaf:    val.Leaf,
+			Key:     val.RawKey,
+			Value:   val.RawValue,
+			Proof:   val.Proof,
+			Path:    val.Path,
+			Storage: storageDiffs,
+		})
+	}
+
+	return accountDiffs, nil
+}
+
+func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) ([]AccountDiff, error) {
+	updatedAccounts := make([]AccountDiff, 0)
+	var err error
+	for _, val := range updatedKeys {
+		hashKey := common.HexToHash(val)
+		createdAcc := creations[hashKey]
+		deletedAcc := deletions[hashKey]
+		var storageDiffs []StorageDiff
+		if deletedAcc.Account != nil && createdAcc.Account != nil {
+			oldSR := deletedAcc.Account.Root
+			newSR := createdAcc.Account.Root
+			storageDiffs, err = sdb.buildStorageDiffsIncremental(oldSR, newSR)
+			if err != nil {
+				return nil, fmt.Errorf("failed building incremental storage diffs for %s\r\nerror: %v", hashKey.Hex(), err)
+			}
+		}
+		updatedAccounts = append(updatedAccounts, AccountDiff{
+			Leaf:    createdAcc.Leaf,
+			Key:     createdAcc.RawKey,
+			Value:   createdAcc.RawValue,
+			Proof:   createdAcc.Proof,
+			Path:    createdAcc.Path,
+			Storage: storageDiffs,
+		})
+		delete(creations, common.HexToHash(val))
+		delete(deletions, common.HexToHash(val))
+	}
+
+	return updatedAccounts, nil
+}
+
+func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, error) {
+	log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
+	stateCache := sdb.blockChain.StateCache()
+	sTrie, err := stateCache.OpenTrie(sr)
+	if err != nil {
+		log.Info("error in build storage diff eventual", "error", err)
+		return nil, err
+	}
+	it := sTrie.NodeIterator(make([]byte, 0))
+	return sdb.buildStorageDiffsFromTrie(it)
+}
+
+func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) ([]StorageDiff, error) {
+	log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
+	stateCache := sdb.blockChain.StateCache()
+
+	oldTrie, err := stateCache.OpenTrie(oldSR)
+	if err != nil {
+		return nil, err
+	}
+	newTrie, err := stateCache.OpenTrie(newSR)
+	if err != nil {
+		return nil, err
+	}
+
+	oldIt := oldTrie.NodeIterator(make([]byte, 0))
+	newIt := newTrie.NodeIterator(make([]byte, 0))
+	it, _ := trie.NewDifferenceIterator(oldIt, newIt)
+	return sdb.buildStorageDiffsFromTrie(it)
+}
+
+func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDiff, error) {
+	storageDiffs := make([]StorageDiff, 0)
+	for {
+		log.Debug("Iterating over state at path ", "path", pathToStr(it))
+		if it.Leaf() {
+			log.Debug("Found leaf in storage", "path", pathToStr(it))
+			leafKey := make([]byte, len(it.LeafKey()))
+			copy(leafKey, it.LeafKey())
+			leafValue := make([]byte, len(it.LeafBlob()))
+			copy(leafValue, it.LeafBlob())
+			sd := StorageDiff{
+				Leaf:  true,
+				Key:   leafKey,
+				Value: leafValue,
+			}
+			if sdb.config.PathsAndProofs {
+				leafProof := make([][]byte, len(it.LeafProof()))
+				copy(leafProof, it.LeafProof())
+				leafPath := make([]byte, len(it.Path()))
+				copy(leafPath, it.Path())
+				sd.Proof = leafProof
+				sd.Path = leafPath
+			}
+			storageDiffs = append(storageDiffs, sd)
+		} else if sdb.config.IntermediateNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) {
+			nodeKey := it.Hash()
+			node, err := sdb.stateCache.TrieDB().Node(nodeKey)
+			if err != nil {
+				return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err)
+			}
+			storageDiffs = append(storageDiffs, StorageDiff{
+				Leaf:  false,
+				Key:   nodeKey.Bytes(),
+				Value: node,
+			})
+			log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node)
+		}
+		cont := it.Next(true)
+		if !cont {
+			break
+		}
+	}
+
+	return storageDiffs, nil
+}

statediff/builder/builder.go

@@ -1,272 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package builder
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/trie"
-)
-
-type Builder interface {
-	BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error)
-}
-
-type builder struct {
-	chainDB    ethdb.Database
-	blockChain *core.BlockChain
-}
-
-type AccountsMap map[common.Hash]*state.Account
-
-func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) *builder {
-	return &builder{
-		chainDB:    db,
-		blockChain: blockChain,
-	}
-}
-
-func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) {
-	// Generate tries for old and new states
-	stateCache := sdb.blockChain.StateCache()
-	oldTrie, err := stateCache.OpenTrie(oldStateRoot)
-	if err != nil {
-		log.Error("Error creating trie for oldStateRoot", "error", err)
-		return nil, err
-	}
-	newTrie, err := stateCache.OpenTrie(newStateRoot)
-	if err != nil {
-		log.Error("Error creating trie for newStateRoot", "error", err)
-		return nil, err
-	}
-
-	// Find created accounts
-	oldIt := oldTrie.NodeIterator([]byte{})
-	newIt := newTrie.NodeIterator([]byte{})
-	creations, err := sdb.collectDiffNodes(oldIt, newIt)
-	if err != nil {
-		log.Error("Error collecting creation diff nodes", "error", err)
-		return nil, err
-	}
-
-	// Find deleted accounts
-	oldIt = oldTrie.NodeIterator([]byte{})
-	newIt = newTrie.NodeIterator([]byte{})
-	deletions, err := sdb.collectDiffNodes(newIt, oldIt)
-	if err != nil {
-		log.Error("Error collecting deletion diff nodes", "error", err)
-		return nil, err
-	}
-
-	// Find all the diffed keys
-	createKeys := sortKeys(creations)
-	deleteKeys := sortKeys(deletions)
-	updatedKeys := findIntersection(createKeys, deleteKeys)
-
-	// Build and return the statediff
-	updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys)
-	if err != nil {
-		log.Error("Error building diff for updated accounts", "error", err)
-		return nil, err
-	}
-	createdAccounts, err := sdb.buildDiffEventual(creations)
-	if err != nil {
-		log.Error("Error building diff for created accounts", "error", err)
-		return nil, err
-	}
-	deletedAccounts, err := sdb.buildDiffEventual(deletions)
-	if err != nil {
-		log.Error("Error building diff for deleted accounts", "error", err)
-		return nil, err
-	}
-
-	return &StateDiff{
-		BlockNumber:     blockNumber,
-		BlockHash:       blockHash,
-		CreatedAccounts: createdAccounts,
-		DeletedAccounts: deletedAccounts,
-		UpdatedAccounts: updatedAccounts,
-	}, nil
-}
-
-func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error) {
-	var diffAccounts = make(AccountsMap)
-	it, _ := trie.NewDifferenceIterator(a, b)
-
-	for {
-		log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash())
-		if it.Leaf() {
-			leafKey := make([]byte, len(it.LeafKey()))
-			copy(leafKey, it.LeafKey())
-			leafKeyHash := common.BytesToHash(leafKey)
-
-			// lookup account state
-			var account state.Account
-			if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil {
-				log.Error("Error looking up account via address", "address", leafKeyHash, "error", err)
-				return nil, err
-			}
-
-			// record account to diffs (creation if we are looking at new - old; deletion if old - new)
-			log.Debug("Account lookup successful", "address", leafKeyHash, "account", account)
-			diffAccounts[leafKeyHash] = &account
-		}
-		cont := it.Next(true)
-		if !cont {
-			break
-		}
-	}
-
-	return diffAccounts, nil
-}
-
-func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) {
-	accountDiffs := make(AccountDiffsMap)
-	for addr, val := range accounts {
-		sr := val.Root
-		storageDiffs, err := sdb.buildStorageDiffsEventual(sr)
-		if err != nil {
-			log.Error("Failed building eventual storage diffs", "Address", addr, "error", err)
-			return nil, err
-		}
-
-		codeHash := hexutil.Encode(val.CodeHash)
-		hexRoot := val.Root.Hex()
-		nonce := DiffUint64{Value: &val.Nonce}
-		balance := DiffBigInt{Value: val.Balance}
-		contractRoot := DiffString{Value: &hexRoot}
-		accountDiffs[addr] = AccountDiff{
-			Nonce:        nonce,
-			Balance:      balance,
-			CodeHash:     codeHash,
-			ContractRoot: contractRoot,
-			Storage:      storageDiffs,
-		}
-	}
-
-	return accountDiffs, nil
-}
-
-func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) {
-	updatedAccounts := make(AccountDiffsMap)
-	for _, val := range updatedKeys {
-		createdAcc := creations[common.HexToHash(val)]
-		deletedAcc := deletions[common.HexToHash(val)]
-		oldSR := deletedAcc.Root
-		newSR := createdAcc.Root
-		if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil {
-			log.Error("Failed building storage diffs", "Address", val, "error", err)
-			return nil, err
-		} else {
-			nonce := DiffUint64{Value: &createdAcc.Nonce}
-			balance := DiffBigInt{Value: createdAcc.Balance}
-			codeHash := hexutil.Encode(createdAcc.CodeHash)
-
-			nHexRoot := createdAcc.Root.Hex()
-			contractRoot := DiffString{Value: &nHexRoot}
-
-			updatedAccounts[common.HexToHash(val)] = AccountDiff{
-				Nonce:        nonce,
-				Balance:      balance,
-				CodeHash:     codeHash,
-				ContractRoot: contractRoot,
-				Storage:      storageDiffs,
-			}
-			delete(creations, common.HexToHash(val))
-			delete(deletions, common.HexToHash(val))
-		}
-	}
-	return updatedAccounts, nil
-}
-
-func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffStorage, error) {
-	log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
-	stateCache := sdb.blockChain.StateCache()
-	sTrie, err := stateCache.OpenTrie(sr)
-	if err != nil {
-		log.Info("error in build storage diff eventual", "error", err)
-		return nil, err
-	}
-	it := sTrie.NodeIterator(make([]byte, 0))
-	storageDiffs := buildStorageDiffsFromTrie(it)
-	return storageDiffs, nil
-}
-
-func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffStorage, error) {
-	log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
-	stateCache := sdb.blockChain.StateCache()
-
-	oldTrie, err := stateCache.OpenTrie(oldSR)
-	if err != nil {
-		return nil, err
-	}
-	newTrie, err := stateCache.OpenTrie(newSR)
-	if err != nil {
-		return nil, err
-	}
-
-	oldIt := oldTrie.NodeIterator(make([]byte, 0))
-	newIt := newTrie.NodeIterator(make([]byte, 0))
-	it, _ := trie.NewDifferenceIterator(oldIt, newIt)
-	storageDiffs := buildStorageDiffsFromTrie(it)
-
-	return storageDiffs, nil
-}
-
-func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage {
-	storageDiffs := make(map[string]DiffStorage)
-	for {
-		log.Debug("Iterating over state at path ", "path", pathToStr(it))
-		if it.Leaf() {
-			log.Debug("Found leaf in storage", "path", pathToStr(it))
-			path := pathToStr(it)
-			storageKey := hexutil.Encode(it.LeafKey())
-			storageValue := hexutil.Encode(it.LeafBlob())
-			storageDiffs[path] = DiffStorage{
-				Key:   &storageKey,
-				Value: &storageValue,
-			}
-		}
-
-		cont := it.Next(true)
-		if !cont {
-			break
-		}
-	}
-
-	return storageDiffs
-}
-
-func (sdb *builder) addressByPath(path []byte) (*common.Address, error) {
-	log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)))
-	if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)); err != nil {
-		log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err)
-		return nil, err
-	} else {
-		addr := common.BytesToAddress(addrBytes)
-		log.Debug("Address found", "Address", addr)
-		return &addr, nil
-	}
-}

statediff/builder/builder_test.go

@@ -1,365 +0,0 @@
-package builder_test
-
-import (
-	"bytes"
-	"math/big"
-	"reflect"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/consensus/ethash"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/params"
-	b "github.com/ethereum/go-ethereum/statediff/builder"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers"
-)
-
-var (
-	testdb = ethdb.NewMemDatabase()
-
-	testBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-	testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
-	bankLeafKey     = testhelpers.AddressToLeafKey(testBankAddress)
-	testBankFunds   = big.NewInt(100000000)
-	genesis         = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds)
-
-	account1Key, _                                 = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
-	account2Key, _                                 = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
-	account1Addr                                   = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
-	account1LeafKey                                = testhelpers.AddressToLeafKey(account1Addr)
-	account2Addr                                   = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
-	account2LeafKey                                = testhelpers.AddressToLeafKey(account2Addr)
-	contractCode                                   = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029")
-	contractAddr                                   common.Address
-	contractLeafKey                                common.Hash
-	emptyAccountDiffEventualMap                    = make(b.AccountDiffsMap)
-	emptyAccountDiffIncrementalMap                 = make(b.AccountDiffsMap)
-	block0Hash, block1Hash, block2Hash, block3Hash common.Hash
-	block0, block1, block2, block3                 *types.Block
-	builder                                        b.Builder
-	miningReward                                   = int64(2000000000000000000)
-	burnAddress                                    = common.HexToAddress("0x0")
-	burnLeafKey                                    = testhelpers.AddressToLeafKey(burnAddress)
-)
-
-func TestBuilder(t *testing.T) {
-	_, blockMap, chain := makeChain(3, genesis)
-	contractLeafKey = testhelpers.AddressToLeafKey(contractAddr)
-	defer chain.Stop()
-	block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661")
-	block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a")
-	block2Hash = common.HexToHash("0xde75663f36a8497b4bdda2a4b52bd9540b705a2728c7391c59b8cb2cde5a2feb")
-	block3Hash = common.HexToHash("0x76c6d0e39285cee40d5e5fadc6141ca88c8ab8bd1a15d46717205af2efbb4a3c")
-
-	block0 = blockMap[block0Hash]
-	block1 = blockMap[block1Hash]
-	block2 = blockMap[block2Hash]
-	block3 = blockMap[block3Hash]
-	builder = b.NewBuilder(testdb, chain)
-
-	type arguments struct {
-		oldStateRoot common.Hash
-		newStateRoot common.Hash
-		blockNumber  int64
-		blockHash    common.Hash
-	}
-
-	var (
-		balanceChange10000      = int64(10000)
-		balanceChange1000       = int64(1000)
-		block1BankBalance       = int64(99990000)
-		block1Account1Balance   = int64(10000)
-		block2Account2Balance   = int64(1000)
-		nonce0                  = uint64(0)
-		nonce1                  = uint64(1)
-		nonce2                  = uint64(2)
-		nonce3                  = uint64(3)
-		originalContractRoot    = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
-		contractContractRoot    = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
-		newContractRoot         = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070"
-		originalStorageLocation = common.HexToHash("0")
-		originalStorageKey      = crypto.Keccak256Hash(originalStorageLocation[:]).String()
-		updatedStorageLocation  = common.HexToHash("2")
-		updatedStorageKey       = crypto.Keccak256Hash(updatedStorageLocation[:]).String()
-		originalStorageValue    = "0x01"
-		updatedStorageValue     = "0x03"
-	)
-
-	var tests = []struct {
-		name              string
-		startingArguments arguments
-		expected          *b.StateDiff
-	}{
-		{
-			"testEmptyDiff",
-			arguments{
-				oldStateRoot: block0.Root(),
-				newStateRoot: block0.Root(),
-				blockNumber:  block0.Number().Int64(),
-				blockHash:    block0Hash,
-			},
-			&b.StateDiff{
-				BlockNumber:     block0.Number().Int64(),
-				BlockHash:       block0Hash,
-				CreatedAccounts: emptyAccountDiffEventualMap,
-				DeletedAccounts: emptyAccountDiffEventualMap,
-				UpdatedAccounts: emptyAccountDiffIncrementalMap,
-			},
-		},
-		{
-			"testBlock1",
-			//10000 transferred from testBankAddress to account1Addr
-			arguments{
-				oldStateRoot: block0.Root(),
-				newStateRoot: block1.Root(),
-				blockNumber:  block1.Number().Int64(),
-				blockHash:    block1Hash,
-			},
-			&b.StateDiff{
-				BlockNumber: block1.Number().Int64(),
-				BlockHash:   block1.Hash(),
-				CreatedAccounts: b.AccountDiffsMap{
-					account1LeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce0},
-						Balance:      b.DiffBigInt{Value: big.NewInt(balanceChange10000)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-					burnLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce0},
-						Balance:      b.DiffBigInt{Value: big.NewInt(miningReward)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-				},
-				DeletedAccounts: emptyAccountDiffEventualMap,
-				UpdatedAccounts: b.AccountDiffsMap{
-					bankLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce1},
-						Balance:      b.DiffBigInt{Value: big.NewInt(testBankFunds.Int64() - balanceChange10000)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-				},
-			},
-		},
-		{
-			"testBlock2",
-			//1000 transferred from testBankAddress to account1Addr
-			//1000 transferred from account1Addr to account2Addr
-			arguments{
-				oldStateRoot: block1.Root(),
-				newStateRoot: block2.Root(),
-				blockNumber:  block2.Number().Int64(),
-				blockHash:    block2Hash,
-			},
-			&b.StateDiff{
-				BlockNumber: block2.Number().Int64(),
-				BlockHash:   block2.Hash(),
-				CreatedAccounts: b.AccountDiffsMap{
-					account2LeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce0},
-						Balance:      b.DiffBigInt{Value: big.NewInt(balanceChange1000)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-					contractLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce1},
-						Balance:      b.DiffBigInt{Value: big.NewInt(0)},
-						CodeHash:     "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea",
-						ContractRoot: b.DiffString{Value: &contractContractRoot},
-						Storage: map[string]b.DiffStorage{
-							originalStorageKey: {
-								Key:   &originalStorageKey,
-								Value: &originalStorageValue},
-						},
-					},
-				},
-				DeletedAccounts: emptyAccountDiffEventualMap,
-				UpdatedAccounts: b.AccountDiffsMap{
-					bankLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce2},
-						Balance:      b.DiffBigInt{Value: big.NewInt(block1BankBalance - balanceChange1000)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-					account1LeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce2},
-						Balance:      b.DiffBigInt{Value: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-					burnLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce0},
-						Balance:      b.DiffBigInt{Value: big.NewInt(miningReward + miningReward)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-				},
-			},
-		},
-		{
-			"testBlock3",
-			//the contract's storage is changed
-			//and the block is mined by account 2
-			arguments{
-				oldStateRoot: block2.Root(),
-				newStateRoot: block3.Root(),
-				blockNumber:  block3.Number().Int64(),
-				blockHash:    block3.Hash(),
-			},
-			&b.StateDiff{
-				BlockNumber:     block3.Number().Int64(),
-				BlockHash:       block3.Hash(),
-				CreatedAccounts: b.AccountDiffsMap{},
-				DeletedAccounts: emptyAccountDiffEventualMap,
-				UpdatedAccounts: b.AccountDiffsMap{
-					account2LeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce0},
-						Balance:      b.DiffBigInt{Value: big.NewInt(block2Account2Balance + miningReward)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-					contractLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce1},
-						Balance:      b.DiffBigInt{Value: big.NewInt(0)},
-						CodeHash:     "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea",
-						ContractRoot: b.DiffString{Value: &newContractRoot},
-						Storage: map[string]b.DiffStorage{
-							"0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace": {
-								Key:   &updatedStorageKey,
-								Value: &updatedStorageValue},
-						},
-					},
-					bankLeafKey: {
-						Nonce:        b.DiffUint64{Value: &nonce3},
-						Balance:      b.DiffBigInt{Value: big.NewInt(99989000)},
-						CodeHash:     "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-						ContractRoot: b.DiffString{Value: &originalContractRoot},
-						Storage:      map[string]b.DiffStorage{},
-					},
-				},
-			},
-		},
-	}
-
-	for _, test := range tests {
-		arguments := test.startingArguments
-		diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash)
-		if err != nil {
-			t.Error(err)
-		}
-		fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"}
-
-		for _, field := range fields {
-			reflectionOfDiff := reflect.ValueOf(diff)
-			diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field)
-
-			reflectionOfExpected := reflect.ValueOf(test.expected)
-			expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field)
-
-			diffValueInterface := diffValue.Interface()
-			expectedValueInterface := expectedValue.Interface()
-
-			if !equals(diffValueInterface, expectedValueInterface) {
-				t.Logf("Test failed: %s", test.name)
-				t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface)
-			}
-		}
-	}
-}
-
-func equals(actual, expected interface{}) (success bool) {
-	if actualByteSlice, ok := actual.([]byte); ok {
-		if expectedByteSlice, ok := expected.([]byte); ok {
-			return bytes.Equal(actualByteSlice, expectedByteSlice)
-		}
-	}
-
-	return reflect.DeepEqual(actual, expected)
-}
-
-// makeChain creates a chain of n blocks starting at and including parent.
-// the returned hash chain is ordered head->parent. In addition, every 3rd block
-// contains a transaction and every 5th an uncle to allow testing correct block
-// reassembly.
-func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) {
-	blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen)
-	headers := make([]*types.Header, len(blocks))
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	chain, _ := core.NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
-
-	hashes := make([]common.Hash, n+1)
-	hashes[len(hashes)-1] = parent.Hash()
-	blockm := make(map[common.Hash]*types.Block, n+1)
-	blockm[parent.Hash()] = parent
-	for i, b := range blocks {
-		hashes[len(hashes)-i-2] = b.Hash()
-		blockm[b.Hash()] = b
-	}
-	return hashes, blockm, chain
-}
-
-func testChainGen(i int, block *core.BlockGen) {
-	signer := types.HomesteadSigner{}
-	switch i {
-	case 0:
-		// In block 1, the test bank sends account #1 some ether.
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey)
-		block.AddTx(tx)
-	case 1:
-		// In block 2, the test bank sends some more ether to account #1.
-		// account1Addr passes it on to account #2.
-		// account1Addr creates a test contract.
-		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
-		nonce := block.TxNonce(account1Addr)
-		tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key)
-		nonce++
-		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key)
-		contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592
-		block.AddTx(tx1)
-		block.AddTx(tx2)
-		block.AddTx(tx3)
-	case 2:
-		// Block 3 is empty but was mined by account #2.
-		block.SetCoinbase(account2Addr)
-		//get function: 60cd2685
-		//put function: c16431b9
-		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003")
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey)
-		block.AddTx(tx)
-	}
-}
-
-/*
-contract test {
-
-    uint256[100] data;
-
-	constructor() public {
-		data = [1];
-	}
-
-    function Put(uint256 addr, uint256 value) {
-        data[addr] = value;
-    }
-
-    function Get(uint256 addr) constant returns (uint256 value) {
-        return data[addr];
-    }
-}
-*/

statediff/builder/struct.go

@@ -1,79 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package builder
-
-import (
-	"encoding/json"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-type AccountDiffsMap map[common.Hash]AccountDiff
-type StateDiff struct {
-	BlockNumber     int64           `json:"blockNumber"      gencodec:"required"`
-	BlockHash       common.Hash     `json:"blockHash"        gencodec:"required"`
-	CreatedAccounts AccountDiffsMap `json:"createdAccounts"  gencodec:"required"`
-	DeletedAccounts AccountDiffsMap `json:"deletedAccounts"  gencodec:"required"`
-	UpdatedAccounts AccountDiffsMap `json:"updatedAccounts"  gencodec:"required"`
-
-	encoded []byte
-	err     error
-}
-
-func (self *StateDiff) ensureEncoded() {
-	if self.encoded == nil && self.err == nil {
-		self.encoded, self.err = json.Marshal(self)
-	}
-}
-
-// Implement Encoder interface for StateDiff
-func (sd *StateDiff) Length() int {
-	sd.ensureEncoded()
-	return len(sd.encoded)
-}
-
-// Implement Encoder interface for StateDiff
-func (sd *StateDiff) Encode() ([]byte, error) {
-	sd.ensureEncoded()
-	return sd.encoded, sd.err
-}
-
-type AccountDiff struct {
-	Nonce        DiffUint64             `json:"nonce"         gencodec:"required"`
-	Balance      DiffBigInt             `json:"balance"       gencodec:"required"`
-	CodeHash     string                 `json:"codeHash"      gencodec:"required"`
-	ContractRoot DiffString             `json:"contractRoot"  gencodec:"required"`
-	Storage      map[string]DiffStorage `json:"storage"       gencodec:"required"`
-}
-
-type DiffStorage struct {
-	Key   *string `json:"key" gencodec:"optional"`
-	Value *string `json:"value"  gencodec:"optional"`
-}
-type DiffString struct {
-	Value *string `json:"value"  gencodec:"optional"`
-}
-type DiffUint64 struct {
-	Value *uint64 `json:"value"  gencodec:"optional"`
-}
-type DiffBigInt struct {
-	Value *big.Int `json:"value"  gencodec:"optional"`
-}

statediff/builder_test.go

@@ -0,0 +1,564 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package statediff_test
+
+import (
+	"bytes"
+	"math/big"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/testhelpers"
+)
+
+var (
+	contractLeafKey                common.Hash
+	emptyAccountDiffEventualMap    = make([]statediff.AccountDiff, 0)
+	emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0)
+	block0, block1, block2, block3 *types.Block
+	builder                        statediff.Builder
+	miningReward                   = int64(2000000000000000000)
+	burnAddress                    = common.HexToAddress("0x0")
+	burnLeafKey                    = testhelpers.AddressToLeafKey(burnAddress)
+
+	block0Hash              = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661")
+	block1Hash              = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a")
+	block2Hash              = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5")
+	block3Hash              = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a")
+	balanceChange10000      = int64(10000)
+	balanceChange1000       = int64(1000)
+	block1BankBalance       = int64(99990000)
+	block1Account1Balance   = int64(10000)
+	block2Account2Balance   = int64(1000)
+	nonce0                  = uint64(0)
+	nonce1                  = uint64(1)
+	nonce2                  = uint64(2)
+	nonce3                  = uint64(3)
+	originalContractRoot    = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+	contractContractRoot    = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+	newContractRoot         = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070"
+	originalStorageLocation = common.HexToHash("0")
+	originalStorageKey      = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes()
+	updatedStorageLocation  = common.HexToHash("2")
+	updatedStorageKey       = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes()
+	originalStorageValue    = common.Hex2Bytes("01")
+	updatedStorageValue     = common.Hex2Bytes("03")
+
+	account1, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  big.NewInt(balanceChange10000),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	burnAccount1, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  big.NewInt(miningReward),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	bankAccount1, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce1,
+		Balance:  big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	account2, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  big.NewInt(balanceChange1000),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	contractAccount, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce1,
+		Balance:  big.NewInt(0),
+		CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(),
+		Root:     common.HexToHash(contractContractRoot),
+	})
+	bankAccount2, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce2,
+		Balance:  big.NewInt(block1BankBalance - balanceChange1000),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	account3, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce2,
+		Balance:  big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	burnAccount2, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  big.NewInt(miningReward + miningReward),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	account4, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  big.NewInt(block2Account2Balance + miningReward),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+	contractAccount2, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce1,
+		Balance:  big.NewInt(0),
+		CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(),
+		Root:     common.HexToHash(newContractRoot),
+	})
+	bankAccount3, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce3,
+		Balance:  big.NewInt(99989000),
+		CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+		Root:     common.HexToHash(originalContractRoot),
+	})
+)
+
+type arguments struct {
+	oldStateRoot common.Hash
+	newStateRoot common.Hash
+	blockNumber  *big.Int
+	blockHash    common.Hash
+}
+
+func TestBuilder(t *testing.T) {
+	_, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis)
+	contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+	defer chain.Stop()
+	block0 = blockMap[block0Hash]
+	block1 = blockMap[block1Hash]
+	block2 = blockMap[block2Hash]
+	block3 = blockMap[block3Hash]
+	config := statediff.Config{
+		PathsAndProofs:    true,
+		IntermediateNodes: false,
+	}
+	builder = statediff.NewBuilder(testhelpers.Testdb, chain, config)
+
+	var tests = []struct {
+		name              string
+		startingArguments arguments
+		expected          *statediff.StateDiff
+	}{
+		{
+			"testEmptyDiff",
+			arguments{
+				oldStateRoot: block0.Root(),
+				newStateRoot: block0.Root(),
+				blockNumber:  block0.Number(),
+				blockHash:    block0Hash,
+			},
+			&statediff.StateDiff{
+				BlockNumber:     block0.Number(),
+				BlockHash:       block0Hash,
+				CreatedAccounts: emptyAccountDiffEventualMap,
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: emptyAccountDiffIncrementalMap,
+			},
+		},
+		{
+			"testBlock1",
+			//10000 transferred from testBankAddress to account1Addr
+			arguments{
+				oldStateRoot: block0.Root(),
+				newStateRoot: block1.Root(),
+				blockNumber:  block1.Number(),
+				blockHash:    block1Hash,
+			},
+			&statediff.StateDiff{
+				BlockNumber: block1.Number(),
+				BlockHash:   block1.Hash(),
+				CreatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   burnLeafKey.Bytes(),
+						Value: burnAccount1,
+						Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+							{248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+					{
+						Leaf:  true,
+						Key:   testhelpers.Account1LeafKey.Bytes(),
+						Value: account1,
+						Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+							{248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   testhelpers.BankLeafKey.Bytes(),
+						Value: bankAccount1,
+						Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+							{248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 1, 132, 5, 245, 185, 240, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+			},
+		},
+		{
+			"testBlock2",
+			//1000 transferred from testBankAddress to account1Addr
+			//1000 transferred from account1Addr to account2Addr
+			arguments{
+				oldStateRoot: block1.Root(),
+				newStateRoot: block2.Root(),
+				blockNumber:  block2.Number(),
+				blockHash:    block2Hash,
+			},
+			&statediff.StateDiff{
+				BlockNumber: block2.Number(),
+				BlockHash:   block2.Hash(),
+				CreatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   contractLeafKey.Bytes(),
+						Value: contractAccount,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 130, 30, 37, 86, 162, 144, 200, 100, 5, 248, 22, 10, 45, 102, 32, 66, 164, 49, 186, 69, 107, 157, 178, 101, 199, 155, 184, 55, 192, 75, 229, 240, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}},
+						Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16},
+						Storage: []statediff.StorageDiff{
+							{
+								Leaf:  true,
+								Key:   originalStorageKey,
+								Value: originalStorageValue,
+								Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}},
+								Path:  []byte{2, 9, 0, 13, 14, 12, 13, 9, 5, 4, 8, 11, 6, 2, 10, 8, 13, 6, 0, 3, 4, 5, 10, 9, 8, 8, 3, 8, 6, 15, 12, 8, 4, 11, 10, 6, 11, 12, 9, 5, 4, 8, 4, 0, 0, 8, 15, 6, 3, 6, 2, 15, 9, 3, 1, 6, 0, 14, 15, 3, 14, 5, 6, 3, 16},
+							},
+						},
+					},
+					{
+						Leaf:  true,
+						Key:   testhelpers.Account2LeafKey.Bytes(),
+						Value: account2,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   testhelpers.BankLeafKey.Bytes(),
+						Value: bankAccount2,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 2, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+					{
+						Leaf:  true,
+						Key:   burnLeafKey.Bytes(),
+						Value: burnAccount2,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 55, 130, 218, 206, 157, 144, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+					{
+						Leaf:  true,
+						Key:   testhelpers.Account1LeafKey.Bytes(),
+						Value: account3,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+			},
+		},
+		{
+			"testBlock3",
+			//the contract's storage is changed
+			//and the block is mined by account 2
+			arguments{
+				oldStateRoot: block2.Root(),
+				newStateRoot: block3.Root(),
+				blockNumber:  block3.Number(),
+				blockHash:    block3.Hash(),
+			},
+			&statediff.StateDiff{
+				BlockNumber:     block3.Number(),
+				BlockHash:       block3.Hash(),
+				CreatedAccounts: []statediff.AccountDiff{},
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   testhelpers.BankLeafKey.Bytes(),
+						Value: bankAccount3,
+						Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+					{
+						Leaf:  true,
+						Key:   contractLeafKey.Bytes(),
+						Value: contractAccount2,
+						Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 113, 224, 209, 75, 43, 147, 229, 199, 249, 116, 142, 105, 225, 254, 95, 23, 73, 138, 28, 58, 195, 206, 194, 159, 150, 175, 19, 215, 248, 164, 224, 112, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}},
+						Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16},
+						Storage: []statediff.StorageDiff{
+							{
+								Leaf:  true,
+								Key:   updatedStorageKey,
+								Value: updatedStorageValue,
+								Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
+									{226, 160, 48, 87, 135, 250, 18, 168, 35, 224, 242, 183, 99, 28, 196, 27, 59, 168, 130, 139, 51, 33, 202, 129, 17, 17, 250, 117, 205, 58, 163, 187, 90, 206, 3}},
+								Path: []byte{4, 0, 5, 7, 8, 7, 15, 10, 1, 2, 10, 8, 2, 3, 14, 0, 15, 2, 11, 7, 6, 3, 1, 12, 12, 4, 1, 11, 3, 11, 10, 8, 8, 2, 8, 11, 3, 3, 2, 1, 12, 10, 8, 1, 1, 1, 1, 1, 15, 10, 7, 5, 12, 13, 3, 10, 10, 3, 11, 11, 5, 10, 12, 14, 16},
+							},
+						},
+					},
+					{
+						Leaf:  true,
+						Key:   testhelpers.Account2LeafKey.Bytes(),
+						Value: account4,
+						Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		arguments := test.startingArguments
+		diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash)
+		if err != nil {
+			t.Error(err)
+		}
+		receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+		if err != nil {
+			t.Error(err)
+		}
+		expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+		sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+		sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+		if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+			t.Logf("Test failed: %s", test.name)
+			t.Errorf("actual state diff rlp: %+v\nexpected state diff rlp: %+v", receivedStateDiffRlp, expectedStateDiffRlp)
+		}
+	}
+}
+
+func TestBuilderWithWatchedAddressList(t *testing.T) {
+	_, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis)
+	contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr)
+	defer chain.Stop()
+	block0 = blockMap[block0Hash]
+	block1 = blockMap[block1Hash]
+	block2 = blockMap[block2Hash]
+	block3 = blockMap[block3Hash]
+	config := statediff.Config{
+		PathsAndProofs:    true,
+		IntermediateNodes: false,
+		WatchedAddresses:  []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()},
+	}
+	builder = statediff.NewBuilder(testhelpers.Testdb, chain, config)
+
+	var tests = []struct {
+		name              string
+		startingArguments arguments
+		expected          *statediff.StateDiff
+	}{
+		{
+			"testEmptyDiff",
+			arguments{
+				oldStateRoot: block0.Root(),
+				newStateRoot: block0.Root(),
+				blockNumber:  block0.Number(),
+				blockHash:    block0Hash,
+			},
+			&statediff.StateDiff{
+				BlockNumber:     block0.Number(),
+				BlockHash:       block0Hash,
+				CreatedAccounts: emptyAccountDiffEventualMap,
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: emptyAccountDiffIncrementalMap,
+			},
+		},
+		{
+			"testBlock1",
+			//10000 transferred from testBankAddress to account1Addr
+			arguments{
+				oldStateRoot: block0.Root(),
+				newStateRoot: block1.Root(),
+				blockNumber:  block1.Number(),
+				blockHash:    block1Hash,
+			},
+			&statediff.StateDiff{
+				BlockNumber: block1.Number(),
+				BlockHash:   block1.Hash(),
+				CreatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   testhelpers.Account1LeafKey.Bytes(),
+						Value: account1,
+						Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+							{248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: []statediff.AccountDiff{},
+			},
+		},
+		{
+			"testBlock2",
+			//1000 transferred from testBankAddress to account1Addr
+			//1000 transferred from account1Addr to account2Addr
+			arguments{
+				oldStateRoot: block1.Root(),
+				newStateRoot: block2.Root(),
+				blockNumber:  block2.Number(),
+				blockHash:    block2Hash,
+			},
+			&statediff.StateDiff{
+				BlockNumber: block2.Number(),
+				BlockHash:   block2.Hash(),
+				CreatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   contractLeafKey.Bytes(),
+						Value: contractAccount,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 130, 30, 37, 86, 162, 144, 200, 100, 5, 248, 22, 10, 45, 102, 32, 66, 164, 49, 186, 69, 107, 157, 178, 101, 199, 155, 184, 55, 192, 75, 229, 240, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}},
+						Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16},
+						Storage: []statediff.StorageDiff{
+							{
+								Leaf:  true,
+								Key:   originalStorageKey,
+								Value: originalStorageValue,
+								Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}},
+								Path:  []byte{2, 9, 0, 13, 14, 12, 13, 9, 5, 4, 8, 11, 6, 2, 10, 8, 13, 6, 0, 3, 4, 5, 10, 9, 8, 8, 3, 8, 6, 15, 12, 8, 4, 11, 10, 6, 11, 12, 9, 5, 4, 8, 4, 0, 0, 8, 15, 6, 3, 6, 2, 15, 9, 3, 1, 6, 0, 14, 15, 3, 14, 5, 6, 3, 16},
+							},
+						},
+					},
+				},
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   testhelpers.Account1LeafKey.Bytes(),
+						Value: account3,
+						Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+						Path:    []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16},
+						Storage: []statediff.StorageDiff{},
+					},
+				},
+			},
+		},
+		{
+			"testBlock3",
+			//the contract's storage is changed
+			//and the block is mined by account 2
+			arguments{
+				oldStateRoot: block2.Root(),
+				newStateRoot: block3.Root(),
+				blockNumber:  block3.Number(),
+				blockHash:    block3.Hash(),
+			},
+			&statediff.StateDiff{
+				BlockNumber:     block3.Number(),
+				BlockHash:       block3.Hash(),
+				CreatedAccounts: []statediff.AccountDiff{},
+				DeletedAccounts: emptyAccountDiffEventualMap,
+				UpdatedAccounts: []statediff.AccountDiff{
+					{
+						Leaf:  true,
+						Key:   contractLeafKey.Bytes(),
+						Value: contractAccount2,
+						Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128},
+							{248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 113, 224, 209, 75, 43, 147, 229, 199, 249, 116, 142, 105, 225, 254, 95, 23, 73, 138, 28, 58, 195, 206, 194, 159, 150, 175, 19, 215, 248, 164, 224, 112, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}},
+						Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16},
+						Storage: []statediff.StorageDiff{
+							{
+								Leaf:  true,
+								Key:   updatedStorageKey,
+								Value: updatedStorageValue,
+								Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128},
+									{226, 160, 48, 87, 135, 250, 18, 168, 35, 224, 242, 183, 99, 28, 196, 27, 59, 168, 130, 139, 51, 33, 202, 129, 17, 17, 250, 117, 205, 58, 163, 187, 90, 206, 3}},
+								Path: []byte{4, 0, 5, 7, 8, 7, 15, 10, 1, 2, 10, 8, 2, 3, 14, 0, 15, 2, 11, 7, 6, 3, 1, 12, 12, 4, 1, 11, 3, 11, 10, 8, 8, 2, 8, 11, 3, 3, 2, 1, 12, 10, 8, 1, 1, 1, 1, 1, 15, 10, 7, 5, 12, 13, 3, 10, 10, 3, 11, 11, 5, 10, 12, 14, 16},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		arguments := test.startingArguments
+		diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash)
+		if err != nil {
+			t.Error(err)
+		}
+		receivedStateDiffRlp, err := rlp.EncodeToBytes(diff)
+		if err != nil {
+			t.Error(err)
+		}
+		expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+		sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] })
+		sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] })
+		if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) {
+			t.Logf("Test failed: %s", test.name)
+			t.Errorf("actual state diff rlp: %+v\nexpected state diff rlp: %+v", receivedStateDiffRlp, expectedStateDiffRlp)
+		}
+	}
+}
+
+/*
+contract test {
+
+    uint256[100] data;
+
+	constructor() public {
+		data = [1];
+	}
+
+    function Put(uint256 addr, uint256 value) {
+        data[addr] = value;
+    }
+
+    function Get(uint256 addr) constant returns (uint256 value) {
+        return data[addr];
+    }
+}
+*/

statediff/config.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,80 +14,12 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
 package statediff
 
-import "fmt"
-
+// Config is used to carry in parameters from CLI configuration
 type Config struct {
-	Mode StateDiffMode // Mode for storing diffs
-	Path string        // Path for storing diffs
-}
-
-type StateDiffMode int
-
-const (
-	CSV StateDiffMode = iota
-	IPLD
-	LDB
-	SQL
-)
-
-func (mode StateDiffMode) IsValid() bool {
-	return mode >= IPLD && mode <= SQL
-}
-
-// String implements the stringer interface.
-func (mode StateDiffMode) String() string {
-	switch mode {
-	case CSV:
-		return "csv"
-	case IPLD:
-		return "ipfs"
-	case LDB:
-		return "ldb"
-	case SQL:
-		return "sql"
-	default:
-		return "unknown"
-	}
-}
-
-func NewMode(mode string) (StateDiffMode, error) {
-	stateDiffMode := StateDiffMode(0)
-	err := stateDiffMode.UnmarshalText([]byte(mode))
-	return stateDiffMode, err
-}
-
-func (mode StateDiffMode) MarshalText() ([]byte, error) {
-	switch mode {
-	case CSV:
-		return []byte("ipfs"), nil
-	case IPLD:
-		return []byte("ipfs"), nil
-	case LDB:
-		return []byte("ldb"), nil
-	case SQL:
-		return []byte("sql"), nil
-	default:
-		return nil, fmt.Errorf("unknown state diff storage mode %d", mode)
-	}
-}
-
-func (mode *StateDiffMode) UnmarshalText(text []byte) error {
-	switch string(text) {
-	case "csv":
-		*mode = CSV
-	case "ipfs":
-		*mode = IPLD
-	case "ldb":
-		*mode = LDB
-	case "sql":
-		*mode = SQL
-	default:
-		return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text)
-	}
-	return nil
+	PathsAndProofs    bool
+	IntermediateNodes bool
+	StreamBlock       bool
+	WatchedAddresses  []string
 }

statediff/config_test.go

@@ -1,24 +0,0 @@
-package statediff_test
-
-import (
-	"testing"
-
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers"
-)
-
-func TestNewMode(t *testing.T) {
-	mode, err := statediff.NewMode("csv")
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	if mode != statediff.CSV {
-		t.Error()
-	}
-
-	_, err = statediff.NewMode("not a real mode")
-	if err == nil {
-		t.Error("Expected an error, and got nil.")
-	}
-}

statediff/doc.go

@@ -0,0 +1,58 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+Package statediff provides an auxiliary service that processes state diff objects from incoming chain events,
+relaying the objects to any rpc subscriptions.
+
+This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go
+
+The service is spun up using the below CLI flags
+--statediff: boolean flag, turns on the service
+--statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs.
+--statediff.intermediatenodes: boolean flag, tells service to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only.
+--statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes.
+--statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3
+
+If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default.
+
+The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node).
+
+e.g.
+
+$ ./geth --statediff --statediff.streamblock --ws --syncmode "full"
+
+This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service.
+Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs.
+
+Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method,
+with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream".
+
+e.g.
+
+cli, _ := rpc.Dial("ipcPathOrWsURL")
+stateDiffPayloadChan := make(chan statediff.Payload, 20000)
+rpcSub, err := cli.Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"})
+for {
+	select {
+	case stateDiffPayload := <- stateDiffPayloadChan:
+		processPayload(stateDiffPayload)
+	case err := <- rpcSub.Err():
+		log.Error(err)
+	}
+}
+*/
+package statediff

statediff/extractor/extractor.go

@@ -1,51 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package extractor
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/statediff/builder"
-	"github.com/ethereum/go-ethereum/statediff/publisher"
-)
-
-type Extractor interface {
-	ExtractStateDiff(parent, current types.Block) (string, error)
-}
-
-type extractor struct {
-	Builder   builder.Builder     // Interface for building state diff objects from two blocks
-	Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS)
-}
-
-func NewExtractor(builder builder.Builder, publisher publisher.Publisher) *extractor {
-	return &extractor{
-		Builder:   builder,
-		Publisher: publisher,
-	}
-}
-
-func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) {
-	stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash())
-	if err != nil {
-		return "", err
-	}
-
-	return e.Publisher.PublishStateDiff(stateDiff)
-}

statediff/extractor/extractor_test.go

@@ -1,123 +0,0 @@
-package extractor_test
-
-import (
-	"bytes"
-	"math/big"
-	"math/rand"
-	"reflect"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	b "github.com/ethereum/go-ethereum/statediff/builder"
-	e "github.com/ethereum/go-ethereum/statediff/extractor"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers/mocks"
-)
-
-var publisher mocks.Publisher
-var builder mocks.Builder
-var currentBlockNumber *big.Int
-var parentBlock, currentBlock *types.Block
-var expectedStateDiff b.StateDiff
-var extractor e.Extractor
-var err error
-
-func TestExtractor(t *testing.T) {
-	publisher = mocks.Publisher{}
-	builder = mocks.Builder{}
-	extractor = e.NewExtractor(&builder, &publisher)
-	if err != nil {
-		t.Error(err)
-	}
-
-	blockNumber := rand.Int63()
-	parentBlockNumber := big.NewInt(blockNumber - int64(1))
-	currentBlockNumber = big.NewInt(blockNumber)
-	parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil)
-	currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil)
-
-	expectedStateDiff = b.StateDiff{
-		BlockNumber:     blockNumber,
-		BlockHash:       currentBlock.Hash(),
-		CreatedAccounts: nil,
-		DeletedAccounts: nil,
-		UpdatedAccounts: nil,
-	}
-
-	testBuildStateDiffStruct(t)
-	testBuildStateDiffErrorHandling(t)
-	testPublishingStateDiff(t)
-	testPublisherErrorHandling(t)
-}
-
-func testBuildStateDiffStruct(t *testing.T) {
-	builder.SetStateDiffToBuild(&expectedStateDiff)
-
-	_, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !equals(builder.OldStateRoot, parentBlock.Root()) {
-		t.Error()
-	}
-	if !equals(builder.NewStateRoot, currentBlock.Root()) {
-		t.Error()
-	}
-	if !equals(builder.BlockNumber, currentBlockNumber.Int64()) {
-		t.Error()
-	}
-	if !equals(builder.BlockHash, currentBlock.Hash()) {
-		t.Error()
-	}
-}
-
-func testBuildStateDiffErrorHandling(t *testing.T) {
-	builder.SetBuilderError(mocks.Error)
-
-	_, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock)
-	if err == nil {
-		t.Error(err)
-	}
-
-	if !equals(err, mocks.Error) {
-		t.Error()
-	}
-	builder.SetBuilderError(nil)
-}
-
-func testPublishingStateDiff(t *testing.T) {
-	builder.SetStateDiffToBuild(&expectedStateDiff)
-
-	_, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !equals(publisher.StateDiff, &expectedStateDiff) {
-		t.Error()
-	}
-}
-
-func testPublisherErrorHandling(t *testing.T) {
-	publisher.SetPublisherError(mocks.Error)
-
-	_, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock)
-	if err == nil {
-		t.Error("Expected an error, but it didn't occur.")
-	}
-	if !equals(err, mocks.Error) {
-		t.Error()
-	}
-
-	publisher.SetPublisherError(nil)
-}
-
-func equals(actual, expected interface{}) (success bool) {
-	if actualByteSlice, ok := actual.([]byte); ok {
-		if expectedByteSlice, ok := expected.([]byte); ok {
-			return bytes.Equal(actualByteSlice, expectedByteSlice)
-		}
-	}
-
-	return reflect.DeepEqual(actual, expected)
-}

statediff/helpers.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
 // Contains a batch of utility type declarations used by the tests. As the node
 // operates on unique types, a lot of them are needed to check various features.
 
-package builder
+package statediff
 
 import (
 	"sort"
@@ -37,43 +37,8 @@ func sortKeys(data AccountsMap) []string {
 	return keys
 }
 
-func findIntersection(a, b []string) []string {
-	lenA := len(a)
-	lenB := len(b)
-	iOfA, iOfB := 0, 0
-	updates := make([]string, 0)
-	if iOfA >= lenA || iOfB >= lenB {
-		return updates
-	}
-	for {
-		switch strings.Compare(a[iOfA], b[iOfB]) {
-		// a[iOfA] < b[iOfB]
-		case -1:
-			iOfA++
-			if iOfA >= lenA {
-				return updates
-			}
-			// a[iOfA] == b[iOfB]
-		case 0:
-			updates = append(updates, a[iOfA])
-			iOfA++
-			iOfB++
-			if iOfA >= lenA || iOfB >= lenB {
-				return updates
-			}
-			// a[iOfA] > b[iOfB]
-		case 1:
-			iOfB++
-			if iOfB >= lenB {
-				return updates
-			}
-		}
-	}
-
-}
-
-func pathToStr(it trie.NodeIterator) string {
-	path := it.Path()
+// bytesToNiblePath converts the byte representation of a path to its string representation
+func bytesToNiblePath(path []byte) string {
 	if hasTerm(path) {
 		path = path[:len(path)-1]
 	}
@@ -88,24 +53,46 @@ func pathToStr(it trie.NodeIterator) string {
 	return nibblePath
 }
 
-// Duplicated from trie/encoding.go
-func hexToKeyBytes(hex []byte) []byte {
-	if hasTerm(hex) {
-		hex = hex[:len(hex)-1]
+// findIntersection finds the set of strings from both arrays that are equivalent (same key as same index)
+// this is used to find which keys have been both "deleted" and "created" i.e. they were "updated".
+func findIntersection(a, b []string) []string {
+	lenA := len(a)
+	lenB := len(b)
+	iOfA, iOfB := 0, 0
+	updates := make([]string, 0)
+	if iOfA >= lenA || iOfB >= lenB {
+		return updates
 	}
-	if len(hex)&1 != 0 {
-		panic("can't convert hex key of odd length")
+	for {
+		switch strings.Compare(a[iOfA], b[iOfB]) {
+		// -1 when a[iOfA] < b[iOfB]
+		case -1:
+			iOfA++
+			if iOfA >= lenA {
+				return updates
+			}
+			// 0 when a[iOfA] == b[iOfB]
+		case 0:
+			updates = append(updates, a[iOfA])
+			iOfA++
+			iOfB++
+			if iOfA >= lenA || iOfB >= lenB {
+				return updates
+			}
+			// 1 when a[iOfA] > b[iOfB]
+		case 1:
+			iOfB++
+			if iOfB >= lenB {
+				return updates
+			}
+		}
 	}
-	key := make([]byte, (len(hex)+1)/2)
-	decodeNibbles(hex, key)
 
-	return key
 }
 
-func decodeNibbles(nibbles []byte, bytes []byte) {
-	for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 {
-		bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1]
-	}
+// pathToStr converts the NodeIterator path to a string representation
+func pathToStr(it trie.NodeIterator) string {
+	return bytesToNiblePath(it.Path())
 }
 
 // hasTerm returns whether a hex key has the terminator flag.

statediff/publisher/csv.go

@@ -1,130 +0,0 @@
-package publisher
-
-import (
-	"encoding/csv"
-	"os"
-	"path/filepath"
-	"strconv"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/statediff/builder"
-)
-
-var (
-	Headers = []string{
-		"blockNumber", "blockHash", "accountAction", "codeHash",
-		"nonceValue", "balanceValue", "contractRoot", "storageDiffPaths",
-		"accountLeafKey", "storageKey", "storageValue",
-	}
-
-	timeStampFormat      = "20060102150405.00000"
-	deletedAccountAction = "deleted"
-	createdAccountAction = "created"
-	updatedAccountAction = "updated"
-)
-
-func createCSVFilePath(path, blockNumber string) string {
-	now := time.Now()
-	timeStamp := now.Format(timeStampFormat)
-	suffix := timeStamp + "-" + blockNumber
-	filePath := filepath.Join(path, suffix)
-	filePath = filePath + ".csv"
-	return filePath
-}
-
-func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) {
-	filePath := createCSVFilePath(p.Config.Path, strconv.FormatInt(sd.BlockNumber, 10))
-
-	file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return "", err
-	}
-	defer file.Close()
-
-	writer := csv.NewWriter(file)
-	defer writer.Flush()
-
-	var data [][]string
-	data = append(data, Headers)
-	data = append(data, accumulateAccountRows(sd)...)
-	for _, value := range data {
-		err := writer.Write(value)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	return filePath, nil
-}
-
-func accumulateAccountRows(sd builder.StateDiff) [][]string {
-	var accountRows [][]string
-	for accountAddr, accountDiff := range sd.CreatedAccounts {
-		formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction)
-
-		accountRows = append(accountRows, formattedAccountData...)
-	}
-
-	for accountAddr, accountDiff := range sd.UpdatedAccounts {
-		formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction)
-
-		accountRows = append(accountRows, formattedAccountData...)
-	}
-
-	for accountAddr, accountDiff := range sd.DeletedAccounts {
-		formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction)
-
-		accountRows = append(accountRows, formattedAccountData...)
-	}
-
-	return accountRows
-}
-
-func formatAccountData(accountAddr common.Hash, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string {
-	blockNumberString := strconv.FormatInt(sd.BlockNumber, 10)
-	blockHash := sd.BlockHash.String()
-	codeHash := accountDiff.CodeHash
-	nonce := strconv.FormatUint(*accountDiff.Nonce.Value, 10)
-	balance := accountDiff.Balance.Value.String()
-	newContractRoot := accountDiff.ContractRoot.Value
-	address := accountAddr.String()
-	var result [][]string
-
-	if len(accountDiff.Storage) > 0 {
-		for storagePath, storage := range accountDiff.Storage {
-			formattedAccountData := []string{
-				blockNumberString,
-				blockHash,
-				accountAction,
-				codeHash,
-				nonce,
-				balance,
-				*newContractRoot,
-				storagePath,
-				address,
-				*storage.Key,
-				*storage.Value,
-			}
-
-			result = append(result, formattedAccountData)
-		}
-	} else {
-		formattedAccountData := []string{
-			blockNumberString,
-			blockHash,
-			accountAction,
-			codeHash,
-			nonce,
-			balance,
-			*newContractRoot,
-			"",
-			address,
-			"",
-			"",
-		}
-		result = append(result, formattedAccountData)
-	}
-
-	return result
-}

statediff/publisher/publisher.go

@@ -1,48 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package publisher
-
-import (
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/ethereum/go-ethereum/statediff/builder"
-)
-
-type Publisher interface {
-	PublishStateDiff(sd *builder.StateDiff) (string, error)
-}
-
-type publisher struct {
-	Config statediff.Config
-}
-
-func NewPublisher(config statediff.Config) (*publisher, error) {
-	return &publisher{
-		Config: config,
-	}, nil
-}
-
-func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) {
-	switch p.Config.Mode {
-	case statediff.CSV:
-		return p.publishStateDiffToCSV(*sd)
-	default:
-		return p.publishStateDiffToCSV(*sd)
-	}
-}

statediff/publisher/publisher_test.go

@@ -1,316 +0,0 @@
-package publisher_test
-
-import (
-	"bytes"
-	"encoding/csv"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"reflect"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/ethereum/go-ethereum/statediff/builder"
-	p "github.com/ethereum/go-ethereum/statediff/publisher"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers"
-	"github.com/pkg/errors"
-)
-
-var (
-	tempDir        = os.TempDir()
-	testFilePrefix = "test-statediff"
-	publisher      p.Publisher
-	dir            string
-	err            error
-)
-
-var expectedCreatedAccountRow = []string{
-	strconv.FormatInt(testhelpers.BlockNumber, 10),
-	testhelpers.BlockHash,
-	"created",
-	testhelpers.CodeHash,
-	strconv.FormatUint(testhelpers.NewNonceValue, 10),
-	strconv.FormatInt(testhelpers.NewBalanceValue, 10),
-	testhelpers.ContractRoot,
-	testhelpers.StoragePath,
-	testhelpers.ContractLeafKey.Hex(),
-	"0000000000000000000000000000000000000000000000000000000000000001",
-	testhelpers.StorageValue,
-}
-
-var expectedCreatedAccountWithoutStorageUpdateRow = []string{
-	strconv.FormatInt(testhelpers.BlockNumber, 10),
-	testhelpers.BlockHash,
-	"created",
-	testhelpers.CodeHash,
-	strconv.FormatUint(testhelpers.NewNonceValue, 10),
-	strconv.FormatInt(testhelpers.NewBalanceValue, 10),
-	testhelpers.ContractRoot,
-	"",
-	testhelpers.AnotherContractLeafKey.Hex(),
-	"",
-	"",
-}
-
-var expectedUpdatedAccountRow = []string{
-	strconv.FormatInt(testhelpers.BlockNumber, 10),
-	testhelpers.BlockHash,
-	"updated",
-	testhelpers.CodeHash,
-	strconv.FormatUint(testhelpers.NewNonceValue, 10),
-	strconv.FormatInt(testhelpers.NewBalanceValue, 10),
-	testhelpers.ContractRoot,
-	testhelpers.StoragePath,
-	testhelpers.ContractLeafKey.Hex(),
-	"0000000000000000000000000000000000000000000000000000000000000001",
-	testhelpers.StorageValue,
-}
-
-var expectedDeletedAccountRow = []string{
-	strconv.FormatInt(testhelpers.BlockNumber, 10),
-	testhelpers.BlockHash,
-	"deleted",
-	testhelpers.CodeHash,
-	strconv.FormatUint(testhelpers.NewNonceValue, 10),
-	strconv.FormatInt(testhelpers.NewBalanceValue, 10),
-	testhelpers.ContractRoot,
-	testhelpers.StoragePath,
-	testhelpers.ContractLeafKey.Hex(),
-	"0000000000000000000000000000000000000000000000000000000000000001",
-	testhelpers.StorageValue,
-}
-
-func TestPublisher(t *testing.T) {
-	dir, err = ioutil.TempDir(tempDir, testFilePrefix)
-	if err != nil {
-		t.Error(err)
-	}
-	config := statediff.Config{
-		Path: dir,
-		Mode: statediff.CSV,
-	}
-	publisher, err = p.NewPublisher(config)
-	if err != nil {
-		t.Error(err)
-	}
-
-	type Test func(t *testing.T)
-
-	var tests = []Test{
-		testFileName,
-		testColumnHeaders,
-		testAccountDiffs,
-		testWhenNoDiff,
-		testDefaultPublisher,
-		testDefaultDirectory,
-	}
-
-	for _, test := range tests {
-		test(t)
-		err := removeFilesFromDir(dir)
-		if err != nil {
-			t.Errorf("Error removing files from temp dir: %s", dir)
-		}
-	}
-}
-
-func removeFilesFromDir(dir string) error {
-	files, err := filepath.Glob(filepath.Join(dir, "*"))
-	if err != nil {
-		return err
-	}
-
-	for _, file := range files {
-		err = os.RemoveAll(file)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func testFileName(t *testing.T) {
-	fileName, err := publisher.PublishStateDiff(&testhelpers.TestStateDiff)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	if !strings.HasPrefix(fileName, dir) {
-		t.Errorf(testhelpers.TestFailureFormatString, t.Name(), dir, fileName)
-	}
-	blockNumberWithFileExt := strconv.FormatInt(testhelpers.BlockNumber, 10) + ".csv"
-	if !strings.HasSuffix(fileName, blockNumberWithFileExt) {
-		t.Errorf(testhelpers.TestFailureFormatString, t.Name(), blockNumberWithFileExt, fileName)
-	}
-}
-
-func testColumnHeaders(t *testing.T) {
-	_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	file, err := getTestDiffFile(dir)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	lines, err := csv.NewReader(file).ReadAll()
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if len(lines) < 1 {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[0], p.Headers) {
-		t.Error()
-	}
-}
-
-func testAccountDiffs(t *testing.T) {
-	// it persists the created, updated and deleted account diffs to a CSV file
-	_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	file, err := getTestDiffFile(dir)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	lines, err := csv.NewReader(file).ReadAll()
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if len(lines) <= 3 {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[1], expectedCreatedAccountRow) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[2], expectedCreatedAccountWithoutStorageUpdateRow) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[3], expectedUpdatedAccountRow) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[4], expectedDeletedAccountRow) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-}
-
-func testWhenNoDiff(t *testing.T) {
-	//it creates an empty CSV when there is no diff
-	emptyDiff := builder.StateDiff{}
-	_, err = publisher.PublishStateDiff(&emptyDiff)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	file, err := getTestDiffFile(dir)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	lines, err := csv.NewReader(file).ReadAll()
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	if !equals(len(lines), 1) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-}
-
-func testDefaultPublisher(t *testing.T) {
-	//it defaults to publishing state diffs to a CSV file when no mode is configured
-	config := statediff.Config{Path: dir}
-	publisher, err = p.NewPublisher(config)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	file, err := getTestDiffFile(dir)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	lines, err := csv.NewReader(file).ReadAll()
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(len(lines), 5) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[0], p.Headers) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-}
-
-func testDefaultDirectory(t *testing.T) {
-	//it defaults to publishing CSV files in the current directory when no path is configured
-	config := statediff.Config{}
-	publisher, err = p.NewPublisher(config)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	err := os.Chdir(dir)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	_, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	file, err := getTestDiffFile(dir)
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-
-	lines, err := csv.NewReader(file).ReadAll()
-	if err != nil {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(len(lines), 5) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-	if !equals(lines[0], p.Headers) {
-		t.Errorf(testhelpers.ErrorFormatString, t.Name(), err)
-	}
-}
-
-func getTestDiffFile(dir string) (*os.File, error) {
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		return nil, err
-	}
-	if len(files) == 0 {
-		return nil, errors.New("There are 0 files.")
-	}
-
-	fileName := files[0].Name()
-	filePath := filepath.Join(dir, fileName)
-
-	return os.Open(filePath)
-}
-
-func equals(actual, expected interface{}) (success bool) {
-	if actualByteSlice, ok := actual.([]byte); ok {
-		if expectedByteSlice, ok := expected.([]byte); ok {
-			return bytes.Equal(actualByteSlice, expectedByteSlice)
-		}
-	}
-
-	return reflect.DeepEqual(actual, expected)
-}

statediff/service.go

@@ -0,0 +1,274 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package statediff
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+const chainEventChanSize = 20000
+
+type blockChain interface {
+	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
+	GetBlockByHash(hash common.Hash) *types.Block
+	AddToStateDiffProcessedCollection(hash common.Hash)
+	GetReceiptsByHash(hash common.Hash) types.Receipts
+}
+
+// IService is the state-diffing service interface
+type IService interface {
+	// APIs(), Protocols(), Start() and Stop()
+	node.Service
+	// Main event loop for processing state diffs
+	Loop(chainEventCh chan core.ChainEvent)
+	// Method to subscribe to receive state diff processing output
+	Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool)
+	// Method to unsubscribe from state diff processing
+	Unsubscribe(id rpc.ID) error
+}
+
+// Service is the underlying struct for the state diffing service
+type Service struct {
+	// Used to sync access to the Subscriptions
+	sync.Mutex
+	// Used to build the state diff objects
+	Builder Builder
+	// Used to subscribe to chain events (blocks)
+	BlockChain blockChain
+	// Used to signal shutdown of the service
+	QuitChan chan bool
+	// A mapping of rpc.IDs to their subscription channels
+	Subscriptions map[rpc.ID]Subscription
+	// Cache the last block so that we can avoid having to lookup the next block's parent
+	lastBlock *types.Block
+	// Whether or not the block data is streamed alongside the state diff data in the subscription payload
+	StreamBlock bool
+	// Whether or not we have any subscribers; only if we do, do we processes state diffs
+	subscribers int32
+}
+
+// NewStateDiffService creates a new StateDiffingService
+func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Config) (*Service, error) {
+	return &Service{
+		Mutex:         sync.Mutex{},
+		BlockChain:    blockChain,
+		Builder:       NewBuilder(db, blockChain, config),
+		QuitChan:      make(chan bool),
+		Subscriptions: make(map[rpc.ID]Subscription),
+		StreamBlock:   config.StreamBlock,
+	}, nil
+}
+
+// Protocols exports the services p2p protocols, this service has none
+func (sds *Service) Protocols() []p2p.Protocol {
+	return []p2p.Protocol{}
+}
+
+// APIs returns the RPC descriptors the StateDiffingService offers
+func (sds *Service) APIs() []rpc.API {
+	return []rpc.API{
+		{
+			Namespace: APIName,
+			Version:   APIVersion,
+			Service:   NewPublicStateDiffAPI(sds),
+			Public:    true,
+		},
+	}
+}
+
+// Loop is the main processing method
+func (sds *Service) Loop(chainEventCh chan core.ChainEvent) {
+	chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh)
+	defer chainEventSub.Unsubscribe()
+	errCh := chainEventSub.Err()
+	for {
+		select {
+		//Notify chain event channel of events
+		case chainEvent := <-chainEventCh:
+			log.Debug("Event received from chainEventCh", "event", chainEvent)
+			// if we don't have any subscribers, do not process a statediff
+			if atomic.LoadInt32(&sds.subscribers) == 0 {
+				log.Debug("Currently no subscribers to the statediffing service; processing is halted")
+				continue
+			}
+			currentBlock := chainEvent.Block
+			parentHash := currentBlock.ParentHash()
+			var parentBlock *types.Block
+			if sds.lastBlock != nil && bytes.Equal(sds.lastBlock.Hash().Bytes(), currentBlock.ParentHash().Bytes()) {
+				parentBlock = sds.lastBlock
+			} else {
+				parentBlock = sds.BlockChain.GetBlockByHash(parentHash)
+			}
+			sds.lastBlock = currentBlock
+			if parentBlock == nil {
+				log.Error(fmt.Sprintf("Parent block is nil, skipping this block (%d)", currentBlock.Number()))
+				continue
+			}
+			if err := sds.processStateDiff(currentBlock, parentBlock); err != nil {
+				log.Error(fmt.Sprintf("Error building statediff for block %d; error: ", currentBlock.Number()) + err.Error())
+			}
+		case err := <-errCh:
+			log.Warn("Error from chain event subscription, breaking loop", "error", err)
+			sds.close()
+			return
+		case <-sds.QuitChan:
+			log.Info("Quitting the statediffing process")
+			sds.close()
+			return
+		}
+	}
+}
+
+// processStateDiff method builds the state diff payload from the current and parent block and sends it to listening subscriptions
+func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) error {
+	stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash())
+	if err != nil {
+		return err
+	}
+	stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+	if err != nil {
+		return err
+	}
+	payload := Payload{
+		StateDiffRlp: stateDiffRlp,
+		Err:          err,
+	}
+	if sds.StreamBlock {
+		blockBuff := new(bytes.Buffer)
+		if err = currentBlock.EncodeRLP(blockBuff); err != nil {
+			return err
+		}
+		payload.BlockRlp = blockBuff.Bytes()
+		receiptBuff := new(bytes.Buffer)
+		receipts := sds.BlockChain.GetReceiptsByHash(currentBlock.Hash())
+		if err = rlp.Encode(receiptBuff, receipts); err != nil {
+			println(err.Error())
+			return err
+		}
+		payload.ReceiptsRlp = receiptBuff.Bytes()
+	}
+
+	// If we have any rpc subscriptions listening in, send the data to them
+	sds.send(payload)
+	return nil
+}
+
+// Subscribe is used by the API to subscribe to the StateDiffingService loop
+func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) {
+	log.Info("Subscribing to the statediff service")
+	if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) {
+		log.Info("State diffing subscription received; beginning statediff processing")
+	}
+	sds.Lock()
+	sds.Subscriptions[id] = Subscription{
+		PayloadChan: sub,
+		QuitChan:    quitChan,
+	}
+	sds.Unlock()
+}
+
+// Unsubscribe is used to unsubscribe to the StateDiffingService loop
+func (sds *Service) Unsubscribe(id rpc.ID) error {
+	log.Info("Unsubscribing from the statediff service")
+	sds.Lock()
+	_, ok := sds.Subscriptions[id]
+	if !ok {
+		return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id)
+	}
+	delete(sds.Subscriptions, id)
+	if len(sds.Subscriptions) == 0 {
+		if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) {
+			log.Info("No more subscriptions; halting statediff processing")
+		}
+	}
+	sds.Unlock()
+	return nil
+}
+
+// Start is used to begin the StateDiffingService
+func (sds *Service) Start(*p2p.Server) error {
+	log.Info("Starting statediff service")
+
+	chainEventCh := make(chan core.ChainEvent, chainEventChanSize)
+	go sds.Loop(chainEventCh)
+
+	return nil
+}
+
+// Stop is used to close down the StateDiffingService
+func (sds *Service) Stop() error {
+	log.Info("Stopping statediff service")
+	close(sds.QuitChan)
+	return nil
+}
+
+// send is used to fan out and serve the statediff payload to all subscriptions
+func (sds *Service) send(payload Payload) {
+	sds.Lock()
+	for id, sub := range sds.Subscriptions {
+		select {
+		case sub.PayloadChan <- payload:
+			log.Info(fmt.Sprintf("sending state diff payload to subscription %s", id))
+		default:
+			log.Info(fmt.Sprintf("unable to send payload to subscription %s; channel has no receiver", id))
+			// in this case, try to close the bad subscription and remove it
+			select {
+			case sub.QuitChan <- true:
+				log.Info(fmt.Sprintf("closing subscription %s", id))
+			default:
+				log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id))
+			}
+			delete(sds.Subscriptions, id)
+		}
+	}
+	// If after removing all bad subscriptions we have none left, halt processing
+	if len(sds.Subscriptions) == 0 {
+		if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) {
+			log.Info("No more subscriptions; halting statediff processing")
+		}
+	}
+	sds.Unlock()
+}
+
+// close is used to close all listening subscriptions
+func (sds *Service) close() {
+	sds.Lock()
+	for id, sub := range sds.Subscriptions {
+		select {
+		case sub.QuitChan <- true:
+			log.Info(fmt.Sprintf("closing subscription %s", id))
+		default:
+			log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id))
+		}
+		delete(sds.Subscriptions, id)
+	}
+	sds.Unlock()
+}

statediff/service/service.go

@@ -1,120 +0,0 @@
-package service
-
-import (
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
-	b "github.com/ethereum/go-ethereum/statediff/builder"
-	e "github.com/ethereum/go-ethereum/statediff/extractor"
-	p "github.com/ethereum/go-ethereum/statediff/publisher"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/log"
-)
-
-type BlockChain interface {
-	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
-	GetBlockByHash(hash common.Hash) *types.Block
-	AddToStateDiffProcessedCollection(hash common.Hash)
-}
-
-type StateDiffService struct {
-	Builder    *b.Builder
-	Extractor  e.Extractor
-	BlockChain BlockChain
-}
-
-func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config statediff.Config) (*StateDiffService, error) {
-	builder := b.NewBuilder(db, blockChain)
-	publisher, err := p.NewPublisher(config)
-	if err != nil {
-		return nil, err
-	}
-
-	extractor := e.NewExtractor(builder, publisher)
-	return &StateDiffService{
-		BlockChain: blockChain,
-		Extractor:  extractor,
-	}, nil
-}
-
-func (StateDiffService) Protocols() []p2p.Protocol {
-	return []p2p.Protocol{}
-}
-
-func (StateDiffService) APIs() []rpc.API {
-	return []rpc.API{}
-}
-
-func (sds *StateDiffService) Loop(chainEventCh chan core.ChainEvent) {
-	chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh)
-	defer chainEventSub.Unsubscribe()
-
-	blocksCh := make(chan *types.Block, 10)
-	errCh := chainEventSub.Err()
-	quitCh := make(chan struct{})
-
-	go func() {
-	HandleChainEventChLoop:
-		for {
-			select {
-			//Notify chain event channel of events
-			case chainEvent := <-chainEventCh:
-				log.Debug("Event received from chainEventCh", "event", chainEvent)
-				blocksCh <- chainEvent.Block
-			//if node stopped
-			case err := <-errCh:
-				log.Warn("Error from chain event subscription, breaking loop.", "error", err)
-				break HandleChainEventChLoop
-			}
-		}
-		close(quitCh)
-	}()
-
-	//loop through chain events until no more
-HandleBlockChLoop:
-	for {
-		select {
-		case block := <-blocksCh:
-			currentBlock := block
-			parentHash := currentBlock.ParentHash()
-			parentBlock := sds.BlockChain.GetBlockByHash(parentHash)
-			if parentBlock == nil {
-				log.Error("Parent block is nil, skipping this block",
-					"parent block hash", parentHash.String(),
-					"current block number", currentBlock.Number())
-				break HandleBlockChLoop
-			}
-
-			stateDiffLocation, err := sds.Extractor.ExtractStateDiff(*parentBlock, *currentBlock)
-			if err != nil {
-				log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err)
-			} else {
-				log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation)
-				sds.BlockChain.AddToStateDiffProcessedCollection(parentBlock.Root())
-				sds.BlockChain.AddToStateDiffProcessedCollection(currentBlock.Root())
-			}
-		case <-quitCh:
-			log.Debug("Quitting the statediff block channel")
-			return
-		}
-	}
-}
-
-func (sds *StateDiffService) Start(server *p2p.Server) error {
-	log.Info("Starting statediff service")
-
-	chainEventCh := make(chan core.ChainEvent, 10)
-	go sds.Loop(chainEventCh)
-
-	return nil
-}
-
-func (StateDiffService) Stop() error {
-	log.Info("Stopping statediff service")
-	return nil
-}

statediff/service/service_test.go

@@ -1,107 +0,0 @@
-package service_test
-
-import (
-	"math/big"
-	"math/rand"
-	"reflect"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	s "github.com/ethereum/go-ethereum/statediff/service"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers/mocks"
-)
-
-func TestServiceLoop(t *testing.T) {
-	testErrorInChainEventLoop(t)
-	testErrorInBlockLoop(t)
-}
-
-var (
-	eventsChannel = make(chan core.ChainEvent, 1)
-
-	parentHeader1 = types.Header{Number: big.NewInt(rand.Int63())}
-	parentHeader2 = types.Header{Number: big.NewInt(rand.Int63())}
-
-	parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil)
-	parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil)
-
-	parentHash1 = parentBlock1.Hash()
-	parentHash2 = parentBlock2.Hash()
-
-	header1 = types.Header{ParentHash: parentHash1}
-	header2 = types.Header{ParentHash: parentHash2}
-	header3 = types.Header{ParentHash: common.HexToHash("parent hash")}
-
-	block1 = types.NewBlock(&header1, nil, nil, nil)
-	block2 = types.NewBlock(&header2, nil, nil, nil)
-	block3 = types.NewBlock(&header3, nil, nil, nil)
-
-	event1 = core.ChainEvent{Block: block1}
-	event2 = core.ChainEvent{Block: block2}
-	event3 = core.ChainEvent{Block: block3}
-)
-
-func testErrorInChainEventLoop(t *testing.T) {
-	//the first chain event causes and error (in blockchain mock)
-	extractor := mocks.Extractor{}
-
-	blockChain := mocks.BlockChain{}
-	service := s.StateDiffService{
-		Builder:    nil,
-		Extractor:  &extractor,
-		BlockChain: &blockChain,
-	}
-
-	blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2})
-	blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3})
-	service.Loop(eventsChannel)
-
-	//parent and current blocks are passed to the extractor
-	expectedCurrentBlocks := []types.Block{*block1, *block2}
-	if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) {
-		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks)
-	}
-	expectedParentBlocks := []types.Block{*parentBlock1, *parentBlock2}
-	if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) {
-		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks)
-	}
-
-	//look up the parent block from its hash
-	expectedHashes := []common.Hash{block1.ParentHash(), block2.ParentHash()}
-	if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) {
-		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes)
-	}
-}
-
-func testErrorInBlockLoop(t *testing.T) {
-	//second block's parent block can't be found
-	extractor := mocks.Extractor{}
-
-	blockChain := mocks.BlockChain{}
-	service := s.StateDiffService{
-		Builder:    nil,
-		Extractor:  &extractor,
-		BlockChain: &blockChain,
-	}
-
-	blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil})
-	blockChain.SetChainEvents([]core.ChainEvent{event1, event2})
-	service.Loop(eventsChannel)
-
-	//only the first current block (and it's parent) are passed to the extractor
-	expectedCurrentBlocks := []types.Block{*block1}
-	if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) {
-		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks)
-	}
-	expectedParentBlocks := []types.Block{*parentBlock1}
-	if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) {
-		t.Error("Test failure:", t.Name())
-		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks)
-	}
-}

statediff/service_test.go

@@ -0,0 +1,196 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package statediff_test
+
+import (
+	"bytes"
+	"math/big"
+	"math/rand"
+	"reflect"
+	"sync"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/testhelpers/mocks"
+)
+
+func TestServiceLoop(t *testing.T) {
+	testErrorInChainEventLoop(t)
+	testErrorInBlockLoop(t)
+}
+
+var (
+	eventsChannel = make(chan core.ChainEvent, 1)
+
+	parentRoot1   = common.HexToHash("0x01")
+	parentRoot2   = common.HexToHash("0x02")
+	parentHeader1 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot1}
+	parentHeader2 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot2}
+
+	parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil)
+	parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil)
+
+	parentHash1 = parentBlock1.Hash()
+	parentHash2 = parentBlock2.Hash()
+
+	testRoot1 = common.HexToHash("0x03")
+	testRoot2 = common.HexToHash("0x04")
+	testRoot3 = common.HexToHash("0x04")
+	header1   = types.Header{ParentHash: parentHash1, Root: testRoot1}
+	header2   = types.Header{ParentHash: parentHash2, Root: testRoot2}
+	header3   = types.Header{ParentHash: common.HexToHash("parent hash"), Root: testRoot3}
+
+	testBlock1 = types.NewBlock(&header1, nil, nil, nil)
+	testBlock2 = types.NewBlock(&header2, nil, nil, nil)
+	testBlock3 = types.NewBlock(&header3, nil, nil, nil)
+
+	receiptRoot1  = common.HexToHash("0x05")
+	receiptRoot2  = common.HexToHash("0x06")
+	receiptRoot3  = common.HexToHash("0x07")
+	testReceipts1 = []*types.Receipt{types.NewReceipt(receiptRoot1.Bytes(), false, 1000), types.NewReceipt(receiptRoot2.Bytes(), false, 2000)}
+	testReceipts2 = []*types.Receipt{types.NewReceipt(receiptRoot3.Bytes(), false, 3000)}
+
+	event1 = core.ChainEvent{Block: testBlock1}
+	event2 = core.ChainEvent{Block: testBlock2}
+	event3 = core.ChainEvent{Block: testBlock3}
+)
+
+func testErrorInChainEventLoop(t *testing.T) {
+	//the first chain event causes and error (in blockchain mock)
+	builder := mocks.Builder{}
+	blockChain := mocks.BlockChain{}
+	service := statediff.Service{
+		Mutex:         sync.Mutex{},
+		Builder:       &builder,
+		BlockChain:    &blockChain,
+		QuitChan:      make(chan bool),
+		Subscriptions: make(map[rpc.ID]statediff.Subscription),
+		StreamBlock:   true,
+	}
+	payloadChan := make(chan statediff.Payload, 2)
+	quitChan := make(chan bool)
+	service.Subscribe(rpc.NewID(), payloadChan, quitChan)
+	testRoot2 = common.HexToHash("0xTestRoot2")
+	blockMapping := make(map[common.Hash]*types.Block)
+	blockMapping[parentBlock1.Hash()] = parentBlock1
+	blockMapping[parentBlock2.Hash()] = parentBlock2
+	blockChain.SetParentBlocksToReturn(blockMapping)
+	blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3})
+	blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1)
+	blockChain.SetReceiptsForHash(testBlock2.Hash(), testReceipts2)
+
+	payloads := make([]statediff.Payload, 0, 2)
+	wg := sync.WaitGroup{}
+	go func() {
+		wg.Add(1)
+		for i := 0; i < 2; i++ {
+			select {
+			case payload := <-payloadChan:
+				payloads = append(payloads, payload)
+			case <-quitChan:
+			}
+		}
+		wg.Done()
+	}()
+
+	service.Loop(eventsChannel)
+	wg.Wait()
+	if len(payloads) != 2 {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads))
+	}
+
+	testReceipts1Rlp, err := rlp.EncodeToBytes(testReceipts1)
+	if err != nil {
+		t.Error(err)
+	}
+	testReceipts2Rlp, err := rlp.EncodeToBytes(testReceipts2)
+	if err != nil {
+		t.Error(err)
+	}
+	expectedReceiptsRlp := [][]byte{testReceipts1Rlp, testReceipts2Rlp, nil}
+	for i, payload := range payloads {
+		if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) {
+			t.Error("Test failure:", t.Name())
+			t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i])
+		}
+	}
+
+	if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash())
+	}
+	if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root())
+	}
+	if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root())
+	}
+	//look up the parent block from its hash
+	expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()}
+	if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual parent hash does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes)
+	}
+}
+
+func testErrorInBlockLoop(t *testing.T) {
+	//second block's parent block can't be found
+	builder := mocks.Builder{}
+	blockChain := mocks.BlockChain{}
+	service := statediff.Service{
+		Builder:       &builder,
+		BlockChain:    &blockChain,
+		QuitChan:      make(chan bool),
+		Subscriptions: make(map[rpc.ID]statediff.Subscription),
+	}
+	payloadChan := make(chan statediff.Payload)
+	quitChan := make(chan bool)
+	service.Subscribe(rpc.NewID(), payloadChan, quitChan)
+	blockMapping := make(map[common.Hash]*types.Block)
+	blockMapping[parentBlock1.Hash()] = parentBlock1
+	blockChain.SetParentBlocksToReturn(blockMapping)
+	blockChain.SetChainEvents([]core.ChainEvent{event1, event2})
+	// Need to have listeners on the channels or the subscription will be closed and the processing halted
+	go func() {
+		select {
+		case <-payloadChan:
+		case <-quitChan:
+		}
+	}()
+	service.Loop(eventsChannel)
+
+	if !bytes.Equal(builder.BlockHash.Bytes(), testBlock1.Hash().Bytes()) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock1.Hash())
+	}
+	if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock1.Root())
+	}
+	if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) {
+		t.Error("Test failure:", t.Name())
+		t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock1.Root())
+	}
+}

statediff/testhelpers/helpers.go

@@ -1,4 +1,83 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package testhelpers
 
-var ErrorFormatString = "Error: %s, %+v"
-var TestFailureFormatString = "Test failed: %s\nexpected %+v, got %+v\n"
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// MakeChain creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent. In addition, every 3rd block
+// contains a transaction and every 5th an uncle to allow testing correct block
+// reassembly.
+func MakeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) {
+	blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), Testdb, n, testChainGen)
+	headers := make([]*types.Header, len(blocks))
+	for i, block := range blocks {
+		headers[i] = block.Header()
+	}
+	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
+
+	hashes := make([]common.Hash, n+1)
+	hashes[len(hashes)-1] = parent.Hash()
+	blockm := make(map[common.Hash]*types.Block, n+1)
+	blockm[parent.Hash()] = parent
+	for i, b := range blocks {
+		hashes[len(hashes)-i-2] = b.Hash()
+		blockm[b.Hash()] = b
+	}
+	return hashes, blockm, chain
+}
+
+func testChainGen(i int, block *core.BlockGen) {
+	signer := types.HomesteadSigner{}
+	switch i {
+	case 0:
+		// In block 1, the test bank sends account #1 some ether.
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
+		block.AddTx(tx)
+	case 1:
+		// In block 2, the test bank sends some more ether to account #1.
+		// account1Addr passes it on to account #2.
+		// account1Addr creates a test contract.
+		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
+		nonce := block.TxNonce(Account1Addr)
+		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
+		nonce++
+		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key)
+		ContractAddr = crypto.CreateAddress(Account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592
+		block.AddTx(tx1)
+		block.AddTx(tx2)
+		block.AddTx(tx3)
+	case 2:
+		// Block 3 is empty but was mined by account #2.
+		block.SetCoinbase(Account2Addr)
+		//get function: 60cd2685
+		//put function: c16431b9
+		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	}
+}

statediff/testhelpers/mocks/api.go

@@ -0,0 +1,188 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// MockStateDiffService is a mock state diff service
+type MockStateDiffService struct {
+	sync.Mutex
+	Builder         statediff.Builder
+	ReturnProtocol  []p2p.Protocol
+	ReturnAPIs      []rpc.API
+	BlockChan       chan *types.Block
+	ParentBlockChan chan *types.Block
+	QuitChan        chan bool
+	Subscriptions   map[rpc.ID]statediff.Subscription
+	streamBlock     bool
+}
+
+// Protocols mock method
+func (sds *MockStateDiffService) Protocols() []p2p.Protocol {
+	return []p2p.Protocol{}
+}
+
+// APIs mock method
+func (sds *MockStateDiffService) APIs() []rpc.API {
+	return []rpc.API{
+		{
+			Namespace: statediff.APIName,
+			Version:   statediff.APIVersion,
+			Service:   statediff.NewPublicStateDiffAPI(sds),
+			Public:    true,
+		},
+	}
+}
+
+// Loop mock method
+func (sds *MockStateDiffService) Loop(chan core.ChainEvent) {
+	//loop through chain events until no more
+	for {
+		select {
+		case block := <-sds.BlockChan:
+			currentBlock := block
+			parentBlock := <-sds.ParentBlockChan
+			parentHash := parentBlock.Hash()
+			if parentBlock == nil {
+				log.Error("Parent block is nil, skipping this block",
+					"parent block hash", parentHash.String(),
+					"current block number", currentBlock.Number())
+				continue
+			}
+			if err := sds.process(currentBlock, parentBlock); err != nil {
+				println(err.Error())
+				log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err)
+			}
+		case <-sds.QuitChan:
+			log.Debug("Quitting the statediff block channel")
+			sds.close()
+			return
+		}
+	}
+}
+
+// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions
+func (sds *MockStateDiffService) process(currentBlock, parentBlock *types.Block) error {
+	stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash())
+	if err != nil {
+		return err
+	}
+
+	stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+	if err != nil {
+		return err
+	}
+	payload := statediff.Payload{
+		StateDiffRlp: stateDiffRlp,
+		Err:          err,
+	}
+	if sds.streamBlock {
+		rlpBuff := new(bytes.Buffer)
+		if err = currentBlock.EncodeRLP(rlpBuff); err != nil {
+			return err
+		}
+		payload.BlockRlp = rlpBuff.Bytes()
+	}
+
+	// If we have any websocket subscription listening in, send the data to them
+	sds.send(payload)
+	return nil
+}
+
+// Subscribe mock method
+func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool) {
+	log.Info("Subscribing to the statediff service")
+	sds.Lock()
+	sds.Subscriptions[id] = statediff.Subscription{
+		PayloadChan: sub,
+		QuitChan:    quitChan,
+	}
+	sds.Unlock()
+}
+
+// Unsubscribe mock method
+func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error {
+	log.Info("Unsubscribing from the statediff service")
+	sds.Lock()
+	_, ok := sds.Subscriptions[id]
+	if !ok {
+		return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id)
+	}
+	delete(sds.Subscriptions, id)
+	sds.Unlock()
+	return nil
+}
+
+func (sds *MockStateDiffService) send(payload statediff.Payload) {
+	sds.Lock()
+	for id, sub := range sds.Subscriptions {
+		select {
+		case sub.PayloadChan <- payload:
+			log.Info("sending state diff payload to subscription %s", id)
+		default:
+			log.Info("unable to send payload to subscription %s; channel has no receiver", id)
+		}
+	}
+	sds.Unlock()
+}
+
+func (sds *MockStateDiffService) close() {
+	sds.Lock()
+	for id, sub := range sds.Subscriptions {
+		select {
+		case sub.QuitChan <- true:
+			delete(sds.Subscriptions, id)
+			log.Info("closing subscription %s", id)
+		default:
+			log.Info("unable to close subscription %s; channel has no receiver", id)
+		}
+	}
+	sds.Unlock()
+}
+
+// Start mock method
+func (sds *MockStateDiffService) Start(server *p2p.Server) error {
+	log.Info("Starting statediff service")
+	if sds.ParentBlockChan == nil || sds.BlockChan == nil {
+		return errors.New("mock StateDiffingService requires preconfiguration with a MockParentBlockChan and MockBlockChan")
+	}
+	chainEventCh := make(chan core.ChainEvent, 10)
+	go sds.Loop(chainEventCh)
+
+	return nil
+}
+
+// Stop mock method
+func (sds *MockStateDiffService) Stop() error {
+	log.Info("Stopping statediff service")
+	close(sds.QuitChan)
+	return nil
+}

statediff/testhelpers/mocks/api_test.go

@@ -0,0 +1,145 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"bytes"
+	"math/big"
+	"sort"
+	"sync"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/testhelpers"
+)
+
+var block0, block1 *types.Block
+var burnLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0"))
+var emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0)
+var account1, _ = rlp.EncodeToBytes(state.Account{
+	Nonce:    uint64(0),
+	Balance:  big.NewInt(10000),
+	CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+	Root:     common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+})
+var burnAccount1, _ = rlp.EncodeToBytes(state.Account{
+	Nonce:    uint64(0),
+	Balance:  big.NewInt(2000000000000000000),
+	CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+	Root:     common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+})
+var bankAccount1, _ = rlp.EncodeToBytes(state.Account{
+	Nonce:    uint64(1),
+	Balance:  big.NewInt(testhelpers.TestBankFunds.Int64() - 10000),
+	CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(),
+	Root:     common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
+})
+
+func TestAPI(t *testing.T) {
+	_, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis)
+	defer chain.Stop()
+	block0Hash := common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661")
+	block1Hash := common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a")
+	block0 = blockMap[block0Hash]
+	block1 = blockMap[block1Hash]
+	blockChan := make(chan *types.Block)
+	parentBlockChain := make(chan *types.Block)
+	serviceQuitChan := make(chan bool)
+	config := statediff.Config{
+		PathsAndProofs:    true,
+		IntermediateNodes: false,
+	}
+	mockService := MockStateDiffService{
+		Mutex:           sync.Mutex{},
+		Builder:         statediff.NewBuilder(testhelpers.Testdb, chain, config),
+		BlockChan:       blockChan,
+		ParentBlockChan: parentBlockChain,
+		QuitChan:        serviceQuitChan,
+		Subscriptions:   make(map[rpc.ID]statediff.Subscription),
+		streamBlock:     true,
+	}
+	mockService.Start(nil)
+	id := rpc.NewID()
+	payloadChan := make(chan statediff.Payload)
+	quitChan := make(chan bool)
+	mockService.Subscribe(id, payloadChan, quitChan)
+	blockChan <- block1
+	parentBlockChain <- block0
+	expectedBlockRlp, _ := rlp.EncodeToBytes(block1)
+	expectedStateDiff := statediff.StateDiff{
+		BlockNumber: block1.Number(),
+		BlockHash:   block1.Hash(),
+		CreatedAccounts: []statediff.AccountDiff{
+			{
+				Leaf:  true,
+				Key:   burnLeafKey.Bytes(),
+				Value: burnAccount1,
+				Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+					{248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+				Path:    []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16},
+				Storage: []statediff.StorageDiff{},
+			},
+			{
+				Leaf:  true,
+				Key:   testhelpers.Account1LeafKey.Bytes(),
+				Value: account1,
+				Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+					{248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+				Path:    []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16},
+				Storage: []statediff.StorageDiff{},
+			},
+		},
+		DeletedAccounts: emptyAccountDiffEventualMap,
+		UpdatedAccounts: []statediff.AccountDiff{
+			{
+				Leaf:  true,
+				Key:   testhelpers.BankLeafKey.Bytes(),
+				Value: bankAccount1,
+				Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128},
+					{248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 1, 132, 5, 245, 185, 240, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}},
+				Path:    []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16},
+				Storage: []statediff.StorageDiff{},
+			},
+		},
+	}
+	expectedStateDiffBytes, err := rlp.EncodeToBytes(expectedStateDiff)
+	if err != nil {
+		t.Error(err)
+	}
+	sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] })
+
+	select {
+	case payload := <-payloadChan:
+		if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) {
+			t.Errorf("payload does not have expected block\r\actual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp)
+		}
+		sort.Slice(payload.StateDiffRlp, func(i, j int) bool { return payload.StateDiffRlp[i] < payload.StateDiffRlp[j] })
+		if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffBytes) {
+			t.Errorf("payload does not have expected state diff\r\actual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateDiffRlp, expectedStateDiffBytes)
+		}
+		if payload.Err != nil {
+			t.Errorf("payload should not contain an error, but does: %v", payload.Err)
+		}
+	case <-quitChan:
+		t.Errorf("channel quit before delivering payload")
+	}
+}

statediff/testhelpers/mocks/blockchain.go

@@ -1,3 +1,19 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package mocks
 
 import (
@@ -11,41 +27,50 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 )
 
+// BlockChain is a mock blockchain for testing
 type BlockChain struct {
 	ParentHashesLookedUp []common.Hash
-	parentBlocksToReturn []*types.Block
+	parentBlocksToReturn map[common.Hash]*types.Block
 	callCount            int
 	ChainEvents          []core.ChainEvent
+	Receipts             map[common.Hash]types.Receipts
 }
 
-func (mc *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {}
+// AddToStateDiffProcessedCollection mock method
+func (blockChain *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {}
 
-func (mc *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) {
-	mc.parentBlocksToReturn = blocks
+// SetParentBlocksToReturn mock method
+func (blockChain *BlockChain) SetParentBlocksToReturn(blocks map[common.Hash]*types.Block) {
+	if blockChain.parentBlocksToReturn == nil {
+		blockChain.parentBlocksToReturn = make(map[common.Hash]*types.Block)
+	}
+	blockChain.parentBlocksToReturn = blocks
 }
 
-func (mc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
-	mc.ParentHashesLookedUp = append(mc.ParentHashesLookedUp, hash)
+// GetBlockByHash mock method
+func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
+	blockChain.ParentHashesLookedUp = append(blockChain.ParentHashesLookedUp, hash)
 
 	var parentBlock *types.Block
-	if len(mc.parentBlocksToReturn) > 0 {
-		parentBlock = mc.parentBlocksToReturn[mc.callCount]
+	if len(blockChain.parentBlocksToReturn) > 0 {
+		parentBlock = blockChain.parentBlocksToReturn[hash]
 	}
 
-	mc.callCount++
 	return parentBlock
 }
 
-func (bc *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
-	bc.ChainEvents = chainEvents
+// SetChainEvents mock method
+func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) {
+	blockChain.ChainEvents = chainEvents
 }
 
-func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+// SubscribeChainEvent mock method
+func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
 	subErr := errors.New("Subscription Error")
 
 	var eventCounter int
 	subscription := event.NewSubscription(func(quit <-chan struct{}) error {
-		for _, chainEvent := range bc.ChainEvents {
+		for _, chainEvent := range blockChain.ChainEvents {
 			if eventCounter > 1 {
 				time.Sleep(250 * time.Millisecond)
 				return subErr
@@ -62,3 +87,16 @@ func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc
 
 	return subscription
 }
+
+// SetReceiptsForHash mock method
+func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) {
+	if blockChain.Receipts == nil {
+		blockChain.Receipts = make(map[common.Hash]types.Receipts)
+	}
+	blockChain.Receipts[hash] = receipts
+}
+
+// GetReceiptsByHash mock method
+func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+	return blockChain.Receipts[hash]
+}

statediff/testhelpers/mocks/builder.go

@@ -1,20 +1,40 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package mocks
 
 import (
+	"math/big"
+
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/statediff/builder"
+	"github.com/ethereum/go-ethereum/statediff"
 )
 
+// Builder is a mock state diff builder
 type Builder struct {
 	OldStateRoot common.Hash
 	NewStateRoot common.Hash
-	BlockNumber  int64
+	BlockNumber  *big.Int
 	BlockHash    common.Hash
-	stateDiff    *builder.StateDiff
+	stateDiff    statediff.StateDiff
 	builderError error
 }
 
-func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) {
+// BuildStateDiff mock method
+func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (statediff.StateDiff, error) {
 	builder.OldStateRoot = oldStateRoot
 	builder.NewStateRoot = newStateRoot
 	builder.BlockNumber = blockNumber
@@ -23,10 +43,12 @@ func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, b
 	return builder.stateDiff, builder.builderError
 }
 
-func (builder *Builder) SetStateDiffToBuild(stateDiff *builder.StateDiff) {
+// SetStateDiffToBuild mock method
+func (builder *Builder) SetStateDiffToBuild(stateDiff statediff.StateDiff) {
 	builder.stateDiff = stateDiff
 }
 
+// SetBuilderError mock method
 func (builder *Builder) SetBuilderError(err error) {
 	builder.builderError = err
 }

statediff/testhelpers/mocks/error.go

@@ -1,5 +0,0 @@
-package mocks
-
-import "errors"
-
-var Error = errors.New("mock error")

statediff/testhelpers/mocks/extractor.go

@@ -1,20 +0,0 @@
-package mocks
-
-import "github.com/ethereum/go-ethereum/core/types"
-
-type Extractor struct {
-	ParentBlocks  []types.Block
-	CurrentBlocks []types.Block
-	extractError  error
-}
-
-func (me *Extractor) ExtractStateDiff(parent, current types.Block) (string, error) {
-	me.ParentBlocks = append(me.ParentBlocks, parent)
-	me.CurrentBlocks = append(me.CurrentBlocks, current)
-
-	return "", me.extractError
-}
-
-func (me *Extractor) SetExtractError(err error) {
-	me.extractError = err
-}

statediff/testhelpers/mocks/publisher.go

@@ -1,17 +0,0 @@
-package mocks
-
-import "github.com/ethereum/go-ethereum/statediff/builder"
-
-type Publisher struct {
-	StateDiff      *builder.StateDiff
-	publisherError error
-}
-
-func (publisher *Publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) {
-	publisher.StateDiff = sd
-	return "", publisher.publisherError
-}
-
-func (publisher *Publisher) SetPublisherError(err error) {
-	publisher.publisherError = err
-}

statediff/testhelpers/test_data.go

@@ -1,3 +1,19 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package testhelpers
 
 import (
@@ -5,71 +21,94 @@ import (
 	"math/rand"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/statediff/builder"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
 )
 
+// AddressToLeafKey hashes an returns an address
 func AddressToLeafKey(address common.Address) common.Hash {
 	return common.BytesToHash(crypto.Keccak256(address[:]))
 }
 
+// Test variables
 var (
-	BlockNumber     = rand.Int63()
+	BlockNumber     = big.NewInt(rand.Int63())
 	BlockHash       = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
-	CodeHash        = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	CodeHash        = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
 	NewNonceValue   = rand.Uint64()
 	NewBalanceValue = rand.Int63()
-	ContractRoot    = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
-	StoragePath     = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
-	StorageKey      = "0000000000000000000000000000000000000000000000000000000000000001"
-	StorageValue    = "0x03"
-	storage         = map[string]builder.DiffStorage{StoragePath: {
-		Key:   &StorageKey,
-		Value: &StorageValue,
+	ContractRoot    = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+	StoragePath     = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+	StorageKey      = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+	StorageValue    = common.Hex2Bytes("0x03")
+	storage         = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: StorageValue,
+		Path:  StoragePath,
+		Proof: [][]byte{},
 	}}
-	emptyStorage           = map[string]builder.DiffStorage{}
+	emptyStorage           = make([]statediff.StorageDiff, 0)
 	address                = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	ContractLeafKey        = AddressToLeafKey(address)
 	anotherAddress         = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
 	AnotherContractLeafKey = AddressToLeafKey(anotherAddress)
-	CreatedAccountDiffs    = builder.AccountDiffsMap{
-		ContractLeafKey: {
-			Nonce:        builder.DiffUint64{Value: &NewNonceValue},
-			Balance:      builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)},
-			ContractRoot: builder.DiffString{Value: &ContractRoot},
-			CodeHash:     CodeHash,
-			Storage:      storage,
+	testAccount            = state.Account{
+		Nonce:    NewNonceValue,
+		Balance:  big.NewInt(NewBalanceValue),
+		Root:     ContractRoot,
+		CodeHash: CodeHash,
+	}
+	valueBytes, _       = rlp.EncodeToBytes(testAccount)
+	CreatedAccountDiffs = []statediff.AccountDiff{
+		{
+			Key:     ContractLeafKey.Bytes(),
+			Value:   valueBytes,
+			Storage: storage,
 		},
-		AnotherContractLeafKey: {
-			Nonce:        builder.DiffUint64{Value: &NewNonceValue},
-			Balance:      builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)},
-			CodeHash:     CodeHash,
-			ContractRoot: builder.DiffString{Value: &ContractRoot},
-			Storage:      emptyStorage,
+		{
+			Key:     AnotherContractLeafKey.Bytes(),
+			Value:   valueBytes,
+			Storage: emptyStorage,
 		},
 	}
 
-	UpdatedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: {
-		Nonce:        builder.DiffUint64{Value: &NewNonceValue},
-		Balance:      builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)},
-		CodeHash:     CodeHash,
-		ContractRoot: builder.DiffString{Value: &ContractRoot},
-		Storage:      storage,
+	UpdatedAccountDiffs = []statediff.AccountDiff{{
+		Key:     ContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storage,
 	}}
 
-	DeletedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: {
-		Nonce:        builder.DiffUint64{Value: &NewNonceValue},
-		Balance:      builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)},
-		ContractRoot: builder.DiffString{Value: &ContractRoot},
-		CodeHash:     CodeHash,
-		Storage:      storage,
+	DeletedAccountDiffs = []statediff.AccountDiff{{
+		Key:     ContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storage,
 	}}
 
-	TestStateDiff = builder.StateDiff{
+	TestStateDiff = statediff.StateDiff{
 		BlockNumber:     BlockNumber,
 		BlockHash:       common.HexToHash(BlockHash),
 		CreatedAccounts: CreatedAccountDiffs,
 		DeletedAccounts: DeletedAccountDiffs,
 		UpdatedAccounts: UpdatedAccountDiffs,
 	}
+	Testdb = rawdb.NewMemoryDatabase()
+
+	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
+	BankLeafKey     = AddressToLeafKey(TestBankAddress)
+	TestBankFunds   = big.NewInt(100000000)
+	Genesis         = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
+
+	Account1Key, _  = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+	Account2Key, _  = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
+	Account1Addr    = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
+	Account2Addr    = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
+	Account1LeafKey = AddressToLeafKey(Account1Addr)
+	Account2LeafKey = AddressToLeafKey(Account2Addr)
+	ContractCode    = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029")
+	ContractAddr    common.Address
 )

statediff/types.go

@@ -0,0 +1,105 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/state"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// Subscription struct holds our subscription channels
+type Subscription struct {
+	PayloadChan chan<- Payload
+	QuitChan    chan<- bool
+}
+
+// Payload packages the data to send to StateDiffingService subscriptions
+type Payload struct {
+	BlockRlp     []byte `json:"blockRlp"`
+	ReceiptsRlp  []byte `json:"receiptsRlp"`
+	StateDiffRlp []byte `json:"stateDiff"    gencodec:"required"`
+	Err          error  `json:"error"`
+}
+
+// StateDiff is the final output structure from the builder
+type StateDiff struct {
+	BlockNumber     *big.Int      `json:"blockNumber"	  gencodec:"required"`
+	BlockHash       common.Hash   `json:"blockHash" 	  gencodec:"required"`
+	CreatedAccounts []AccountDiff `json:"createdAccounts" gencodec:"required"`
+	DeletedAccounts []AccountDiff `json:"deletedAccounts" gencodec:"required"`
+	UpdatedAccounts []AccountDiff `json:"updatedAccounts" gencodec:"required"`
+
+	encoded []byte
+	err     error
+}
+
+func (sd *StateDiff) ensureEncoded() {
+	if sd.encoded == nil && sd.err == nil {
+		sd.encoded, sd.err = json.Marshal(sd)
+	}
+}
+
+// Length to implement Encoder interface for StateDiff
+func (sd *StateDiff) Length() int {
+	sd.ensureEncoded()
+	return len(sd.encoded)
+}
+
+// Encode to implement Encoder interface for StateDiff
+func (sd *StateDiff) Encode() ([]byte, error) {
+	sd.ensureEncoded()
+	return sd.encoded, sd.err
+}
+
+// AccountDiff holds the data for a single state diff node
+type AccountDiff struct {
+	Leaf    bool          `json:"leaf"	      gencodec:"required"`
+	Key     []byte        `json:"key"         gencodec:"required"`
+	Value   []byte        `json:"value"       gencodec:"required"`
+	Proof   [][]byte      `json:"proof"       gencodec:"required"`
+	Path    []byte        `json:"path"        gencodec:"required"`
+	Storage []StorageDiff `json:"storage"     gencodec:"required"`
+}
+
+// StorageDiff holds the data for a single storage diff node
+type StorageDiff struct {
+	Leaf  bool     `json:"leaf"	       gencodec:"required"`
+	Key   []byte   `json:"key"         gencodec:"required"`
+	Value []byte   `json:"value"       gencodec:"required"`
+	Proof [][]byte `json:"proof"       gencodec:"required"`
+	Path  []byte   `json:"path"        gencodec:"required"`
+}
+
+// AccountsMap is a mapping of keccak256(address) => accountWrapper
+type AccountsMap map[common.Hash]accountWrapper
+
+// AccountWrapper is used to temporary associate the unpacked account with its raw values
+type accountWrapper struct {
+	Account  *state.Account
+	Leaf     bool
+	RawKey   []byte
+	RawValue []byte
+	Proof    [][]byte
+	Path     []byte
+}