From 33d4b8d744d8362561017003cb72787fab48bc09 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Thu, 21 Feb 2019 14:36:04 -0600 Subject: [PATCH 01/14] Statediff for full node (#6) * Open a trie from the in-memory database * Use a node's LeafKey as an identifier instead of the address It was proving difficult to find look the address up from a given path with a full node (sometimes the value wouldn't exist in the disk db). So, instead, for now we are using the node's LeafKey with is a Keccak256 hash of the address, so if we know the address we can figure out which LeafKey it matches up to. * Make sure that statediff has been processed before pruning * Use blockchain stateCache.OpenTrie for storage diffs * Clean up log lines and remove unnecessary fields from builder * Apply go fmt changes * Add a sleep to the blockchain test * Address PR comments * Address PR comments --- eth/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth/config.go b/eth/config.go index 0a5fe000b..8ed014e24 100644 --- a/eth/config.go +++ b/eth/config.go @@ -158,4 +158,6 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + + StateDiff bool } -- 2.45.2 From 4f7dca667e7333a3c2d89835daab1708e3fc08be Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 26 Apr 2019 11:13:59 -0500 Subject: [PATCH 02/14] refactoring/reorganizing packages --- cmd/geth/main.go | 2 - cmd/geth/usage.go | 2 - cmd/utils/flags.go | 43 +-- core/blockchain.go | 17 +- eth/backend.go | 10 +- statediff/config.go | 93 ------- statediff/config_test.go | 24 -- statediff/extractor/extractor.go | 51 ---- statediff/extractor/extractor_test.go | 123 --------- statediff/helpers.go | 118 ++++++++ statediff/publisher/csv.go | 130 --------- statediff/publisher/publisher.go | 48 ---- statediff/publisher/publisher_test.go | 316 ---------------------- statediff/testhelpers/helpers.go | 83 +++++- statediff/testhelpers/mocks/blockchain.go | 46 +++- statediff/testhelpers/mocks/builder.go | 28 +- statediff/testhelpers/mocks/error.go | 5 - statediff/testhelpers/mocks/extractor.go | 20 -- statediff/testhelpers/mocks/publisher.go | 25 +- statediff/testhelpers/test_data.go | 107 +++++--- 20 files changed, 377 insertions(+), 914 deletions(-) delete mode 100644 statediff/config.go delete mode 100644 statediff/config_test.go delete mode 100644 statediff/extractor/extractor.go delete mode 100644 statediff/extractor/extractor_test.go create mode 100644 statediff/helpers.go delete mode 100644 statediff/publisher/csv.go delete mode 100644 statediff/publisher/publisher.go delete mode 100644 statediff/publisher/publisher_test.go delete mode 100644 statediff/testhelpers/mocks/error.go delete mode 100644 statediff/testhelpers/mocks/extractor.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7d2874090..d27c3b109 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -148,8 +148,6 @@ var ( utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, utils.StateDiffFlag, - utils.StateDiffModeFlag, - utils.StateDiffPathFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index ce6eab279..77d816476 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -266,8 +266,6 @@ var AppHelpFlagGroups = []flagGroup{ Name: "STATE DIFF", Flags: []cli.Flag{ utils.StateDiffFlag, - utils.StateDiffModeFlag, - utils.StateDiffPathFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1ab05773d..b76c5dfe0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -29,6 +29,8 @@ import ( "strings" "time" + cli "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" @@ -59,11 +61,10 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" - pcsclite "github.com/gballet/go-libpcsclite" - cli "gopkg.in/urfave/cli.v1" - "github.com/ethereum/go-ethereum/statediff/service" "github.com/ethereum/go-ethereum/statediff" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" + + pcsclite "github.com/gballet/go-libpcsclite" ) var ( @@ -744,18 +745,6 @@ var ( Name: "statediff", Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", } - - StateDiffModeFlag = cli.StringFlag{ - Name: "statediff.mode", - Usage: "Enables the user to determine which persistence mode they'd like to store the state diffs in.", - Value: "csv", - } - - StateDiffPathFlag = cli.StringFlag{ - Name: "statediff.path", - Usage: "Enables the user to determine where to persist the state diffs.", - Value: ".", - } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -965,6 +954,9 @@ func setWS(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(WSApiFlag.Name) { cfg.WSModules = splitAndTrim(ctx.GlobalString(WSApiFlag.Name)) } + if ctx.GlobalBool(StateDiffFlag.Name) { + cfg.WSModules = append(cfg.WSModules, "statediff") + } } // setIPC creates an IPC path configuration from the set command line flags, @@ -1606,29 +1598,14 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st } } +// RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { - //based on the context, if path and mode are set, update the config here - //otherwise pass in an empty config - - modeFlag := ctx.GlobalString(StateDiffModeFlag.Name) - mode, err := statediff.NewMode(modeFlag) - if err != nil { - Fatalf("Failed to register State Diff Service", err) - } - - path := ctx.GlobalString(StateDiffPathFlag.Name) - - config := statediff.Config{ - Mode: mode, - Path: path, - } - if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum ctx.Service(ðServ) chainDb := ethServ.ChainDb() blockChain := ethServ.BlockChain() - return service.NewStateDiffService(chainDb, blockChain, config) + return statediff.NewStateDiffService(chainDb, blockChain) }); err != nil { Fatalf("Failed to register State Diff Service", err) } diff --git a/core/blockchain.go b/core/blockchain.go index 18bcd4113..cf1a65cd5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -106,11 +106,11 @@ const ( // CacheConfig contains the configuration values for the trie caching/pruning // that's resident in a blockchain. type CacheConfig struct { - TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory - TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks - TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk - TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk ProcessingStateDiffs bool // Whether statediffs processing should be taken into a account before a trie is pruned } @@ -1336,16 +1336,19 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, number) break } - if bc.cacheConfig.ProcessingStateDiffs { if !bc.allowedRootToBeDereferenced(root.(common.Hash)) { bc.triegc.Push(root, number) break } else { + log.Debug("Current root found in stateDiffsProcessed collection with a count of 2, okay to dereference", + "root", root.(common.Hash).Hex(), + "blockNumber", uint64(-number), + "size of stateDiffsProcessed", len(bc.stateDiffsProcessed)) delete(bc.stateDiffsProcessed, root.(common.Hash)) } } - + log.Debug("Dereferencing", "root", root.(common.Hash).Hex()) triedb.Dereference(root.(common.Hash)) } } diff --git a/eth/backend.go b/eth/backend.go index b3ca91373..f46fde656 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -179,11 +179,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EVMInterpreter: config.EVMInterpreter, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, ProcessingStateDiffs: config.StateDiff, } ) diff --git a/statediff/config.go b/statediff/config.go deleted file mode 100644 index 7f5ec3c35..000000000 --- a/statediff/config.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package statediff - -import "fmt" - -type Config struct { - Mode StateDiffMode // Mode for storing diffs - Path string // Path for storing diffs -} - -type StateDiffMode int - -const ( - CSV StateDiffMode = iota - IPLD - LDB - SQL -) - -func (mode StateDiffMode) IsValid() bool { - return mode >= IPLD && mode <= SQL -} - -// String implements the stringer interface. -func (mode StateDiffMode) String() string { - switch mode { - case CSV: - return "csv" - case IPLD: - return "ipfs" - case LDB: - return "ldb" - case SQL: - return "sql" - default: - return "unknown" - } -} - -func NewMode(mode string) (StateDiffMode, error) { - stateDiffMode := StateDiffMode(0) - err := stateDiffMode.UnmarshalText([]byte(mode)) - return stateDiffMode, err -} - -func (mode StateDiffMode) MarshalText() ([]byte, error) { - switch mode { - case CSV: - return []byte("ipfs"), nil - case IPLD: - return []byte("ipfs"), nil - case LDB: - return []byte("ldb"), nil - case SQL: - return []byte("sql"), nil - default: - return nil, fmt.Errorf("unknown state diff storage mode %d", mode) - } -} - -func (mode *StateDiffMode) UnmarshalText(text []byte) error { - switch string(text) { - case "csv": - *mode = CSV - case "ipfs": - *mode = IPLD - case "ldb": - *mode = LDB - case "sql": - *mode = SQL - default: - return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text) - } - return nil -} diff --git a/statediff/config_test.go b/statediff/config_test.go deleted file mode 100644 index 5246d1cd1..000000000 --- a/statediff/config_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package statediff_test - -import ( - "testing" - - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/testhelpers" -) - -func TestNewMode(t *testing.T) { - mode, err := statediff.NewMode("csv") - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - if mode != statediff.CSV { - t.Error() - } - - _, err = statediff.NewMode("not a real mode") - if err == nil { - t.Error("Expected an error, and got nil.") - } -} diff --git a/statediff/extractor/extractor.go b/statediff/extractor/extractor.go deleted file mode 100644 index 770973c8d..000000000 --- a/statediff/extractor/extractor.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package extractor - -import ( - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/statediff/builder" - "github.com/ethereum/go-ethereum/statediff/publisher" -) - -type Extractor interface { - ExtractStateDiff(parent, current types.Block) (string, error) -} - -type extractor struct { - Builder builder.Builder // Interface for building state diff objects from two blocks - Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) -} - -func NewExtractor(builder builder.Builder, publisher publisher.Publisher) *extractor { - return &extractor{ - Builder: builder, - Publisher: publisher, - } -} - -func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { - stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) - if err != nil { - return "", err - } - - return e.Publisher.PublishStateDiff(stateDiff) -} diff --git a/statediff/extractor/extractor_test.go b/statediff/extractor/extractor_test.go deleted file mode 100644 index 0ed036c12..000000000 --- a/statediff/extractor/extractor_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package extractor_test - -import ( - "bytes" - "math/big" - "math/rand" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/core/types" - b "github.com/ethereum/go-ethereum/statediff/builder" - e "github.com/ethereum/go-ethereum/statediff/extractor" - "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" -) - -var publisher mocks.Publisher -var builder mocks.Builder -var currentBlockNumber *big.Int -var parentBlock, currentBlock *types.Block -var expectedStateDiff b.StateDiff -var extractor e.Extractor -var err error - -func TestExtractor(t *testing.T) { - publisher = mocks.Publisher{} - builder = mocks.Builder{} - extractor = e.NewExtractor(&builder, &publisher) - if err != nil { - t.Error(err) - } - - blockNumber := rand.Int63() - parentBlockNumber := big.NewInt(blockNumber - int64(1)) - currentBlockNumber = big.NewInt(blockNumber) - parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) - currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) - - expectedStateDiff = b.StateDiff{ - BlockNumber: blockNumber, - BlockHash: currentBlock.Hash(), - CreatedAccounts: nil, - DeletedAccounts: nil, - UpdatedAccounts: nil, - } - - testBuildStateDiffStruct(t) - testBuildStateDiffErrorHandling(t) - testPublishingStateDiff(t) - testPublisherErrorHandling(t) -} - -func testBuildStateDiffStruct(t *testing.T) { - builder.SetStateDiffToBuild(&expectedStateDiff) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - t.Error(err) - } - - if !equals(builder.OldStateRoot, parentBlock.Root()) { - t.Error() - } - if !equals(builder.NewStateRoot, currentBlock.Root()) { - t.Error() - } - if !equals(builder.BlockNumber, currentBlockNumber.Int64()) { - t.Error() - } - if !equals(builder.BlockHash, currentBlock.Hash()) { - t.Error() - } -} - -func testBuildStateDiffErrorHandling(t *testing.T) { - builder.SetBuilderError(mocks.Error) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err == nil { - t.Error(err) - } - - if !equals(err, mocks.Error) { - t.Error() - } - builder.SetBuilderError(nil) -} - -func testPublishingStateDiff(t *testing.T) { - builder.SetStateDiffToBuild(&expectedStateDiff) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - t.Error(err) - } - - if !equals(publisher.StateDiff, &expectedStateDiff) { - t.Error() - } -} - -func testPublisherErrorHandling(t *testing.T) { - publisher.SetPublisherError(mocks.Error) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err == nil { - t.Error("Expected an error, but it didn't occur.") - } - if !equals(err, mocks.Error) { - t.Error() - } - - publisher.SetPublisherError(nil) -} - -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} diff --git a/statediff/helpers.go b/statediff/helpers.go new file mode 100644 index 000000000..96f9ddf30 --- /dev/null +++ b/statediff/helpers.go @@ -0,0 +1,118 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "sort" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/trie" +) + +func sortKeys(data AccountsMap) []string { + var keys []string + for key := range data { + keys = append(keys, key.Hex()) + } + sort.Strings(keys) + + return keys +} + +// BytesToNiblePath +func bytesToNiblePath(path []byte) string { + if hasTerm(path) { + path = path[:len(path)-1] + } + nibblePath := "" + for i, v := range common.ToHex(path) { + if i%2 == 0 && i > 1 { + continue + } + nibblePath = nibblePath + string(v) + } + + return nibblePath +} + +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + // a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + // a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + } + } + +} + +func pathToStr(it trie.NodeIterator) string { + return bytesToNiblePath(it.Path()) +} + +// Duplicated from trie/encoding.go +func hexToKeyBytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go deleted file mode 100644 index 13971a5c8..000000000 --- a/statediff/publisher/csv.go +++ /dev/null @@ -1,130 +0,0 @@ -package publisher - -import ( - "encoding/csv" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/builder" -) - -var ( - Headers = []string{ - "blockNumber", "blockHash", "accountAction", "codeHash", - "nonceValue", "balanceValue", "contractRoot", "storageDiffPaths", - "accountLeafKey", "storageKey", "storageValue", - } - - timeStampFormat = "20060102150405.00000" - deletedAccountAction = "deleted" - createdAccountAction = "created" - updatedAccountAction = "updated" -) - -func createCSVFilePath(path, blockNumber string) string { - now := time.Now() - timeStamp := now.Format(timeStampFormat) - suffix := timeStamp + "-" + blockNumber - filePath := filepath.Join(path, suffix) - filePath = filePath + ".csv" - return filePath -} - -func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) { - filePath := createCSVFilePath(p.Config.Path, strconv.FormatInt(sd.BlockNumber, 10)) - - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return "", err - } - defer file.Close() - - writer := csv.NewWriter(file) - defer writer.Flush() - - var data [][]string - data = append(data, Headers) - data = append(data, accumulateAccountRows(sd)...) - for _, value := range data { - err := writer.Write(value) - if err != nil { - return "", err - } - } - - return filePath, nil -} - -func accumulateAccountRows(sd builder.StateDiff) [][]string { - var accountRows [][]string - for accountAddr, accountDiff := range sd.CreatedAccounts { - formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction) - - accountRows = append(accountRows, formattedAccountData...) - } - - for accountAddr, accountDiff := range sd.UpdatedAccounts { - formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction) - - accountRows = append(accountRows, formattedAccountData...) - } - - for accountAddr, accountDiff := range sd.DeletedAccounts { - formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction) - - accountRows = append(accountRows, formattedAccountData...) - } - - return accountRows -} - -func formatAccountData(accountAddr common.Hash, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { - blockNumberString := strconv.FormatInt(sd.BlockNumber, 10) - blockHash := sd.BlockHash.String() - codeHash := accountDiff.CodeHash - nonce := strconv.FormatUint(*accountDiff.Nonce.Value, 10) - balance := accountDiff.Balance.Value.String() - newContractRoot := accountDiff.ContractRoot.Value - address := accountAddr.String() - var result [][]string - - if len(accountDiff.Storage) > 0 { - for storagePath, storage := range accountDiff.Storage { - formattedAccountData := []string{ - blockNumberString, - blockHash, - accountAction, - codeHash, - nonce, - balance, - *newContractRoot, - storagePath, - address, - *storage.Key, - *storage.Value, - } - - result = append(result, formattedAccountData) - } - } else { - formattedAccountData := []string{ - blockNumberString, - blockHash, - accountAction, - codeHash, - nonce, - balance, - *newContractRoot, - "", - address, - "", - "", - } - result = append(result, formattedAccountData) - } - - return result -} diff --git a/statediff/publisher/publisher.go b/statediff/publisher/publisher.go deleted file mode 100644 index ff1925513..000000000 --- a/statediff/publisher/publisher.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package publisher - -import ( - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/builder" -) - -type Publisher interface { - PublishStateDiff(sd *builder.StateDiff) (string, error) -} - -type publisher struct { - Config statediff.Config -} - -func NewPublisher(config statediff.Config) (*publisher, error) { - return &publisher{ - Config: config, - }, nil -} - -func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { - switch p.Config.Mode { - case statediff.CSV: - return p.publishStateDiffToCSV(*sd) - default: - return p.publishStateDiffToCSV(*sd) - } -} diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go deleted file mode 100644 index 76aaf961e..000000000 --- a/statediff/publisher/publisher_test.go +++ /dev/null @@ -1,316 +0,0 @@ -package publisher_test - -import ( - "bytes" - "encoding/csv" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - "testing" - - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/builder" - p "github.com/ethereum/go-ethereum/statediff/publisher" - "github.com/ethereum/go-ethereum/statediff/testhelpers" - "github.com/pkg/errors" -) - -var ( - tempDir = os.TempDir() - testFilePrefix = "test-statediff" - publisher p.Publisher - dir string - err error -) - -var expectedCreatedAccountRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "created", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - testhelpers.StoragePath, - testhelpers.ContractLeafKey.Hex(), - "0000000000000000000000000000000000000000000000000000000000000001", - testhelpers.StorageValue, -} - -var expectedCreatedAccountWithoutStorageUpdateRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "created", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - "", - testhelpers.AnotherContractLeafKey.Hex(), - "", - "", -} - -var expectedUpdatedAccountRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "updated", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - testhelpers.StoragePath, - testhelpers.ContractLeafKey.Hex(), - "0000000000000000000000000000000000000000000000000000000000000001", - testhelpers.StorageValue, -} - -var expectedDeletedAccountRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "deleted", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - testhelpers.StoragePath, - testhelpers.ContractLeafKey.Hex(), - "0000000000000000000000000000000000000000000000000000000000000001", - testhelpers.StorageValue, -} - -func TestPublisher(t *testing.T) { - dir, err = ioutil.TempDir(tempDir, testFilePrefix) - if err != nil { - t.Error(err) - } - config := statediff.Config{ - Path: dir, - Mode: statediff.CSV, - } - publisher, err = p.NewPublisher(config) - if err != nil { - t.Error(err) - } - - type Test func(t *testing.T) - - var tests = []Test{ - testFileName, - testColumnHeaders, - testAccountDiffs, - testWhenNoDiff, - testDefaultPublisher, - testDefaultDirectory, - } - - for _, test := range tests { - test(t) - err := removeFilesFromDir(dir) - if err != nil { - t.Errorf("Error removing files from temp dir: %s", dir) - } - } -} - -func removeFilesFromDir(dir string) error { - files, err := filepath.Glob(filepath.Join(dir, "*")) - if err != nil { - return err - } - - for _, file := range files { - err = os.RemoveAll(file) - if err != nil { - return err - } - } - return nil -} - -func testFileName(t *testing.T) { - fileName, err := publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - if !strings.HasPrefix(fileName, dir) { - t.Errorf(testhelpers.TestFailureFormatString, t.Name(), dir, fileName) - } - blockNumberWithFileExt := strconv.FormatInt(testhelpers.BlockNumber, 10) + ".csv" - if !strings.HasSuffix(fileName, blockNumberWithFileExt) { - t.Errorf(testhelpers.TestFailureFormatString, t.Name(), blockNumberWithFileExt, fileName) - } -} - -func testColumnHeaders(t *testing.T) { - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if len(lines) < 1 { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[0], p.Headers) { - t.Error() - } -} - -func testAccountDiffs(t *testing.T) { - // it persists the created, updated and deleted account diffs to a CSV file - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if len(lines) <= 3 { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[1], expectedCreatedAccountRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[2], expectedCreatedAccountWithoutStorageUpdateRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[3], expectedUpdatedAccountRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[4], expectedDeletedAccountRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func testWhenNoDiff(t *testing.T) { - //it creates an empty CSV when there is no diff - emptyDiff := builder.StateDiff{} - _, err = publisher.PublishStateDiff(&emptyDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - if !equals(len(lines), 1) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func testDefaultPublisher(t *testing.T) { - //it defaults to publishing state diffs to a CSV file when no mode is configured - config := statediff.Config{Path: dir} - publisher, err = p.NewPublisher(config) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(len(lines), 5) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[0], p.Headers) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func testDefaultDirectory(t *testing.T) { - //it defaults to publishing CSV files in the current directory when no path is configured - config := statediff.Config{} - publisher, err = p.NewPublisher(config) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - err := os.Chdir(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(len(lines), 5) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[0], p.Headers) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func getTestDiffFile(dir string) (*os.File, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - if len(files) == 0 { - return nil, errors.New("There are 0 files.") - } - - fileName := files[0].Name() - filePath := filepath.Join(dir, fileName) - - return os.Open(filePath) -} - -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go index 0bfc53e5e..5126c6556 100644 --- a/statediff/testhelpers/helpers.go +++ b/statediff/testhelpers/helpers.go @@ -1,4 +1,83 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package testhelpers -var ErrorFormatString = "Error: %s, %+v" -var TestFailureFormatString = "Test failed: %s\nexpected %+v, got %+v\n" +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +// MakeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func MakeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), Testdb, n, testChainGen) + headers := make([]*types.Header, len(blocks)) + for i, block := range blocks { + headers[i] = block.Header() + } + chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil) + + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm, chain +} + +func testChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // account1Addr passes it on to account #2. + // account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey) + nonce := block.TxNonce(Account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key) + ContractAddr = crypto.CreateAddress(Account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(Account2Addr) + //get function: 60cd2685 + //put function: c16431b9 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) + } +} diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index 7b0d74c59..cececde6f 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -1,3 +1,19 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package mocks import ( @@ -11,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/event" ) +// BlockChain is a mock blockchain for testing type BlockChain struct { ParentHashesLookedUp []common.Hash parentBlocksToReturn []*types.Block @@ -18,34 +35,39 @@ type BlockChain struct { ChainEvents []core.ChainEvent } -func (mc *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} +// AddToStateDiffProcessedCollection mock method +func (blockChain *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} -func (mc *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { - mc.parentBlocksToReturn = blocks +// SetParentBlocksToReturn mock method +func (blockChain *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { + blockChain.parentBlocksToReturn = blocks } -func (mc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { - mc.ParentHashesLookedUp = append(mc.ParentHashesLookedUp, hash) +// GetBlockByHash mock method +func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + blockChain.ParentHashesLookedUp = append(blockChain.ParentHashesLookedUp, hash) var parentBlock *types.Block - if len(mc.parentBlocksToReturn) > 0 { - parentBlock = mc.parentBlocksToReturn[mc.callCount] + if len(blockChain.parentBlocksToReturn) > 0 { + parentBlock = blockChain.parentBlocksToReturn[blockChain.callCount] } - mc.callCount++ + blockChain.callCount++ return parentBlock } -func (bc *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { - bc.ChainEvents = chainEvents +// SetChainEvents mock method +func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { + blockChain.ChainEvents = chainEvents } -func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { +// SubscribeChainEvent mock method +func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { subErr := errors.New("Subscription Error") var eventCounter int subscription := event.NewSubscription(func(quit <-chan struct{}) error { - for _, chainEvent := range bc.ChainEvents { + for _, chainEvent := range blockChain.ChainEvents { if eventCounter > 1 { time.Sleep(250 * time.Millisecond) return subErr diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index ae9ff5ced..e9668629e 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -1,20 +1,38 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package mocks import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff" ) +// Builder is a mock state diff builder type Builder struct { OldStateRoot common.Hash NewStateRoot common.Hash BlockNumber int64 BlockHash common.Hash - stateDiff *builder.StateDiff + stateDiff statediff.StateDiff builderError error } -func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) { +// BuildStateDiff mock method +func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (statediff.StateDiff, error) { builder.OldStateRoot = oldStateRoot builder.NewStateRoot = newStateRoot builder.BlockNumber = blockNumber @@ -23,10 +41,12 @@ func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, b return builder.stateDiff, builder.builderError } -func (builder *Builder) SetStateDiffToBuild(stateDiff *builder.StateDiff) { +// SetStateDiffToBuild mock method +func (builder *Builder) SetStateDiffToBuild(stateDiff statediff.StateDiff) { builder.stateDiff = stateDiff } +// SetBuilderError mock method func (builder *Builder) SetBuilderError(err error) { builder.builderError = err } diff --git a/statediff/testhelpers/mocks/error.go b/statediff/testhelpers/mocks/error.go deleted file mode 100644 index 7c40452ae..000000000 --- a/statediff/testhelpers/mocks/error.go +++ /dev/null @@ -1,5 +0,0 @@ -package mocks - -import "errors" - -var Error = errors.New("mock error") diff --git a/statediff/testhelpers/mocks/extractor.go b/statediff/testhelpers/mocks/extractor.go deleted file mode 100644 index 067497646..000000000 --- a/statediff/testhelpers/mocks/extractor.go +++ /dev/null @@ -1,20 +0,0 @@ -package mocks - -import "github.com/ethereum/go-ethereum/core/types" - -type Extractor struct { - ParentBlocks []types.Block - CurrentBlocks []types.Block - extractError error -} - -func (me *Extractor) ExtractStateDiff(parent, current types.Block) (string, error) { - me.ParentBlocks = append(me.ParentBlocks, parent) - me.CurrentBlocks = append(me.CurrentBlocks, current) - - return "", me.extractError -} - -func (me *Extractor) SetExtractError(err error) { - me.extractError = err -} diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go index efbe4e2ab..3ea18abf0 100644 --- a/statediff/testhelpers/mocks/publisher.go +++ b/statediff/testhelpers/mocks/publisher.go @@ -1,17 +1,36 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package mocks -import "github.com/ethereum/go-ethereum/statediff/builder" +import "github.com/ethereum/go-ethereum/statediff" +// Publisher mock type Publisher struct { - StateDiff *builder.StateDiff + StateDiff *statediff.StateDiff publisherError error } -func (publisher *Publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { +// PublishStateDiff mock method +func (publisher *Publisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { publisher.StateDiff = sd return "", publisher.publisherError } +// SetPublisherError mock method func (publisher *Publisher) SetPublisherError(err error) { publisher.publisherError = err } diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 831bed218..17dac8473 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -1,3 +1,19 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package testhelpers import ( @@ -5,71 +21,94 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" ) +// AddressToLeafKey hashes an returns an address func AddressToLeafKey(address common.Address) common.Hash { return common.BytesToHash(crypto.Keccak256(address[:])) } +// Test variables var ( BlockNumber = rand.Int63() BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" - CodeHash = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") NewNonceValue = rand.Uint64() NewBalanceValue = rand.Int63() - ContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - StoragePath = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - StorageKey = "0000000000000000000000000000000000000000000000000000000000000001" - StorageValue = "0x03" - storage = map[string]builder.DiffStorage{StoragePath: { - Key: &StorageKey, - Value: &StorageValue, + ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() + StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() + StorageValue = common.Hex2Bytes("0x03") + storage = []statediff.StorageDiff{{ + Key: StorageKey, + Value: StorageValue, + Path: StoragePath, + Proof: [][]byte{}, }} - emptyStorage = map[string]builder.DiffStorage{} + emptyStorage = make([]statediff.StorageDiff, 0) address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") ContractLeafKey = AddressToLeafKey(address) anotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") AnotherContractLeafKey = AddressToLeafKey(anotherAddress) - CreatedAccountDiffs = builder.AccountDiffsMap{ + testAccount = state.Account{ + Nonce: NewNonceValue, + Balance: big.NewInt(NewBalanceValue), + Root: ContractRoot, + CodeHash: CodeHash, + } + valueBytes, _ = rlp.EncodeToBytes(testAccount) + CreatedAccountDiffs = statediff.AccountDiffsMap{ ContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - CodeHash: CodeHash, - Storage: storage, + Key: ContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storage, }, AnotherContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - CodeHash: CodeHash, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - Storage: emptyStorage, + Key: AnotherContractLeafKey.Bytes(), + Value: valueBytes, + Storage: emptyStorage, }, } - UpdatedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - CodeHash: CodeHash, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - Storage: storage, + UpdatedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + Key: ContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storage, }} - DeletedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - CodeHash: CodeHash, - Storage: storage, + DeletedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + Key: ContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storage, }} - TestStateDiff = builder.StateDiff{ + TestStateDiff = statediff.StateDiff{ BlockNumber: BlockNumber, BlockHash: common.HexToHash(BlockHash), CreatedAccounts: CreatedAccountDiffs, DeletedAccounts: DeletedAccountDiffs, UpdatedAccounts: UpdatedAccountDiffs, } + Testdb = rawdb.NewMemoryDatabase() + + TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + BankLeafKey = AddressToLeafKey(TestBankAddress) + TestBankFunds = big.NewInt(100000000) + Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) + + Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + Account1LeafKey = AddressToLeafKey(Account1Addr) + Account2LeafKey = AddressToLeafKey(Account2Addr) + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") + ContractAddr common.Address ) -- 2.45.2 From aa1345b43da3b62902eec794c27a7efe274a8eaa Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 26 Apr 2019 11:16:50 -0500 Subject: [PATCH 03/14] refactoring statediff builder and types and adjusted to relay proofs and paths (still need to make this optional) --- statediff/{builder => }/builder.go | 147 +++++------ statediff/builder/builder_test.go | 365 --------------------------- statediff/builder/helpers.go | 114 --------- statediff/builder/struct.go | 79 ------ statediff/builder_test.go | 392 +++++++++++++++++++++++++++++ statediff/types.go | 113 +++++++++ 6 files changed, 581 insertions(+), 629 deletions(-) rename statediff/{builder => }/builder.go (70%) delete mode 100644 statediff/builder/builder_test.go delete mode 100644 statediff/builder/helpers.go delete mode 100644 statediff/builder/struct.go create mode 100644 statediff/builder_test.go create mode 100644 statediff/types.go diff --git a/statediff/builder/builder.go b/statediff/builder.go similarity index 70% rename from statediff/builder/builder.go rename to statediff/builder.go index a106c3ad8..4e8ab0e21 100644 --- a/statediff/builder/builder.go +++ b/statediff/builder.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package builder +package statediff import ( "github.com/ethereum/go-ethereum/common" @@ -30,8 +30,9 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// Builder interface exposes the method for building a state diff between two blocks type Builder interface { - BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) } type builder struct { @@ -39,27 +40,27 @@ type builder struct { blockChain *core.BlockChain } -type AccountsMap map[common.Hash]*state.Account - -func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) *builder { +// NewBuilder is used to create a builder +func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) Builder { return &builder{ chainDB: db, blockChain: blockChain, } } -func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { +// BuildStateDiff builds a StateDiff object from two blocks +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states stateCache := sdb.blockChain.StateCache() oldTrie, err := stateCache.OpenTrie(oldStateRoot) if err != nil { log.Error("Error creating trie for oldStateRoot", "error", err) - return nil, err + return StateDiff{}, err } newTrie, err := stateCache.OpenTrie(newStateRoot) if err != nil { log.Error("Error creating trie for newStateRoot", "error", err) - return nil, err + return StateDiff{}, err } // Find created accounts @@ -68,7 +69,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { log.Error("Error collecting creation diff nodes", "error", err) - return nil, err + return StateDiff{}, err } // Find deleted accounts @@ -77,7 +78,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { log.Error("Error collecting deletion diff nodes", "error", err) - return nil, err + return StateDiff{}, err } // Find all the diffed keys @@ -89,20 +90,20 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { log.Error("Error building diff for updated accounts", "error", err) - return nil, err + return StateDiff{}, err } createdAccounts, err := sdb.buildDiffEventual(creations) if err != nil { log.Error("Error building diff for created accounts", "error", err) - return nil, err + return StateDiff{}, err } deletedAccounts, err := sdb.buildDiffEventual(deletions) if err != nil { log.Error("Error building diff for deleted accounts", "error", err) - return nil, err + return StateDiff{}, err } - return &StateDiff{ + return StateDiff{ BlockNumber: blockNumber, BlockHash: blockHash, CreatedAccounts: createdAccounts, @@ -118,20 +119,31 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error for { log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) if it.Leaf() { + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) leafKey := make([]byte, len(it.LeafKey())) copy(leafKey, it.LeafKey()) leafKeyHash := common.BytesToHash(leafKey) - + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) // lookup account state var account state.Account - if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { + if err := rlp.DecodeBytes(leafValue, &account); err != nil { log.Error("Error looking up account via address", "address", leafKeyHash, "error", err) return nil, err } - + aw := accountWrapper{ + Account: account, + RawKey: leafKey, + RawValue: leafValue, + Proof: leafProof, + Path: leafPath, + } // record account to diffs (creation if we are looking at new - old; deletion if old - new) log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = &account + diffAccounts[leafKeyHash] = aw } cont := it.Next(true) if !cont { @@ -144,25 +156,18 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { accountDiffs := make(AccountDiffsMap) - for addr, val := range accounts { - sr := val.Root - storageDiffs, err := sdb.buildStorageDiffsEventual(sr) + for _, val := range accounts { + storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root) if err != nil { - log.Error("Failed building eventual storage diffs", "Address", addr, "error", err) + log.Error("Failed building eventual storage diffs", "Address", common.BytesToHash(val.RawKey), "error", err) return nil, err } - - codeHash := hexutil.Encode(val.CodeHash) - hexRoot := val.Root.Hex() - nonce := DiffUint64{Value: &val.Nonce} - balance := DiffBigInt{Value: val.Balance} - contractRoot := DiffString{Value: &hexRoot} - accountDiffs[addr] = AccountDiff{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - ContractRoot: contractRoot, - Storage: storageDiffs, + accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ + Key: val.RawKey, + Value: val.RawValue, + Proof: val.Proof, + Path: val.Path, + Storage: storageDiffs, } } @@ -174,34 +179,28 @@ func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions Accoun for _, val := range updatedKeys { createdAcc := creations[common.HexToHash(val)] deletedAcc := deletions[common.HexToHash(val)] - oldSR := deletedAcc.Root - newSR := createdAcc.Root - if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { + oldSR := deletedAcc.Account.Root + newSR := createdAcc.Account.Root + storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR) + if err != nil { log.Error("Failed building storage diffs", "Address", val, "error", err) return nil, err - } else { - nonce := DiffUint64{Value: &createdAcc.Nonce} - balance := DiffBigInt{Value: createdAcc.Balance} - codeHash := hexutil.Encode(createdAcc.CodeHash) - - nHexRoot := createdAcc.Root.Hex() - contractRoot := DiffString{Value: &nHexRoot} - - updatedAccounts[common.HexToHash(val)] = AccountDiff{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - ContractRoot: contractRoot, - Storage: storageDiffs, - } - delete(creations, common.HexToHash(val)) - delete(deletions, common.HexToHash(val)) } + updatedAccounts[common.HexToHash(val)] = AccountDiff{ + Key: createdAcc.RawKey, + Value: createdAcc.RawValue, + Proof: createdAcc.Proof, + Path: createdAcc.Path, + Storage: storageDiffs, + } + delete(creations, common.HexToHash(val)) + delete(deletions, common.HexToHash(val)) } + return updatedAccounts, nil } -func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffStorage, error) { +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) stateCache := sdb.blockChain.StateCache() sTrie, err := stateCache.OpenTrie(sr) @@ -214,7 +213,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffSt return storageDiffs, nil } -func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffStorage, error) { +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) ([]StorageDiff, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) stateCache := sdb.blockChain.StateCache() @@ -235,21 +234,27 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common return storageDiffs, nil } -func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage { - storageDiffs := make(map[string]DiffStorage) +func buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { + storageDiffs := make([]StorageDiff, 0) for { log.Debug("Iterating over state at path ", "path", pathToStr(it)) if it.Leaf() { log.Debug("Found leaf in storage", "path", pathToStr(it)) - path := pathToStr(it) - storageKey := hexutil.Encode(it.LeafKey()) - storageValue := hexutil.Encode(it.LeafBlob()) - storageDiffs[path] = DiffStorage{ - Key: &storageKey, - Value: &storageValue, - } + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: leafPath, + Proof: leafProof, + }) } - cont := it.Next(true) if !cont { break @@ -261,12 +266,12 @@ func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage { func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...))) - if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)); err != nil { + addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)) + if err != nil { log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err) return nil, err - } else { - addr := common.BytesToAddress(addrBytes) - log.Debug("Address found", "Address", addr) - return &addr, nil } + addr := common.BytesToAddress(addrBytes) + log.Debug("Address found", "Address", addr) + return &addr, nil } diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go deleted file mode 100644 index 4f8b7c468..000000000 --- a/statediff/builder/builder_test.go +++ /dev/null @@ -1,365 +0,0 @@ -package builder_test - -import ( - "bytes" - "math/big" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" - b "github.com/ethereum/go-ethereum/statediff/builder" - "github.com/ethereum/go-ethereum/statediff/testhelpers" -) - -var ( - testdb = ethdb.NewMemDatabase() - - testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 - bankLeafKey = testhelpers.AddressToLeafKey(testBankAddress) - testBankFunds = big.NewInt(100000000) - genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) - - account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 - account1LeafKey = testhelpers.AddressToLeafKey(account1Addr) - account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e - account2LeafKey = testhelpers.AddressToLeafKey(account2Addr) - contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") - contractAddr common.Address - contractLeafKey common.Hash - emptyAccountDiffEventualMap = make(b.AccountDiffsMap) - emptyAccountDiffIncrementalMap = make(b.AccountDiffsMap) - block0Hash, block1Hash, block2Hash, block3Hash common.Hash - block0, block1, block2, block3 *types.Block - builder b.Builder - miningReward = int64(2000000000000000000) - burnAddress = common.HexToAddress("0x0") - burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) -) - -func TestBuilder(t *testing.T) { - _, blockMap, chain := makeChain(3, genesis) - contractLeafKey = testhelpers.AddressToLeafKey(contractAddr) - defer chain.Stop() - block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") - block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") - block2Hash = common.HexToHash("0xde75663f36a8497b4bdda2a4b52bd9540b705a2728c7391c59b8cb2cde5a2feb") - block3Hash = common.HexToHash("0x76c6d0e39285cee40d5e5fadc6141ca88c8ab8bd1a15d46717205af2efbb4a3c") - - block0 = blockMap[block0Hash] - block1 = blockMap[block1Hash] - block2 = blockMap[block2Hash] - block3 = blockMap[block3Hash] - builder = b.NewBuilder(testdb, chain) - - type arguments struct { - oldStateRoot common.Hash - newStateRoot common.Hash - blockNumber int64 - blockHash common.Hash - } - - var ( - balanceChange10000 = int64(10000) - balanceChange1000 = int64(1000) - block1BankBalance = int64(99990000) - block1Account1Balance = int64(10000) - block2Account2Balance = int64(1000) - nonce0 = uint64(0) - nonce1 = uint64(1) - nonce2 = uint64(2) - nonce3 = uint64(3) - originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" - newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" - originalStorageLocation = common.HexToHash("0") - originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).String() - updatedStorageLocation = common.HexToHash("2") - updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).String() - originalStorageValue = "0x01" - updatedStorageValue = "0x03" - ) - - var tests = []struct { - name string - startingArguments arguments - expected *b.StateDiff - }{ - { - "testEmptyDiff", - arguments{ - oldStateRoot: block0.Root(), - newStateRoot: block0.Root(), - blockNumber: block0.Number().Int64(), - blockHash: block0Hash, - }, - &b.StateDiff{ - BlockNumber: block0.Number().Int64(), - BlockHash: block0Hash, - CreatedAccounts: emptyAccountDiffEventualMap, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: emptyAccountDiffIncrementalMap, - }, - }, - { - "testBlock1", - //10000 transferred from testBankAddress to account1Addr - arguments{ - oldStateRoot: block0.Root(), - newStateRoot: block1.Root(), - blockNumber: block1.Number().Int64(), - blockHash: block1Hash, - }, - &b.StateDiff{ - BlockNumber: block1.Number().Int64(), - BlockHash: block1.Hash(), - CreatedAccounts: b.AccountDiffsMap{ - account1LeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(balanceChange10000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - burnLeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(miningReward)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: b.AccountDiffsMap{ - bankLeafKey: { - Nonce: b.DiffUint64{Value: &nonce1}, - Balance: b.DiffBigInt{Value: big.NewInt(testBankFunds.Int64() - balanceChange10000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - }, - }, - { - "testBlock2", - //1000 transferred from testBankAddress to account1Addr - //1000 transferred from account1Addr to account2Addr - arguments{ - oldStateRoot: block1.Root(), - newStateRoot: block2.Root(), - blockNumber: block2.Number().Int64(), - blockHash: block2Hash, - }, - &b.StateDiff{ - BlockNumber: block2.Number().Int64(), - BlockHash: block2.Hash(), - CreatedAccounts: b.AccountDiffsMap{ - account2LeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(balanceChange1000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - contractLeafKey: { - Nonce: b.DiffUint64{Value: &nonce1}, - Balance: b.DiffBigInt{Value: big.NewInt(0)}, - CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", - ContractRoot: b.DiffString{Value: &contractContractRoot}, - Storage: map[string]b.DiffStorage{ - originalStorageKey: { - Key: &originalStorageKey, - Value: &originalStorageValue}, - }, - }, - }, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: b.AccountDiffsMap{ - bankLeafKey: { - Nonce: b.DiffUint64{Value: &nonce2}, - Balance: b.DiffBigInt{Value: big.NewInt(block1BankBalance - balanceChange1000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - account1LeafKey: { - Nonce: b.DiffUint64{Value: &nonce2}, - Balance: b.DiffBigInt{Value: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - burnLeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(miningReward + miningReward)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - }, - }, - { - "testBlock3", - //the contract's storage is changed - //and the block is mined by account 2 - arguments{ - oldStateRoot: block2.Root(), - newStateRoot: block3.Root(), - blockNumber: block3.Number().Int64(), - blockHash: block3.Hash(), - }, - &b.StateDiff{ - BlockNumber: block3.Number().Int64(), - BlockHash: block3.Hash(), - CreatedAccounts: b.AccountDiffsMap{}, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: b.AccountDiffsMap{ - account2LeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(block2Account2Balance + miningReward)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - contractLeafKey: { - Nonce: b.DiffUint64{Value: &nonce1}, - Balance: b.DiffBigInt{Value: big.NewInt(0)}, - CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", - ContractRoot: b.DiffString{Value: &newContractRoot}, - Storage: map[string]b.DiffStorage{ - "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace": { - Key: &updatedStorageKey, - Value: &updatedStorageValue}, - }, - }, - bankLeafKey: { - Nonce: b.DiffUint64{Value: &nonce3}, - Balance: b.DiffBigInt{Value: big.NewInt(99989000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - }, - }, - } - - for _, test := range tests { - arguments := test.startingArguments - diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) - if err != nil { - t.Error(err) - } - fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} - - for _, field := range fields { - reflectionOfDiff := reflect.ValueOf(diff) - diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) - - reflectionOfExpected := reflect.ValueOf(test.expected) - expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) - - diffValueInterface := diffValue.Interface() - expectedValueInterface := expectedValue.Interface() - - if !equals(diffValueInterface, expectedValueInterface) { - t.Logf("Test failed: %s", test.name) - t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) - } - } - } -} - -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) { - blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - chain, _ := core.NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil) - - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - blockm[b.Hash()] = b - } - return hashes, blockm, chain -} - -func testChainGen(i int, block *core.BlockGen) { - signer := types.HomesteadSigner{} - switch i { - case 0: - // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) - block.AddTx(tx) - case 1: - // In block 2, the test bank sends some more ether to account #1. - // account1Addr passes it on to account #2. - // account1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - nonce := block.TxNonce(account1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key) - nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key) - contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 - block.AddTx(tx1) - block.AddTx(tx2) - block.AddTx(tx3) - case 2: - // Block 3 is empty but was mined by account #2. - block.SetCoinbase(account2Addr) - //get function: 60cd2685 - //put function: c16431b9 - data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) - block.AddTx(tx) - } -} - -/* -contract test { - - uint256[100] data; - - constructor() public { - data = [1]; - } - - function Put(uint256 addr, uint256 value) { - data[addr] = value; - } - - function Get(uint256 addr) constant returns (uint256 value) { - return data[addr]; - } -} -*/ diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go deleted file mode 100644 index 339454776..000000000 --- a/statediff/builder/helpers.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package builder - -import ( - "sort" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/trie" -) - -func sortKeys(data AccountsMap) []string { - var keys []string - for key := range data { - keys = append(keys, key.Hex()) - } - sort.Strings(keys) - - return keys -} - -func findIntersection(a, b []string) []string { - lenA := len(a) - lenB := len(b) - iOfA, iOfB := 0, 0 - updates := make([]string, 0) - if iOfA >= lenA || iOfB >= lenB { - return updates - } - for { - switch strings.Compare(a[iOfA], b[iOfB]) { - // a[iOfA] < b[iOfB] - case -1: - iOfA++ - if iOfA >= lenA { - return updates - } - // a[iOfA] == b[iOfB] - case 0: - updates = append(updates, a[iOfA]) - iOfA++ - iOfB++ - if iOfA >= lenA || iOfB >= lenB { - return updates - } - // a[iOfA] > b[iOfB] - case 1: - iOfB++ - if iOfB >= lenB { - return updates - } - } - } - -} - -func pathToStr(it trie.NodeIterator) string { - path := it.Path() - if hasTerm(path) { - path = path[:len(path)-1] - } - nibblePath := "" - for i, v := range common.ToHex(path) { - if i%2 == 0 && i > 1 { - continue - } - nibblePath = nibblePath + string(v) - } - - return nibblePath -} - -// Duplicated from trie/encoding.go -func hexToKeyBytes(hex []byte) []byte { - if hasTerm(hex) { - hex = hex[:len(hex)-1] - } - if len(hex)&1 != 0 { - panic("can't convert hex key of odd length") - } - key := make([]byte, (len(hex)+1)/2) - decodeNibbles(hex, key) - - return key -} - -func decodeNibbles(nibbles []byte, bytes []byte) { - for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { - bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] - } -} - -// hasTerm returns whether a hex key has the terminator flag. -func hasTerm(s []byte) bool { - return len(s) > 0 && s[len(s)-1] == 16 -} diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go deleted file mode 100644 index 416d8e816..000000000 --- a/statediff/builder/struct.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package builder - -import ( - "encoding/json" - "math/big" - - "github.com/ethereum/go-ethereum/common" -) - -type AccountDiffsMap map[common.Hash]AccountDiff -type StateDiff struct { - BlockNumber int64 `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` - DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` - UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` - - encoded []byte - err error -} - -func (self *StateDiff) ensureEncoded() { - if self.encoded == nil && self.err == nil { - self.encoded, self.err = json.Marshal(self) - } -} - -// Implement Encoder interface for StateDiff -func (sd *StateDiff) Length() int { - sd.ensureEncoded() - return len(sd.encoded) -} - -// Implement Encoder interface for StateDiff -func (sd *StateDiff) Encode() ([]byte, error) { - sd.ensureEncoded() - return sd.encoded, sd.err -} - -type AccountDiff struct { - Nonce DiffUint64 `json:"nonce" gencodec:"required"` - Balance DiffBigInt `json:"balance" gencodec:"required"` - CodeHash string `json:"codeHash" gencodec:"required"` - ContractRoot DiffString `json:"contractRoot" gencodec:"required"` - Storage map[string]DiffStorage `json:"storage" gencodec:"required"` -} - -type DiffStorage struct { - Key *string `json:"key" gencodec:"optional"` - Value *string `json:"value" gencodec:"optional"` -} -type DiffString struct { - Value *string `json:"value" gencodec:"optional"` -} -type DiffUint64 struct { - Value *uint64 `json:"value" gencodec:"optional"` -} -type DiffBigInt struct { - Value *big.Int `json:"value" gencodec:"optional"` -} diff --git a/statediff/builder_test.go b/statediff/builder_test.go new file mode 100644 index 000000000..b4df80f81 --- /dev/null +++ b/statediff/builder_test.go @@ -0,0 +1,392 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" +) + +var ( + contractLeafKey common.Hash + emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) + emptyAccountDiffIncrementalMap = make(statediff.AccountDiffsMap) + block0Hash, block1Hash, block2Hash, block3Hash common.Hash + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(2000000000000000000) + burnAddress = common.HexToAddress("0x0") + burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) +) + +func TestBuilder(t *testing.T) { + _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block2Hash = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5") + block3Hash = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a") + + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + block2 = blockMap[block2Hash] + block3 = blockMap[block3Hash] + builder = statediff.NewBuilder(testhelpers.Testdb, chain) + + type arguments struct { + oldStateRoot common.Hash + newStateRoot common.Hash + blockNumber int64 + blockHash common.Hash + } + + var ( + balanceChange10000 = int64(10000) + balanceChange1000 = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + block2Account2Balance = int64(1000) + nonce0 = uint64(0) + nonce1 = uint64(1) + nonce2 = uint64(2) + nonce3 = uint64(3) + originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" + originalStorageLocation = common.HexToHash("0") + originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes() + updatedStorageLocation = common.HexToHash("2") + updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes() + originalStorageValue = common.Hex2Bytes("01") + updatedStorageValue = common.Hex2Bytes("03") + account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + bankAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(contractContractRoot), + }) + bankAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1BankBalance - balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(block2Account2Balance + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(newContractRoot), + }) + bankAccount3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce3, + Balance: big.NewInt(99989000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + ) + + var tests = []struct { + name string + startingArguments arguments + expected *statediff.StateDiff + }{ + { + "testEmptyDiff", + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block0.Root(), + blockNumber: block0.Number().Int64(), + blockHash: block0Hash, + }, + &statediff.StateDiff{ + BlockNumber: block0.Number().Int64(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block1.Root(), + blockNumber: block1.Number().Int64(), + blockHash: block1Hash, + }, + &statediff.StateDiff{ + BlockNumber: block1.Number().Int64(), + BlockHash: block1.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account1LeafKey: { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + burnLeafKey: { + Key: burnLeafKey.Bytes(), + Value: burnAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 1, 132, 5, 245, 185, 240, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + arguments{ + oldStateRoot: block1.Root(), + newStateRoot: block2.Root(), + blockNumber: block2.Number().Int64(), + blockHash: block2Hash, + }, + &statediff.StateDiff{ + BlockNumber: block2.Number().Int64(), + BlockHash: block2.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account2LeafKey: { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + Storage: []statediff.StorageDiff{}, + }, + contractLeafKey: { + Key: contractLeafKey.Bytes(), + Value: contractAccount, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 130, 30, 37, 86, 162, 144, 200, 100, 5, 248, 22, 10, 45, 102, 32, 66, 164, 49, 186, 69, 107, 157, 178, 101, 199, 155, 184, 55, 192, 75, 229, 240, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Key: originalStorageKey, + Value: originalStorageValue, + Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}}, + Path: []byte{2, 9, 0, 13, 14, 12, 13, 9, 5, 4, 8, 11, 6, 2, 10, 8, 13, 6, 0, 3, 4, 5, 10, 9, 8, 8, 3, 8, 6, 15, 12, 8, 4, 11, 10, 6, 11, 12, 9, 5, 4, 8, 4, 0, 0, 8, 15, 6, 3, 6, 2, 15, 9, 3, 1, 6, 0, 14, 15, 3, 14, 5, 6, 3, 16}, + }, + }, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 2, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + testhelpers.Account1LeafKey: { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account3, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + burnLeafKey: { + Key: burnLeafKey.Bytes(), + Value: burnAccount2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 55, 130, 218, 206, 157, 144, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + arguments{ + oldStateRoot: block2.Root(), + newStateRoot: block3.Root(), + blockNumber: block3.Number().Int64(), + blockHash: block3.Hash(), + }, + &statediff.StateDiff{ + BlockNumber: block3.Number().Int64(), + BlockHash: block3.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{}, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account2LeafKey: { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account4, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + Storage: []statediff.StorageDiff{}, + }, + contractLeafKey: { + Key: contractLeafKey.Bytes(), + Value: contractAccount2, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 113, 224, 209, 75, 43, 147, 229, 199, 249, 116, 142, 105, 225, 254, 95, 23, 73, 138, 28, 58, 195, 206, 194, 159, 150, 175, 19, 215, 248, 164, 224, 112, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Key: updatedStorageKey, + Value: updatedStorageValue, + Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, + {226, 160, 48, 87, 135, 250, 18, 168, 35, 224, 242, 183, 99, 28, 196, 27, 59, 168, 130, 139, 51, 33, 202, 129, 17, 17, 250, 117, 205, 58, 163, 187, 90, 206, 3}}, + Path: []byte{4, 0, 5, 7, 8, 7, 15, 10, 1, 2, 10, 8, 2, 3, 14, 0, 15, 2, 11, 7, 6, 3, 1, 12, 12, 4, 1, 11, 3, 11, 10, 8, 8, 2, 8, 11, 3, 3, 2, 1, 12, 10, 8, 1, 1, 1, 1, 1, 15, 10, 7, 5, 12, 13, 3, 10, 10, 3, 11, 11, 5, 10, 12, 14, 16}, + }, + }, + }, + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount3, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + } + + for _, test := range tests { + arguments := test.startingArguments + diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) + if err != nil { + t.Error(err) + } + fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} + + for _, field := range fields { + reflectionOfDiff := reflect.ValueOf(diff) + diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) + + reflectionOfExpected := reflect.ValueOf(test.expected) + expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) + + diffValueInterface := diffValue.Interface() + expectedValueInterface := expectedValue.Interface() + + if !equals(diffValueInterface, expectedValueInterface) { + t.Logf("Test failed: %s", test.name) + t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) + } + } + } +} + +func equals(actual, expected interface{}) (success bool) { + if actualByteSlice, ok := actual.([]byte); ok { + if expectedByteSlice, ok := expected.([]byte); ok { + return bytes.Equal(actualByteSlice, expectedByteSlice) + } + } + + return reflect.DeepEqual(actual, expected) +} + +/* +contract test { + + uint256[100] data; + + constructor() public { + data = [1]; + } + + function Put(uint256 addr, uint256 value) { + data[addr] = value; + } + + function Get(uint256 addr) constant returns (uint256 value) { + return data[addr]; + } +} +*/ diff --git a/statediff/types.go b/statediff/types.go new file mode 100644 index 000000000..41e83dddb --- /dev/null +++ b/statediff/types.go @@ -0,0 +1,113 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" +) + +// AccountsMap is a mapping of keccak256(address) => accountWrapper +type AccountsMap map[common.Hash]accountWrapper + +// AccountWrapper is used to temporary associate the unpacked account with its raw values +type accountWrapper struct { + Account state.Account + RawKey []byte + RawValue []byte + Proof [][]byte + Path []byte +} + +// StateDiff is the final output structure from the builder +type StateDiff struct { + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` + DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` + + encoded []byte + err error +} + +func (sd *StateDiff) ensureEncoded() { + if sd.encoded == nil && sd.err == nil { + sd.encoded, sd.err = json.Marshal(sd) + } +} + +// Length to implement Encoder interface for StateDiff +func (sd *StateDiff) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Encode to implement Encoder interface for StateDiff +func (sd *StateDiff) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +// AccountDiffsMap is a mapping of keccak256(address) => AccountDiff +type AccountDiffsMap map[common.Hash]AccountDiff + +// AccountDiff holds the data for a single state diff leaf node +type AccountDiff struct { + Key []byte `json:"key" gencodec:"required"` + Value []byte `json:"value" gencodec:"required"` + Proof [][]byte `json:"proof" gencodec:"required"` + Storage []StorageDiff `json:"storage" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` +} + +// StorageDiff holds the data for a single storage diff leaf node +type StorageDiff struct { + Key []byte `json:"key" gencodec:"required"` + Value []byte `json:"value" gencodec:"required"` + Proof [][]byte `json:"proof" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` +} + +/* +// State trie leaf is just a short node, below +// that has an rlp encoded account as the value + + +// SO each account diffs map is reall a map of shortnode keys to values +// Flatten to a slice of short nodes? + +// Need to coerce into: + +type TrieNode struct { + // leaf, extension or branch + nodeKind string + + // If leaf or extension: [0] is key, [1] is val. + // If branch: [0] - [16] are children. + elements []interface{} + + // IPLD block information + cid *cid.Cid + rawdata []byte +} +*/ -- 2.45.2 From 0471cf2826ee45613c73e1f20c46b3147e8bc081 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 26 Apr 2019 11:18:23 -0500 Subject: [PATCH 04/14] refactoring state diff service and adding api which allows for streaming state diff payloads over an rpc websocket subscription --- statediff/api.go | 91 ++++++++ statediff/builder.go | 2 +- statediff/builder_test.go | 2 +- statediff/helpers.go | 2 +- statediff/service.go | 244 ++++++++++++++++++++ statediff/service/service.go | 120 ---------- statediff/service/service_test.go | 107 --------- statediff/service_test.go | 130 +++++++++++ statediff/testhelpers/helpers.go | 2 +- statediff/testhelpers/mocks/blockchain.go | 2 +- statediff/testhelpers/mocks/builder.go | 2 +- statediff/testhelpers/mocks/publisher.go | 2 +- statediff/testhelpers/mocks/service.go | 171 ++++++++++++++ statediff/testhelpers/mocks/service_test.go | 127 ++++++++++ statediff/testhelpers/test_data.go | 2 +- statediff/types.go | 2 +- 16 files changed, 772 insertions(+), 236 deletions(-) create mode 100644 statediff/api.go create mode 100644 statediff/service.go delete mode 100644 statediff/service/service.go delete mode 100644 statediff/service/service_test.go create mode 100644 statediff/service_test.go create mode 100644 statediff/testhelpers/mocks/service.go create mode 100644 statediff/testhelpers/mocks/service_test.go diff --git a/statediff/api.go b/statediff/api.go new file mode 100644 index 000000000..26f03c9c3 --- /dev/null +++ b/statediff/api.go @@ -0,0 +1,91 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "context" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +// APIName is the namespace used for the state diffing service API +const APIName = "statediff" + +// APIVersion is the version of the state diffing service API +const APIVersion = "0.0.1" + +// PublicStateDiffAPI provides the a websocket service +// that can be used to stream out state diffs as they +// are produced by a full node +type PublicStateDiffAPI struct { + sds IService + + mu sync.Mutex + lastUsed map[string]time.Time // keeps track when a filter was polled for the last time. +} + +// NewPublicStateDiffAPI create a new state diff websocket streaming service. +func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { + return &PublicStateDiffAPI{ + sds: sds, + lastUsed: make(map[string]time.Time), + mu: sync.Mutex{}, + } +} + +// Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created +func (api *PublicStateDiffAPI) Subscribe(ctx context.Context) (*rpc.Subscription, error) { + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // create subscription and start waiting for statediff events + rpcSub := notifier.CreateSubscription() + + go func() { + // subscribe to events from the state diff service + payloadChannel := make(chan Payload) + quitChan := make(chan bool) + api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) + + // loop and await state diff payloads and relay them to the subscriber with then notifier + for { + select { + case packet := <-payloadChannel: + if err := notifier.Notify(rpcSub.ID, packet); err != nil { + log.Error("Failed to send state diff packet", "err", err) + } + case <-rpcSub.Err(): + err := api.sds.Unsubscribe(rpcSub.ID) + if err != nil { + log.Error("Failed to unsubscribe from the state diff service", err) + } + return + case <-quitChan: + // don't need to unsubscribe, statediff service does so before sending the quit signal + return + } + } + }() + + return rpcSub, nil +} diff --git a/statediff/builder.go b/statediff/builder.go index 4e8ab0e21..05e7d8572 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/builder_test.go b/statediff/builder_test.go index b4df80f81..5ecf647e3 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/helpers.go b/statediff/helpers.go index 96f9ddf30..89852b55c 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000..d03a7b513 --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,244 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "bytes" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rlp" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" +) + +type blockChain interface { + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + GetBlockByHash(hash common.Hash) *types.Block + AddToStateDiffProcessedCollection(hash common.Hash) +} + +// IService is the state-diffing service interface +type IService interface { + // APIs(), Protocols(), Start() and Stop() + node.Service + // Main event loop for processing state diffs + Loop(chainEventCh chan core.ChainEvent) + // Method to subscribe to receive state diff processing output + Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) + // Method to unsubscribe from state diff processing + Unsubscribe(id rpc.ID) error +} + +// Service is the underlying struct for the state diffing service +type Service struct { + // Used to sync access to the Subscriptions + sync.Mutex + // Used to build the state diff objects + Builder Builder + // Used to subscribe to chain events (blocks) + BlockChain blockChain + // Used to signal shutdown of the service + QuitChan chan bool + // A mapping of rpc.IDs to their subscription channels + Subscriptions map[rpc.ID]Subscription +} + +// Subscription struct holds our subscription channels +type Subscription struct { + PayloadChan chan<- Payload + QuitChan chan<- bool +} + +// Payload packages the data to send to StateDiffingService subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp" gencodec:"required"` + StateDiffRlp []byte `json:"stateDiffRlp" gencodec:"required"` + Err error `json:"error"` +} + +// NewStateDiffService creates a new StateDiffingService +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*Service, error) { + return &Service{ + Mutex: sync.Mutex{}, + BlockChain: blockChain, + Builder: NewBuilder(db, blockChain), + QuitChan: make(chan bool), + Subscriptions: make(map[rpc.ID]Subscription), + }, nil +} + +// Protocols exports the services p2p protocols, this service has none +func (sds *Service) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs returns the RPC descriptors the StateDiffingService offers +func (sds *Service) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: APIName, + Version: APIVersion, + Service: NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Loop is the main processing method +func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { + + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() + + blocksCh := make(chan *types.Block, 10) + errCh := chainEventSub.Err() + + go func() { + HandleChainEventChLoop: + for { + select { + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + log.Debug("Event received from chainEventCh", "event", chainEvent) + blocksCh <- chainEvent.Block + //if node stopped + case err := <-errCh: + log.Warn("Error from chain event subscription, breaking loop.", "error", err) + close(sds.QuitChan) + break HandleChainEventChLoop + case <-sds.QuitChan: + break HandleChainEventChLoop + } + } + }() + + //loop through chain events until no more +HandleBlockChLoop: + for { + select { + case block := <-blocksCh: + currentBlock := block + parentHash := currentBlock.ParentHash() + parentBlock := sds.BlockChain.GetBlockByHash(parentHash) + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + break HandleBlockChLoop + } + + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + if err != nil { + log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) + } + rlpBuff := new(bytes.Buffer) + currentBlock.EncodeRLP(rlpBuff) + blockRlp := rlpBuff.Bytes() + stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) + payload := Payload{ + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, + } + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// Subscribe is used by the API to subscribe to the StateDiffingService loop +func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { + log.Info("Subscribing to the statediff service") + sds.Lock() + sds.Subscriptions[id] = Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.Unlock() +} + +// Unsubscribe is used to unsubscribe to the StateDiffingService loop +func (sds *Service) Unsubscribe(id rpc.ID) error { + log.Info("Unsubscribing from the statediff service") + sds.Lock() + _, ok := sds.Subscriptions[id] + if !ok { + return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id) + } + delete(sds.Subscriptions, id) + sds.Unlock() + return nil +} + +// Start is used to begin the StateDiffingService +func (sds *Service) Start(*p2p.Server) error { + log.Info("Starting statediff service") + + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + + return nil +} + +// Stop is used to close down the StateDiffingService +func (sds *Service) Stop() error { + log.Info("Stopping statediff service") + close(sds.QuitChan) + return nil +} + +// send is used to fan out and serve a payload to any subscriptions +func (sds *Service) send(payload Payload) { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.PayloadChan <- payload: + log.Info("sending state diff payload to subscription %s", id) + default: + log.Info("unable to send payload to subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} + +// close is used to close all listening subscriptions +func (sds *Service) close() { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.QuitChan <- true: + delete(sds.Subscriptions, id) + log.Info("closing subscription %s", id) + default: + log.Info("unable to close subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} diff --git a/statediff/service/service.go b/statediff/service/service.go deleted file mode 100644 index 19ea0c644..000000000 --- a/statediff/service/service.go +++ /dev/null @@ -1,120 +0,0 @@ -package service - -import ( - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff" - b "github.com/ethereum/go-ethereum/statediff/builder" - e "github.com/ethereum/go-ethereum/statediff/extractor" - p "github.com/ethereum/go-ethereum/statediff/publisher" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" -) - -type BlockChain interface { - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - GetBlockByHash(hash common.Hash) *types.Block - AddToStateDiffProcessedCollection(hash common.Hash) -} - -type StateDiffService struct { - Builder *b.Builder - Extractor e.Extractor - BlockChain BlockChain -} - -func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config statediff.Config) (*StateDiffService, error) { - builder := b.NewBuilder(db, blockChain) - publisher, err := p.NewPublisher(config) - if err != nil { - return nil, err - } - - extractor := e.NewExtractor(builder, publisher) - return &StateDiffService{ - BlockChain: blockChain, - Extractor: extractor, - }, nil -} - -func (StateDiffService) Protocols() []p2p.Protocol { - return []p2p.Protocol{} -} - -func (StateDiffService) APIs() []rpc.API { - return []rpc.API{} -} - -func (sds *StateDiffService) Loop(chainEventCh chan core.ChainEvent) { - chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) - defer chainEventSub.Unsubscribe() - - blocksCh := make(chan *types.Block, 10) - errCh := chainEventSub.Err() - quitCh := make(chan struct{}) - - go func() { - HandleChainEventChLoop: - for { - select { - //Notify chain event channel of events - case chainEvent := <-chainEventCh: - log.Debug("Event received from chainEventCh", "event", chainEvent) - blocksCh <- chainEvent.Block - //if node stopped - case err := <-errCh: - log.Warn("Error from chain event subscription, breaking loop.", "error", err) - break HandleChainEventChLoop - } - } - close(quitCh) - }() - - //loop through chain events until no more -HandleBlockChLoop: - for { - select { - case block := <-blocksCh: - currentBlock := block - parentHash := currentBlock.ParentHash() - parentBlock := sds.BlockChain.GetBlockByHash(parentHash) - if parentBlock == nil { - log.Error("Parent block is nil, skipping this block", - "parent block hash", parentHash.String(), - "current block number", currentBlock.Number()) - break HandleBlockChLoop - } - - stateDiffLocation, err := sds.Extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err) - } else { - log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation) - sds.BlockChain.AddToStateDiffProcessedCollection(parentBlock.Root()) - sds.BlockChain.AddToStateDiffProcessedCollection(currentBlock.Root()) - } - case <-quitCh: - log.Debug("Quitting the statediff block channel") - return - } - } -} - -func (sds *StateDiffService) Start(server *p2p.Server) error { - log.Info("Starting statediff service") - - chainEventCh := make(chan core.ChainEvent, 10) - go sds.Loop(chainEventCh) - - return nil -} - -func (StateDiffService) Stop() error { - log.Info("Stopping statediff service") - return nil -} diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go deleted file mode 100644 index daf3445c3..000000000 --- a/statediff/service/service_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package service_test - -import ( - "math/big" - "math/rand" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - s "github.com/ethereum/go-ethereum/statediff/service" - "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" -) - -func TestServiceLoop(t *testing.T) { - testErrorInChainEventLoop(t) - testErrorInBlockLoop(t) -} - -var ( - eventsChannel = make(chan core.ChainEvent, 1) - - parentHeader1 = types.Header{Number: big.NewInt(rand.Int63())} - parentHeader2 = types.Header{Number: big.NewInt(rand.Int63())} - - parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil) - parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil) - - parentHash1 = parentBlock1.Hash() - parentHash2 = parentBlock2.Hash() - - header1 = types.Header{ParentHash: parentHash1} - header2 = types.Header{ParentHash: parentHash2} - header3 = types.Header{ParentHash: common.HexToHash("parent hash")} - - block1 = types.NewBlock(&header1, nil, nil, nil) - block2 = types.NewBlock(&header2, nil, nil, nil) - block3 = types.NewBlock(&header3, nil, nil, nil) - - event1 = core.ChainEvent{Block: block1} - event2 = core.ChainEvent{Block: block2} - event3 = core.ChainEvent{Block: block3} -) - -func testErrorInChainEventLoop(t *testing.T) { - //the first chain event causes and error (in blockchain mock) - extractor := mocks.Extractor{} - - blockChain := mocks.BlockChain{} - service := s.StateDiffService{ - Builder: nil, - Extractor: &extractor, - BlockChain: &blockChain, - } - - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) - blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) - service.Loop(eventsChannel) - - //parent and current blocks are passed to the extractor - expectedCurrentBlocks := []types.Block{*block1, *block2} - if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks) - } - expectedParentBlocks := []types.Block{*parentBlock1, *parentBlock2} - if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks) - } - - //look up the parent block from its hash - expectedHashes := []common.Hash{block1.ParentHash(), block2.ParentHash()} - if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) - } -} - -func testErrorInBlockLoop(t *testing.T) { - //second block's parent block can't be found - extractor := mocks.Extractor{} - - blockChain := mocks.BlockChain{} - service := s.StateDiffService{ - Builder: nil, - Extractor: &extractor, - BlockChain: &blockChain, - } - - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) - blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) - service.Loop(eventsChannel) - - //only the first current block (and it's parent) are passed to the extractor - expectedCurrentBlocks := []types.Block{*block1} - if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks) - } - expectedParentBlocks := []types.Block{*parentBlock1} - if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks) - } -} diff --git a/statediff/service_test.go b/statediff/service_test.go new file mode 100644 index 000000000..d5edee04e --- /dev/null +++ b/statediff/service_test.go @@ -0,0 +1,130 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "math/rand" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" +) + +func TestServiceLoop(t *testing.T) { + testErrorInChainEventLoop(t) + testErrorInBlockLoop(t) +} + +var ( + eventsChannel = make(chan core.ChainEvent, 1) + + parentRoot1 = common.HexToHash("0x01") + parentRoot2 = common.HexToHash("0x02") + parentHeader1 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot1} + parentHeader2 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot2} + + parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil) + parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil) + + parentHash1 = parentBlock1.Hash() + parentHash2 = parentBlock2.Hash() + + testRoot1 = common.HexToHash("0x03") + testRoot2 = common.HexToHash("0x04") + testRoot3 = common.HexToHash("0x04") + header1 = types.Header{ParentHash: parentHash1, Root: testRoot1} + header2 = types.Header{ParentHash: parentHash2, Root: testRoot2} + header3 = types.Header{ParentHash: common.HexToHash("parent hash"), Root: testRoot3} + + testBlock1 = types.NewBlock(&header1, nil, nil, nil) + testBlock2 = types.NewBlock(&header2, nil, nil, nil) + testBlock3 = types.NewBlock(&header3, nil, nil, nil) + + event1 = core.ChainEvent{Block: testBlock1} + event2 = core.ChainEvent{Block: testBlock2} + event3 = core.ChainEvent{Block: testBlock3} +) + +func testErrorInChainEventLoop(t *testing.T) { + //the first chain event causes and error (in blockchain mock) + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + service := statediff.Service{ + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[rpc.ID]statediff.Subscription), + } + testRoot2 = common.HexToHash("0xTestRoot2") + blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) + service.Loop(eventsChannel) + if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) + } + if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) + } + if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) + } + //look up the parent block from its hash + expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} + if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) + } +} + +func testErrorInBlockLoop(t *testing.T) { + //second block's parent block can't be found + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + service := statediff.Service{ + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[rpc.ID]statediff.Subscription), + } + + blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + service.Loop(eventsChannel) + + if !bytes.Equal(builder.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock1.Hash()) + } + if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock1.Root()) + } + if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock1.Root()) + } +} diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go index 5126c6556..8f52bc8cc 100644 --- a/statediff/testhelpers/helpers.go +++ b/statediff/testhelpers/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index cececde6f..f2c097d38 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index e9668629e..bb2e38f76 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go index 3ea18abf0..6a6018746 100644 --- a/statediff/testhelpers/mocks/publisher.go +++ b/statediff/testhelpers/mocks/publisher.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go new file mode 100644 index 000000000..edb2d453d --- /dev/null +++ b/statediff/testhelpers/mocks/service.go @@ -0,0 +1,171 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" +) + +// MockStateDiffService is a mock state diff service +type MockStateDiffService struct { + sync.Mutex + Builder statediff.Builder + ReturnProtocol []p2p.Protocol + ReturnAPIs []rpc.API + BlockChan chan *types.Block + ParentBlockChan chan *types.Block + QuitChan chan bool + Subscriptions map[rpc.ID]statediff.Subscription +} + +// Protocols mock method +func (sds *MockStateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs mock method +func (sds *MockStateDiffService) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: statediff.APIName, + Version: statediff.APIVersion, + Service: statediff.NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Loop mock method +func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { + //loop through chain events until no more +HandleBlockChLoop: + for { + select { + case block := <-sds.BlockChan: + currentBlock := block + parentBlock := <-sds.ParentBlockChan + parentHash := parentBlock.Hash() + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + break HandleBlockChLoop + } + + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + if err != nil { + log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) + } + rlpBuff := new(bytes.Buffer) + currentBlock.EncodeRLP(rlpBuff) + blockRlp := rlpBuff.Bytes() + stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) + payload := statediff.Payload{ + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, + } + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// Subscribe mock method +func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool) { + log.Info("Subscribing to the statediff service") + sds.Lock() + sds.Subscriptions[id] = statediff.Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.Unlock() +} + +// Unsubscribe mock method +func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error { + log.Info("Unsubscribing from the statediff service") + sds.Lock() + _, ok := sds.Subscriptions[id] + if !ok { + return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id) + } + delete(sds.Subscriptions, id) + sds.Unlock() + return nil +} + +func (sds *MockStateDiffService) send(payload statediff.Payload) { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.PayloadChan <- payload: + log.Info("sending state diff payload to subscription %s", id) + default: + log.Info("unable to send payload to subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} + +func (sds *MockStateDiffService) close() { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.QuitChan <- true: + delete(sds.Subscriptions, id) + log.Info("closing subscription %s", id) + default: + log.Info("unable to close subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} + +// Start mock method +func (sds *MockStateDiffService) Start(server *p2p.Server) error { + log.Info("Starting statediff service") + if sds.ParentBlockChan == nil || sds.BlockChan == nil { + return errors.New("mock StateDiffingService requires preconfiguration with a MockParentBlockChan and MockBlockChan") + } + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + + return nil +} + +// Stop mock method +func (sds *MockStateDiffService) Stop() error { + log.Info("Stopping statediff service") + close(sds.QuitChan) + return nil +} diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go new file mode 100644 index 000000000..1a4f83cf5 --- /dev/null +++ b/statediff/testhelpers/mocks/service_test.go @@ -0,0 +1,127 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "math/big" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" +) + +var block0, block1 *types.Block +var burnLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0")) +var emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) +var account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), +}) +var burnAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(2000000000000000000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), +}) +var bankAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(1), + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - 10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), +}) + +func TestAPI(t *testing.T) { + _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) + defer chain.Stop() + block0Hash := common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash := common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + blockChan := make(chan *types.Block) + parentBlockChain := make(chan *types.Block) + serviceQuitChan := make(chan bool) + mockService := MockStateDiffService{ + Mutex: sync.Mutex{}, + Builder: statediff.NewBuilder(testhelpers.Testdb, chain), + BlockChan: blockChan, + ParentBlockChan: parentBlockChain, + QuitChan: serviceQuitChan, + Subscriptions: make(map[rpc.ID]statediff.Subscription), + } + mockService.Start(nil) + id := rpc.NewID() + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + mockService.Subscribe(id, payloadChan, quitChan) + blockChan <- block1 + parentBlockChain <- block0 + expectedBlockRlp, _ := rlp.EncodeToBytes(block1) + expectedStateDiff := &statediff.StateDiff{ + BlockNumber: block1.Number().Int64(), + BlockHash: block1.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account1LeafKey: { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + burnLeafKey: { + Key: burnLeafKey.Bytes(), + Value: burnAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 1, 132, 5, 245, 185, 240, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + } + expectedStateDiffRlp, _ := rlp.EncodeToBytes(expectedStateDiff) + select { + case payload := <-payloadChan: + if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { + t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp) + } + if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffRlp) { + t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiffRlp, expectedStateDiffRlp) + } + case <-quitChan: + t.Errorf("channel quit before delivering payload") + } +} diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 17dac8473..1203be611 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/types.go b/statediff/types.go index 41e83dddb..816aae6d6 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify -- 2.45.2 From d1a1b8a298988ed05e6a3f561a3b7f1a0e42391c Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 13 May 2019 10:31:42 -0500 Subject: [PATCH 05/14] make proofs and paths optional + compress service loop into single for loop (may be missing something here) --- cmd/geth/main.go | 2 + cmd/geth/usage.go | 2 + cmd/utils/flags.go | 14 +- statediff/builder.go | 191 ++++++++++++-------- statediff/builder_test.go | 5 +- statediff/config.go | 23 +++ statediff/service.go | 97 ++++------ statediff/testhelpers/mocks/service.go | 13 +- statediff/testhelpers/mocks/service_test.go | 20 +- statediff/types.go | 13 ++ 10 files changed, 235 insertions(+), 145 deletions(-) create mode 100644 statediff/config.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d27c3b109..a0831c8f0 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -148,6 +148,8 @@ var ( utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, utils.StateDiffFlag, + utils.StateDiffPathsAndProofs, + utils.StateDiffLeafNodesOnly, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 77d816476..c30746b76 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -266,6 +266,8 @@ var AppHelpFlagGroups = []flagGroup{ Name: "STATE DIFF", Flags: []cli.Flag{ utils.StateDiffFlag, + utils.StateDiffPathsAndProofs, + utils.StateDiffLeafNodesOnly, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b76c5dfe0..48ccb434a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -745,6 +745,14 @@ var ( Name: "statediff", Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", } + StateDiffPathsAndProofs = cli.BoolFlag{ + Name: "statediff.pathsandproofs", + Usage: "Path and proof sets for the state and storage nodes are generated", + } + StateDiffLeafNodesOnly = cli.BoolFlag{ + Name: "statediff.leafs", + Usage: "Consider only leaf nodes of the storage and state tries", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1600,12 +1608,16 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st // RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { + config := statediff.Config{ + PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), + LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name), + } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum ctx.Service(ðServ) chainDb := ethServ.ChainDb() blockChain := ethServ.BlockChain() - return statediff.NewStateDiffService(chainDb, blockChain) + return statediff.NewStateDiffService(chainDb, blockChain, config) }); err != nil { Fatalf("Failed to register State Diff Service", err) } diff --git a/statediff/builder.go b/statediff/builder.go index 05e7d8572..04e00ae95 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -20,6 +20,7 @@ package statediff import ( + "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" @@ -37,13 +38,15 @@ type Builder interface { type builder struct { chainDB ethdb.Database + config Config blockChain *core.BlockChain } -// NewBuilder is used to create a builder -func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) Builder { +// NewBuilder is used to create a state diff builder +func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) Builder { return &builder{ chainDB: db, + config: config, blockChain: blockChain, } } @@ -54,13 +57,11 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block stateCache := sdb.blockChain.StateCache() oldTrie, err := stateCache.OpenTrie(oldStateRoot) if err != nil { - log.Error("Error creating trie for oldStateRoot", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err) } newTrie, err := stateCache.OpenTrie(newStateRoot) if err != nil { - log.Error("Error creating trie for newStateRoot", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error creating trie for newStateRoot: %v", err) } // Find created accounts @@ -68,8 +69,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt := newTrie.NodeIterator([]byte{}) creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { - log.Error("Error collecting creation diff nodes", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error collecting creation diff nodes: %v", err) } // Find deleted accounts @@ -77,8 +77,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt = newTrie.NodeIterator([]byte{}) deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { - log.Error("Error collecting deletion diff nodes", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error collecting deletion diff nodes: %v", err) } // Find all the diffed keys @@ -89,18 +88,15 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Build and return the statediff updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { - log.Error("Error building diff for updated accounts", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error building diff for updated accounts: %v", err) } createdAccounts, err := sdb.buildDiffEventual(creations) if err != nil { - log.Error("Error building diff for created accounts", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error building diff for created accounts: %v", err) } deletedAccounts, err := sdb.buildDiffEventual(deletions) if err != nil { - log.Error("Error building diff for deleted accounts", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error building diff for deleted accounts: %v", err) } return StateDiff{ @@ -116,38 +112,69 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error var diffAccounts = make(AccountsMap) it, _ := trie.NewDifferenceIterator(a, b) - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) - if it.Leaf() { - leafProof := make([][]byte, len(it.LeafProof())) - copy(leafProof, it.LeafProof()) - leafPath := make([]byte, len(it.Path())) - copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - log.Error("Error looking up account via address", "address", leafKeyHash, "error", err) - return nil, err + if sdb.config.PathsAndProofs { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) + if it.Leaf() { + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: account, + RawKey: leafKey, + RawValue: leafValue, + Proof: leafProof, + Path: leafPath, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw } - aw := accountWrapper{ - Account: account, - RawKey: leafKey, - RawValue: leafValue, - Proof: leafProof, - Path: leafPath, + cont := it.Next(true) + if !cont { + break } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: account, + RawKey: leafKey, + RawValue: leafValue, + Proof: nil, + Path: nil, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw + } + cont := it.Next(true) + if !cont { + break + } } } @@ -159,8 +186,7 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er for _, val := range accounts { storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root) if err != nil { - log.Error("Failed building eventual storage diffs", "Address", common.BytesToHash(val.RawKey), "error", err) - return nil, err + return nil, fmt.Errorf("failed building eventual storage diffs for address: %s, error: %v", common.BytesToHash(val.RawKey), err) } accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ Key: val.RawKey, @@ -209,7 +235,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, er return nil, err } it := sTrie.NodeIterator(make([]byte, 0)) - storageDiffs := buildStorageDiffsFromTrie(it) + storageDiffs := sdb.buildStorageDiffsFromTrie(it) return storageDiffs, nil } @@ -229,35 +255,58 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common oldIt := oldTrie.NodeIterator(make([]byte, 0)) newIt := newTrie.NodeIterator(make([]byte, 0)) it, _ := trie.NewDifferenceIterator(oldIt, newIt) - storageDiffs := buildStorageDiffsFromTrie(it) + storageDiffs := sdb.buildStorageDiffsFromTrie(it) return storageDiffs, nil } -func buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { +func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { storageDiffs := make([]StorageDiff, 0) - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafProof := make([][]byte, len(it.LeafProof())) - copy(leafProof, it.LeafProof()) - leafPath := make([]byte, len(it.Path())) - copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: leafPath, - Proof: leafProof, - }) + if sdb.config.PathsAndProofs { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: leafPath, + Proof: leafProof, + }) + } + cont := it.Next(true) + if !cont { + break + } } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: nil, + Proof: nil, + }) + } + cont := it.Next(true) + if !cont { + break + } } } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 5ecf647e3..49bd39043 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -56,7 +56,10 @@ func TestBuilder(t *testing.T) { block1 = blockMap[block1Hash] block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] - builder = statediff.NewBuilder(testhelpers.Testdb, chain) + config := statediff.Config{ + PathsAndProofs: true, + } + builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) type arguments struct { oldStateRoot common.Hash diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000..a18f2c75a --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,23 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +// Config is used to carry in parameters from CLI configuration +type Config struct { + PathsAndProofs bool + LeafsOnly bool +} diff --git a/statediff/service.go b/statediff/service.go index d03a7b513..2b6b167f9 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -18,18 +18,17 @@ package statediff import ( "bytes" + "encoding/json" "fmt" "sync" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" ) @@ -66,25 +65,12 @@ type Service struct { Subscriptions map[rpc.ID]Subscription } -// Subscription struct holds our subscription channels -type Subscription struct { - PayloadChan chan<- Payload - QuitChan chan<- bool -} - -// Payload packages the data to send to StateDiffingService subscriptions -type Payload struct { - BlockRlp []byte `json:"blockRlp" gencodec:"required"` - StateDiffRlp []byte `json:"stateDiffRlp" gencodec:"required"` - Err error `json:"error"` -} - // NewStateDiffService creates a new StateDiffingService -func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*Service, error) { +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Config) (*Service, error) { return &Service{ Mutex: sync.Mutex{}, BlockChain: blockChain, - Builder: NewBuilder(db, blockChain), + Builder: NewBuilder(db, blockChain, config), QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]Subscription), }, nil @@ -109,70 +95,61 @@ func (sds *Service) APIs() []rpc.API { // Loop is the main processing method func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { - chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) defer chainEventSub.Unsubscribe() - - blocksCh := make(chan *types.Block, 10) errCh := chainEventSub.Err() - go func() { - HandleChainEventChLoop: - for { - select { - //Notify chain event channel of events - case chainEvent := <-chainEventCh: - log.Debug("Event received from chainEventCh", "event", chainEvent) - blocksCh <- chainEvent.Block - //if node stopped - case err := <-errCh: - log.Warn("Error from chain event subscription, breaking loop.", "error", err) - close(sds.QuitChan) - break HandleChainEventChLoop - case <-sds.QuitChan: - break HandleChainEventChLoop - } - } - }() - - //loop through chain events until no more -HandleBlockChLoop: for { select { - case block := <-blocksCh: - currentBlock := block + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + log.Debug("Event received from chainEventCh", "event", chainEvent) + currentBlock := chainEvent.Block parentHash := currentBlock.ParentHash() parentBlock := sds.BlockChain.GetBlockByHash(parentHash) if parentBlock == nil { log.Error("Parent block is nil, skipping this block", "parent block hash", parentHash.String(), "current block number", currentBlock.Number()) - break HandleBlockChLoop + continue } - - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) - if err != nil { + if err := sds.process(currentBlock, parentBlock); err != nil { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } - rlpBuff := new(bytes.Buffer) - currentBlock.EncodeRLP(rlpBuff) - blockRlp := rlpBuff.Bytes() - stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) - payload := Payload{ - BlockRlp: blockRlp, - StateDiffRlp: stateDiffRlp, - Err: err, - } - // If we have any websocket subscription listening in, send the data to them - sds.send(payload) + case err := <-errCh: + log.Warn("Error from chain event subscription, breaking loop.", "error", err) + sds.close() + return case <-sds.QuitChan: - log.Debug("Quitting the statediff block channel") + log.Info("Quitting the statediff block channel") sds.close() return } } } +// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions +func (sds *Service) process(currentBlock, parentBlock *types.Block) error { + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + if err != nil { + return err + } + + rlpBuff := new(bytes.Buffer) + currentBlock.EncodeRLP(rlpBuff) + blockRlp := rlpBuff.Bytes() + stateDiffBytes, _ := json.Marshal(stateDiff) + payload := Payload{ + BlockRlp: blockRlp, + StateDiff: stateDiffBytes, + Err: err, + } + + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + return nil +} + // Subscribe is used by the API to subscribe to the StateDiffingService loop func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index edb2d453d..65ab0f8c8 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -18,6 +18,7 @@ package mocks import ( "bytes" + "encoding/json" "errors" "fmt" "sync" @@ -26,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" ) @@ -63,7 +63,6 @@ func (sds *MockStateDiffService) APIs() []rpc.API { // Loop mock method func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { //loop through chain events until no more -HandleBlockChLoop: for { select { case block := <-sds.BlockChan: @@ -74,7 +73,7 @@ HandleBlockChLoop: log.Error("Parent block is nil, skipping this block", "parent block hash", parentHash.String(), "current block number", currentBlock.Number()) - break HandleBlockChLoop + continue } stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) @@ -84,11 +83,11 @@ HandleBlockChLoop: rlpBuff := new(bytes.Buffer) currentBlock.EncodeRLP(rlpBuff) blockRlp := rlpBuff.Bytes() - stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) + stateDiffBytes, _ := json.Marshal(stateDiff) payload := statediff.Payload{ - BlockRlp: blockRlp, - StateDiffRlp: stateDiffRlp, - Err: err, + BlockRlp: blockRlp, + StateDiff: stateDiffBytes, + Err: err, } // If we have any websocket subscription listening in, send the data to them sds.send(payload) diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index 1a4f83cf5..99420460a 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -18,6 +18,7 @@ package mocks import ( "bytes" + "encoding/json" "math/big" "sync" "testing" @@ -63,9 +64,12 @@ func TestAPI(t *testing.T) { blockChan := make(chan *types.Block) parentBlockChain := make(chan *types.Block) serviceQuitChan := make(chan bool) + config := statediff.Config{ + PathsAndProofs: true, + } mockService := MockStateDiffService{ Mutex: sync.Mutex{}, - Builder: statediff.NewBuilder(testhelpers.Testdb, chain), + Builder: statediff.NewBuilder(testhelpers.Testdb, chain, config), BlockChan: blockChan, ParentBlockChan: parentBlockChain, QuitChan: serviceQuitChan, @@ -79,7 +83,7 @@ func TestAPI(t *testing.T) { blockChan <- block1 parentBlockChain <- block0 expectedBlockRlp, _ := rlp.EncodeToBytes(block1) - expectedStateDiff := &statediff.StateDiff{ + expectedStateDiff := statediff.StateDiff{ BlockNumber: block1.Number().Int64(), BlockHash: block1.Hash(), CreatedAccounts: statediff.AccountDiffsMap{ @@ -112,14 +116,20 @@ func TestAPI(t *testing.T) { }, }, } - expectedStateDiffRlp, _ := rlp.EncodeToBytes(expectedStateDiff) + expectedStateDiffBytes, err := json.Marshal(expectedStateDiff) + if err != nil { + t.Error(err) + } select { case payload := <-payloadChan: if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp) } - if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffRlp) { - t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiffRlp, expectedStateDiffRlp) + if !bytes.Equal(payload.StateDiff, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiff, expectedStateDiffBytes) + } + if payload.Err != nil { + t.Errorf("payload should not contain an error, but does: %v", payload.Err) } case <-quitChan: t.Errorf("channel quit before delivering payload") diff --git a/statediff/types.go b/statediff/types.go index 816aae6d6..70f9d2e44 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -26,6 +26,19 @@ import ( "github.com/ethereum/go-ethereum/core/state" ) +// Subscription struct holds our subscription channels +type Subscription struct { + PayloadChan chan<- Payload + QuitChan chan<- bool +} + +// Payload packages the data to send to StateDiffingService subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp" gencodec:"required"` + StateDiff []byte `json:"stateDiff" gencodec:"required"` + Err error `json:"error"` +} + // AccountsMap is a mapping of keccak256(address) => accountWrapper type AccountsMap map[common.Hash]accountWrapper -- 2.45.2 From 63b89dd2be05a3e05f44dd8914bd8d7ee8e7d8ee Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 15 May 2019 13:06:57 -0500 Subject: [PATCH 06/14] option to process intermediate nodes --- cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 +- statediff/builder.go | 221 +++++++++++++++++++++++++++----------- statediff/builder_test.go | 1 + statediff/config.go | 1 + statediff/types.go | 2 +- 6 files changed, 166 insertions(+), 67 deletions(-) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index c30746b76..e92964aec 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -268,6 +268,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, utils.StateDiffLeafNodesOnly, + utils.StateDiffWatchedAddresses, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 48ccb434a..7627b5a46 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -747,12 +747,16 @@ var ( } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", - Usage: "Path and proof sets for the state and storage nodes are generated", + Usage: "Path and proof sets for the state and storage nodes are generated; only works with leaf nodes", } StateDiffLeafNodesOnly = cli.BoolFlag{ Name: "statediff.leafs", Usage: "Consider only leaf nodes of the storage and state tries", } + StateDiffWatchedAddresses = cli.StringSliceFlag{ + Name: "statediff.watchedaddresses", + Usage: "If provided, state diffing process is restricted to these addresses", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1611,6 +1615,7 @@ func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name), + WatchedAddress: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum diff --git a/statediff/builder.go b/statediff/builder.go index 04e00ae95..8c71322cf 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -21,6 +21,7 @@ package statediff import ( "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" @@ -40,6 +41,7 @@ type builder struct { chainDB ethdb.Database config Config blockChain *core.BlockChain + stateCache state.Database } // NewBuilder is used to create a state diff builder @@ -54,12 +56,12 @@ func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) B // BuildStateDiff builds a StateDiff object from two blocks func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states - stateCache := sdb.blockChain.StateCache() - oldTrie, err := stateCache.OpenTrie(oldStateRoot) + sdb.stateCache = sdb.blockChain.StateCache() + oldTrie, err := sdb.stateCache.OpenTrie(oldStateRoot) if err != nil { return StateDiff{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err) } - newTrie, err := stateCache.OpenTrie(newStateRoot) + newTrie, err := sdb.stateCache.OpenTrie(newStateRoot) if err != nil { return StateDiff{}, fmt.Errorf("error creating trie for newStateRoot: %v", err) } @@ -114,7 +116,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error if sdb.config.PathsAndProofs { for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) if it.Leaf() { leafProof := make([][]byte, len(it.LeafProof())) copy(leafProof, it.LeafProof()) @@ -128,10 +130,10 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error // lookup account state var account state.Account if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) } aw := accountWrapper{ - Account: account, + Account: &account, RawKey: leafKey, RawValue: leafValue, Proof: leafProof, @@ -147,33 +149,80 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error } } } else { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + if sdb.config.LeafsOnly { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: &account, + RawKey: leafKey, + RawValue: leafValue, + Proof: nil, + Path: nil, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw } - aw := accountWrapper{ - Account: account, - RawKey: leafKey, - RawValue: leafValue, - Proof: nil, - Path: nil, + cont := it.Next(true) + if !cont { + break } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: &account, + RawKey: leafKey, + RawValue: leafValue, + Proof: nil, + Path: nil, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw + } else { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err) + } + aw := accountWrapper{ + Account: nil, + RawKey: nodeKey.Bytes(), + RawValue: node, + Proof: nil, + Path: nil, + } + log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node) + diffAccounts[nodeKey] = aw + } + cont := it.Next(true) + if !cont { + break + } } } } @@ -183,10 +232,15 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { accountDiffs := make(AccountDiffsMap) + var err error for _, val := range accounts { - storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root) - if err != nil { - return nil, fmt.Errorf("failed building eventual storage diffs for address: %s, error: %v", common.BytesToHash(val.RawKey), err) + // If account is not nil, we need to process storage diffs + var storageDiffs []StorageDiff + if val.Account != nil { + storageDiffs, err = sdb.buildStorageDiffsEventual(val.Account.Root) + if err != nil { + return nil, fmt.Errorf("failed building eventual storage diffs for %s\r\nerror: %v", common.BytesToHash(val.RawKey), err) + } } accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ Key: val.RawKey, @@ -202,15 +256,19 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) { updatedAccounts := make(AccountDiffsMap) + var err error for _, val := range updatedKeys { - createdAcc := creations[common.HexToHash(val)] - deletedAcc := deletions[common.HexToHash(val)] - oldSR := deletedAcc.Account.Root - newSR := createdAcc.Account.Root - storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR) - if err != nil { - log.Error("Failed building storage diffs", "Address", val, "error", err) - return nil, err + hashKey := common.HexToHash(val) + createdAcc := creations[hashKey] + deletedAcc := deletions[hashKey] + var storageDiffs []StorageDiff + if deletedAcc.Account != nil && createdAcc.Account != nil { + oldSR := deletedAcc.Account.Root + newSR := createdAcc.Account.Root + storageDiffs, err = sdb.buildStorageDiffsIncremental(oldSR, newSR) + if err != nil { + return nil, fmt.Errorf("failed building incremental storage diffs for %s\r\nerror: %v", hashKey.Hex(), err) + } } updatedAccounts[common.HexToHash(val)] = AccountDiff{ Key: createdAcc.RawKey, @@ -235,8 +293,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, er return nil, err } it := sTrie.NodeIterator(make([]byte, 0)) - storageDiffs := sdb.buildStorageDiffsFromTrie(it) - return storageDiffs, nil + return sdb.buildStorageDiffsFromTrie(it) } func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) ([]StorageDiff, error) { @@ -255,12 +312,10 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common oldIt := oldTrie.NodeIterator(make([]byte, 0)) newIt := newTrie.NodeIterator(make([]byte, 0)) it, _ := trie.NewDifferenceIterator(oldIt, newIt) - storageDiffs := sdb.buildStorageDiffsFromTrie(it) - - return storageDiffs, nil + return sdb.buildStorageDiffsFromTrie(it) } -func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { +func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDiff, error) { storageDiffs := make([]StorageDiff, 0) if sdb.config.PathsAndProofs { for { @@ -288,29 +343,65 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDif } } } else { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: nil, - Proof: nil, - }) + if sdb.config.LeafsOnly { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: nil, + Proof: nil, + }) + } + cont := it.Next(true) + if !cont { + break + } } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: nil, + Proof: nil, + }) + } else { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err) + } + storageDiffs = append(storageDiffs, StorageDiff{ + Key: nodeKey.Bytes(), + Value: node, + Path: nil, + Proof: nil, + }) + log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node) + } + cont := it.Next(true) + if !cont { + break + } } } } - return storageDiffs + return storageDiffs, nil } func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 49bd39043..b7e3138d1 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -58,6 +58,7 @@ func TestBuilder(t *testing.T) { block3 = blockMap[block3Hash] config := statediff.Config{ PathsAndProofs: true, + LeafsOnly: true, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) diff --git a/statediff/config.go b/statediff/config.go index a18f2c75a..4a568982f 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -20,4 +20,5 @@ package statediff type Config struct { PathsAndProofs bool LeafsOnly bool + WatchedAddress []string } diff --git a/statediff/types.go b/statediff/types.go index 70f9d2e44..aa9460fde 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -44,7 +44,7 @@ type AccountsMap map[common.Hash]accountWrapper // AccountWrapper is used to temporary associate the unpacked account with its raw values type accountWrapper struct { - Account state.Account + Account *state.Account RawKey []byte RawValue []byte Proof [][]byte -- 2.45.2 From 371cef728278a3740a4bc6c7ea4ffce011acc744 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 16 May 2019 12:03:46 -0500 Subject: [PATCH 07/14] make state diff rlp serializable --- statediff/builder.go | 21 +-- statediff/builder_test.go | 155 +++++++++----------- statediff/service.go | 26 +++- statediff/testhelpers/mocks/builder.go | 6 +- statediff/testhelpers/mocks/service.go | 16 +- statediff/testhelpers/mocks/service_test.go | 41 +++--- statediff/testhelpers/test_data.go | 12 +- statediff/types.go | 50 ++----- 8 files changed, 155 insertions(+), 172 deletions(-) diff --git a/statediff/builder.go b/statediff/builder.go index 8c71322cf..26c4faaac 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -21,6 +21,7 @@ package statediff import ( "fmt" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -34,7 +35,7 @@ import ( // Builder interface exposes the method for building a state diff between two blocks type Builder interface { - BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) } type builder struct { @@ -54,7 +55,7 @@ func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) B } // BuildStateDiff builds a StateDiff object from two blocks -func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) { +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states sdb.stateCache = sdb.blockChain.StateCache() oldTrie, err := sdb.stateCache.OpenTrie(oldStateRoot) @@ -230,8 +231,8 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error return diffAccounts, nil } -func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { - accountDiffs := make(AccountDiffsMap) +func (sdb *builder) buildDiffEventual(accounts AccountsMap) ([]AccountDiff, error) { + accountDiffs := make([]AccountDiff, 0) var err error for _, val := range accounts { // If account is not nil, we need to process storage diffs @@ -242,20 +243,20 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er return nil, fmt.Errorf("failed building eventual storage diffs for %s\r\nerror: %v", common.BytesToHash(val.RawKey), err) } } - accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ + accountDiffs = append(accountDiffs, AccountDiff{ Key: val.RawKey, Value: val.RawValue, Proof: val.Proof, Path: val.Path, Storage: storageDiffs, - } + }) } return accountDiffs, nil } -func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) { - updatedAccounts := make(AccountDiffsMap) +func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) ([]AccountDiff, error) { + updatedAccounts := make([]AccountDiff, 0) var err error for _, val := range updatedKeys { hashKey := common.HexToHash(val) @@ -270,13 +271,13 @@ func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions Accoun return nil, fmt.Errorf("failed building incremental storage diffs for %s\r\nerror: %v", hashKey.Hex(), err) } } - updatedAccounts[common.HexToHash(val)] = AccountDiff{ + updatedAccounts = append(updatedAccounts, AccountDiff{ Key: createdAcc.RawKey, Value: createdAcc.RawValue, Proof: createdAcc.Proof, Path: createdAcc.Path, Storage: storageDiffs, - } + }) delete(creations, common.HexToHash(val)) delete(deletions, common.HexToHash(val)) } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index b7e3138d1..372241d10 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -19,7 +19,7 @@ package statediff_test import ( "bytes" "math/big" - "reflect" + "sort" "testing" "github.com/ethereum/go-ethereum/common" @@ -33,8 +33,8 @@ import ( var ( contractLeafKey common.Hash - emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) - emptyAccountDiffIncrementalMap = make(statediff.AccountDiffsMap) + emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) + emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0) block0Hash, block1Hash, block2Hash, block3Hash common.Hash block0, block1, block2, block3 *types.Block builder statediff.Builder @@ -65,7 +65,7 @@ func TestBuilder(t *testing.T) { type arguments struct { oldStateRoot common.Hash newStateRoot common.Hash - blockNumber int64 + blockNumber *big.Int blockHash common.Hash } @@ -166,11 +166,11 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block0.Root(), newStateRoot: block0.Root(), - blockNumber: block0.Number().Int64(), + blockNumber: block0.Number(), blockHash: block0Hash, }, &statediff.StateDiff{ - BlockNumber: block0.Number().Int64(), + BlockNumber: block0.Number(), BlockHash: block0Hash, CreatedAccounts: emptyAccountDiffEventualMap, DeletedAccounts: emptyAccountDiffEventualMap, @@ -183,22 +183,14 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block0.Root(), newStateRoot: block1.Root(), - blockNumber: block1.Number().Int64(), + blockNumber: block1.Number(), blockHash: block1Hash, }, &statediff.StateDiff{ - BlockNumber: block1.Number().Int64(), + BlockNumber: block1.Number(), BlockHash: block1.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account1LeafKey: { - Key: testhelpers.Account1LeafKey.Bytes(), - Value: account1, - Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, - {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, - Storage: []statediff.StorageDiff{}, - }, - burnLeafKey: { + CreatedAccounts: []statediff.AccountDiff{ + { Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -206,10 +198,18 @@ func TestBuilder(t *testing.T) { Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, + { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.BankLeafKey: { + UpdatedAccounts: []statediff.AccountDiff{ + { Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -227,22 +227,14 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block1.Root(), newStateRoot: block2.Root(), - blockNumber: block2.Number().Int64(), + blockNumber: block2.Number(), blockHash: block2Hash, }, &statediff.StateDiff{ - BlockNumber: block2.Number().Int64(), + BlockNumber: block2.Number(), BlockHash: block2.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account2LeafKey: { - Key: testhelpers.Account2LeafKey.Bytes(), - Value: account2, - Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, - Storage: []statediff.StorageDiff{}, - }, - contractLeafKey: { + CreatedAccounts: []statediff.AccountDiff{ + { Key: contractLeafKey.Bytes(), Value: contractAccount, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -257,10 +249,18 @@ func TestBuilder(t *testing.T) { }, }, }, + { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + Storage: []statediff.StorageDiff{}, + }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.BankLeafKey: { + UpdatedAccounts: []statediff.AccountDiff{ + { Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -268,15 +268,7 @@ func TestBuilder(t *testing.T) { Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, - testhelpers.Account1LeafKey: { - Key: testhelpers.Account1LeafKey.Bytes(), - Value: account3, - Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, - Storage: []statediff.StorageDiff{}, - }, - burnLeafKey: { + { Key: burnLeafKey.Bytes(), Value: burnAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -284,6 +276,14 @@ func TestBuilder(t *testing.T) { Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, + { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account3, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, }, }, }, @@ -294,24 +294,24 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block2.Root(), newStateRoot: block3.Root(), - blockNumber: block3.Number().Int64(), + blockNumber: block3.Number(), blockHash: block3.Hash(), }, &statediff.StateDiff{ - BlockNumber: block3.Number().Int64(), + BlockNumber: block3.Number(), BlockHash: block3.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{}, + CreatedAccounts: []statediff.AccountDiff{}, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account2LeafKey: { - Key: testhelpers.Account2LeafKey.Bytes(), - Value: account4, + UpdatedAccounts: []statediff.AccountDiff{ + { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount3, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, - contractLeafKey: { + { Key: contractLeafKey.Bytes(), Value: contractAccount2, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -327,12 +327,12 @@ func TestBuilder(t *testing.T) { }, }, }, - testhelpers.BankLeafKey: { - Key: testhelpers.BankLeafKey.Bytes(), - Value: bankAccount3, + { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account4, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + {248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, Storage: []statediff.StorageDiff{}, }, }, @@ -346,36 +346,23 @@ func TestBuilder(t *testing.T) { if err != nil { t.Error(err) } - fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} - - for _, field := range fields { - reflectionOfDiff := reflect.ValueOf(diff) - diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) - - reflectionOfExpected := reflect.ValueOf(test.expected) - expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) - - diffValueInterface := diffValue.Interface() - expectedValueInterface := expectedValue.Interface() - - if !equals(diffValueInterface, expectedValueInterface) { - t.Logf("Test failed: %s", test.name) - t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) - } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff rlp: %+v\nexpected state diff rlp: %+v", receivedStateDiffRlp, expectedStateDiffRlp) } } } -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} - /* contract test { diff --git a/statediff/service.go b/statediff/service.go index 2b6b167f9..897032781 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -18,10 +18,11 @@ package statediff import ( "bytes" - "encoding/json" "fmt" "sync" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -63,6 +64,8 @@ type Service struct { QuitChan chan bool // A mapping of rpc.IDs to their subscription channels Subscriptions map[rpc.ID]Subscription + // Cache the last block so that we can avoid having to lookup the next block's parent + lastBlock *types.Block } // NewStateDiffService creates a new StateDiffingService @@ -106,7 +109,13 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { log.Debug("Event received from chainEventCh", "event", chainEvent) currentBlock := chainEvent.Block parentHash := currentBlock.ParentHash() - parentBlock := sds.BlockChain.GetBlockByHash(parentHash) + var parentBlock *types.Block + if sds.lastBlock != nil && bytes.Equal(sds.lastBlock.Hash().Bytes(), currentBlock.ParentHash().Bytes()) { + parentBlock = sds.lastBlock + } else { + parentBlock = sds.BlockChain.GetBlockByHash(parentHash) + } + sds.lastBlock = currentBlock if parentBlock == nil { log.Error("Parent block is nil, skipping this block", "parent block hash", parentHash.String(), @@ -130,7 +139,7 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { // process method builds the state diff payload from the current and parent block and streams it to listening subscriptions func (sds *Service) process(currentBlock, parentBlock *types.Block) error { - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { return err } @@ -138,11 +147,14 @@ func (sds *Service) process(currentBlock, parentBlock *types.Block) error { rlpBuff := new(bytes.Buffer) currentBlock.EncodeRLP(rlpBuff) blockRlp := rlpBuff.Bytes() - stateDiffBytes, _ := json.Marshal(stateDiff) + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + return err + } payload := Payload{ - BlockRlp: blockRlp, - StateDiff: stateDiffBytes, - Err: err, + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, } // If we have any websocket subscription listening in, send the data to them diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index bb2e38f76..034af0415 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -17,6 +17,8 @@ package mocks import ( + "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff" ) @@ -25,14 +27,14 @@ import ( type Builder struct { OldStateRoot common.Hash NewStateRoot common.Hash - BlockNumber int64 + BlockNumber *big.Int BlockHash common.Hash stateDiff statediff.StateDiff builderError error } // BuildStateDiff mock method -func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (statediff.StateDiff, error) { +func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (statediff.StateDiff, error) { builder.OldStateRoot = oldStateRoot builder.NewStateRoot = newStateRoot builder.BlockNumber = blockNumber diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index 65ab0f8c8..aaa0d0202 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -18,11 +18,12 @@ package mocks import ( "bytes" - "encoding/json" "errors" "fmt" "sync" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -76,18 +77,21 @@ func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { continue } - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } rlpBuff := new(bytes.Buffer) currentBlock.EncodeRLP(rlpBuff) blockRlp := rlpBuff.Bytes() - stateDiffBytes, _ := json.Marshal(stateDiff) + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + log.Error("Error encoding statediff", "block number", currentBlock.Number(), "error", err) + } payload := statediff.Payload{ - BlockRlp: blockRlp, - StateDiff: stateDiffBytes, - Err: err, + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, } // If we have any websocket subscription listening in, send the data to them sds.send(payload) diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index 99420460a..97e4893c6 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -18,8 +18,8 @@ package mocks import ( "bytes" - "encoding/json" "math/big" + "sort" "sync" "testing" @@ -34,7 +34,7 @@ import ( var block0, block1 *types.Block var burnLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0")) -var emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) +var emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) var account1, _ = rlp.EncodeToBytes(state.Account{ Nonce: uint64(0), Balance: big.NewInt(10000), @@ -84,18 +84,10 @@ func TestAPI(t *testing.T) { parentBlockChain <- block0 expectedBlockRlp, _ := rlp.EncodeToBytes(block1) expectedStateDiff := statediff.StateDiff{ - BlockNumber: block1.Number().Int64(), + BlockNumber: block1.Number(), BlockHash: block1.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account1LeafKey: { - Key: testhelpers.Account1LeafKey.Bytes(), - Value: account1, - Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, - {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, - Storage: []statediff.StorageDiff{}, - }, - burnLeafKey: { + CreatedAccounts: []statediff.AccountDiff{ + { Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -103,10 +95,18 @@ func TestAPI(t *testing.T) { Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, + { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.BankLeafKey: { + UpdatedAccounts: []statediff.AccountDiff{ + { Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -116,17 +116,20 @@ func TestAPI(t *testing.T) { }, }, } - expectedStateDiffBytes, err := json.Marshal(expectedStateDiff) + expectedStateDiffBytes, err := rlp.EncodeToBytes(expectedStateDiff) if err != nil { t.Error(err) } + sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] }) + select { case payload := <-payloadChan: if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { - t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp) + t.Errorf("payload does not have expected block\r\actual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp) } - if !bytes.Equal(payload.StateDiff, expectedStateDiffBytes) { - t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiff, expectedStateDiffBytes) + sort.Slice(payload.StateDiffRlp, func(i, j int) bool { return payload.StateDiffRlp[i] < payload.StateDiffRlp[j] }) + if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\actual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateDiffRlp, expectedStateDiffBytes) } if payload.Err != nil { t.Errorf("payload should not contain an error, but does: %v", payload.Err) diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 1203be611..2f6088f86 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -36,7 +36,7 @@ func AddressToLeafKey(address common.Address) common.Hash { // Test variables var ( - BlockNumber = rand.Int63() + BlockNumber = big.NewInt(rand.Int63()) BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") NewNonceValue = rand.Uint64() @@ -63,26 +63,26 @@ var ( CodeHash: CodeHash, } valueBytes, _ = rlp.EncodeToBytes(testAccount) - CreatedAccountDiffs = statediff.AccountDiffsMap{ - ContractLeafKey: { + CreatedAccountDiffs = []statediff.AccountDiff{ + { Key: ContractLeafKey.Bytes(), Value: valueBytes, Storage: storage, }, - AnotherContractLeafKey: { + { Key: AnotherContractLeafKey.Bytes(), Value: valueBytes, Storage: emptyStorage, }, } - UpdatedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + UpdatedAccountDiffs = []statediff.AccountDiff{{ Key: ContractLeafKey.Bytes(), Value: valueBytes, Storage: storage, }} - DeletedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + DeletedAccountDiffs = []statediff.AccountDiff{{ Key: ContractLeafKey.Bytes(), Value: valueBytes, Storage: storage, diff --git a/statediff/types.go b/statediff/types.go index aa9460fde..3bcbed333 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -21,6 +21,7 @@ package statediff import ( "encoding/json" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" @@ -34,9 +35,9 @@ type Subscription struct { // Payload packages the data to send to StateDiffingService subscriptions type Payload struct { - BlockRlp []byte `json:"blockRlp" gencodec:"required"` - StateDiff []byte `json:"stateDiff" gencodec:"required"` - Err error `json:"error"` + BlockRlp []byte `json:"blockRlp" gencodec:"required"` + StateDiffRlp []byte `json:"stateDiff" gencodec:"required"` + Err error `json:"error"` } // AccountsMap is a mapping of keccak256(address) => accountWrapper @@ -53,11 +54,11 @@ type accountWrapper struct { // StateDiff is the final output structure from the builder type StateDiff struct { - BlockNumber int64 `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` - DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` - UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts []AccountDiff `json:"createdAccounts" gencodec:"required"` + DeletedAccounts []AccountDiff `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts []AccountDiff `json:"updatedAccounts" gencodec:"required"` encoded []byte err error @@ -81,46 +82,19 @@ func (sd *StateDiff) Encode() ([]byte, error) { return sd.encoded, sd.err } -// AccountDiffsMap is a mapping of keccak256(address) => AccountDiff -type AccountDiffsMap map[common.Hash]AccountDiff - -// AccountDiff holds the data for a single state diff leaf node +// AccountDiff holds the data for a single state diff node type AccountDiff struct { Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` - Storage []StorageDiff `json:"storage" gencodec:"required"` Path []byte `json:"path" gencodec:"required"` + Storage []StorageDiff `json:"storage" gencodec:"required"` } -// StorageDiff holds the data for a single storage diff leaf node +// StorageDiff holds the data for a single storage diff node type StorageDiff struct { Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` Path []byte `json:"path" gencodec:"required"` } - -/* -// State trie leaf is just a short node, below -// that has an rlp encoded account as the value - - -// SO each account diffs map is reall a map of shortnode keys to values -// Flatten to a slice of short nodes? - -// Need to coerce into: - -type TrieNode struct { - // leaf, extension or branch - nodeKind string - - // If leaf or extension: [0] is key, [1] is val. - // If branch: [0] - [16] are children. - elements []interface{} - - // IPLD block information - cid *cid.Cid - rawdata []byte -} -*/ -- 2.45.2 From 436f8e9f83ee3f1586cae2c494186347512a4dc0 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 16 May 2019 20:58:32 -0500 Subject: [PATCH 08/14] cli parameter to limit statediffing to select account addresses + test --- cmd/geth/main.go | 4 +- cmd/geth/usage.go | 3 +- cmd/utils/flags.go | 19 +- statediff/api.go | 9 +- statediff/builder.go | 270 +++++-------- statediff/builder_test.go | 399 ++++++++++++++------ statediff/config.go | 7 +- statediff/service.go | 17 +- statediff/testhelpers/mocks/service.go | 48 ++- statediff/testhelpers/mocks/service_test.go | 5 + statediff/types.go | 30 +- 11 files changed, 468 insertions(+), 343 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a0831c8f0..13f22a7f6 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -149,7 +149,9 @@ var ( utils.EVMInterpreterFlag, utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffLeafNodesOnly, + utils.StateDiffAllNodeTypes, + utils.StateDiffStreamBlock, + utils.StateDiffWatchedAddresses, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index e92964aec..d6664d5c9 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -267,8 +267,9 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffLeafNodesOnly, + utils.StateDiffAllNodeTypes, utils.StateDiffWatchedAddresses, + utils.StateDiffStreamBlock, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7627b5a46..aeb876432 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -747,16 +747,20 @@ var ( } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", - Usage: "Path and proof sets for the state and storage nodes are generated; only works with leaf nodes", + Usage: "Set to true to generate paths and proof sets for diffed state and storage trie lead nodes", } - StateDiffLeafNodesOnly = cli.BoolFlag{ - Name: "statediff.leafs", - Usage: "Consider only leaf nodes of the storage and state tries", + StateDiffAllNodeTypes = cli.BoolFlag{ + Name: "statediff.allnodes", + Usage: "Set to true to consider all node types: leaf, branch, and extension; default (false) processes leaf nodes only", } StateDiffWatchedAddresses = cli.StringSliceFlag{ Name: "statediff.watchedaddresses", Usage: "If provided, state diffing process is restricted to these addresses", } + StateDiffStreamBlock = cli.BoolFlag{ + Name: "statediff.streamblock", + Usage: "Set to true to stream the block data alongside state diff data", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1613,9 +1617,10 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st // RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ - PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), - LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name), - WatchedAddress: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), + StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), + PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), + AllNodes: ctx.GlobalBool(StateDiffAllNodeTypes.Name), + WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum diff --git a/statediff/api.go b/statediff/api.go index 26f03c9c3..498c2f759 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -18,8 +18,6 @@ package statediff import ( "context" - "sync" - "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -36,17 +34,12 @@ const APIVersion = "0.0.1" // are produced by a full node type PublicStateDiffAPI struct { sds IService - - mu sync.Mutex - lastUsed map[string]time.Time // keeps track when a filter was polled for the last time. } // NewPublicStateDiffAPI create a new state diff websocket streaming service. func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { return &PublicStateDiffAPI{ - sds: sds, - lastUsed: make(map[string]time.Time), - mu: sync.Mutex{}, + sds: sds, } } diff --git a/statediff/builder.go b/statediff/builder.go index 26c4faaac..7de9d8beb 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -20,6 +20,7 @@ package statediff import ( + "bytes" "fmt" "math/big" @@ -27,12 +28,15 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) +var nullNode = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") + // Builder interface exposes the method for building a state diff between two blocks type Builder interface { BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) @@ -111,120 +115,71 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block }, nil } +func (sdb *builder) isWatchedAddress(hashKey []byte) bool { + // If we aren't watching any addresses, we are watching everything + if len(sdb.config.WatchedAddresses) == 0 { + return true + } + for _, addrStr := range sdb.config.WatchedAddresses { + addr := common.HexToAddress(addrStr) + addrHashKey := crypto.Keccak256(addr[:]) + if bytes.Equal(addrHashKey, hashKey) { + return true + } + } + return false +} + func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error) { var diffAccounts = make(AccountsMap) it, _ := trie.NewDifferenceIterator(a, b) - - if sdb.config.PathsAndProofs { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() && sdb.isWatchedAddress(it.LeafKey()) { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Leaf: true, + Account: &account, + RawKey: leafKey, + RawValue: leafValue, + } + if sdb.config.PathsAndProofs { leafProof := make([][]byte, len(it.LeafProof())) copy(leafProof, it.LeafProof()) leafPath := make([]byte, len(it.Path())) copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) - } - aw := accountWrapper{ - Account: &account, - RawKey: leafKey, - RawValue: leafValue, - Proof: leafProof, - Path: leafPath, - } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw + aw.Proof = leafProof + aw.Path = leafPath } - cont := it.Next(true) - if !cont { - break + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw + } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err) } + aw := accountWrapper{ + Leaf: false, + RawKey: nodeKey.Bytes(), + RawValue: node, + } + log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node) + diffAccounts[nodeKey] = aw } - } else { - if sdb.config.LeafsOnly { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) - } - aw := accountWrapper{ - Account: &account, - RawKey: leafKey, - RawValue: leafValue, - Proof: nil, - Path: nil, - } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw - } - cont := it.Next(true) - if !cont { - break - } - } - } else { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) - } - aw := accountWrapper{ - Account: &account, - RawKey: leafKey, - RawValue: leafValue, - Proof: nil, - Path: nil, - } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw - } else { - nodeKey := it.Hash() - node, err := sdb.stateCache.TrieDB().Node(nodeKey) - if err != nil { - return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err) - } - aw := accountWrapper{ - Account: nil, - RawKey: nodeKey.Bytes(), - RawValue: node, - Proof: nil, - Path: nil, - } - log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node) - diffAccounts[nodeKey] = aw - } - cont := it.Next(true) - if !cont { - break - } - } + cont := it.Next(true) + if !cont { + break } } @@ -244,6 +199,7 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) ([]AccountDiff, erro } } accountDiffs = append(accountDiffs, AccountDiff{ + Leaf: val.Leaf, Key: val.RawKey, Value: val.RawValue, Proof: val.Proof, @@ -272,6 +228,7 @@ func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions Accoun } } updatedAccounts = append(updatedAccounts, AccountDiff{ + Leaf: createdAcc.Leaf, Key: createdAcc.RawKey, Value: createdAcc.RawValue, Proof: createdAcc.Proof, @@ -318,87 +275,44 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDiff, error) { storageDiffs := make([]StorageDiff, 0) - if sdb.config.PathsAndProofs { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + sd := StorageDiff{ + Leaf: true, + Key: leafKey, + Value: leafValue, + } + if sdb.config.PathsAndProofs { leafProof := make([][]byte, len(it.LeafProof())) copy(leafProof, it.LeafProof()) leafPath := make([]byte, len(it.Path())) copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: leafPath, - Proof: leafProof, - }) + sd.Proof = leafProof + sd.Path = leafPath } - cont := it.Next(true) - if !cont { - break + storageDiffs = append(storageDiffs, sd) + } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err) } + storageDiffs = append(storageDiffs, StorageDiff{ + Leaf: false, + Key: nodeKey.Bytes(), + Value: node, + }) + log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node) } - } else { - if sdb.config.LeafsOnly { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: nil, - Proof: nil, - }) - } - cont := it.Next(true) - if !cont { - break - } - } - } else { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: nil, - Proof: nil, - }) - } else { - nodeKey := it.Hash() - node, err := sdb.stateCache.TrieDB().Node(nodeKey) - if err != nil { - return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err) - } - storageDiffs = append(storageDiffs, StorageDiff{ - Key: nodeKey.Bytes(), - Value: node, - Path: nil, - Proof: nil, - }) - log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node) - } - cont := it.Next(true) - if !cont { - break - } - } + cont := it.Next(true) + if !cont { + break } } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 372241d10..7575b060a 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -32,130 +32,127 @@ import ( ) var ( - contractLeafKey common.Hash - emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) - emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0) - block0Hash, block1Hash, block2Hash, block3Hash common.Hash - block0, block1, block2, block3 *types.Block - builder statediff.Builder - miningReward = int64(2000000000000000000) - burnAddress = common.HexToAddress("0x0") - burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) + contractLeafKey common.Hash + emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) + emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0) + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(2000000000000000000) + burnAddress = common.HexToAddress("0x0") + burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) + + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block2Hash = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5") + block3Hash = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a") + balanceChange10000 = int64(10000) + balanceChange1000 = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + block2Account2Balance = int64(1000) + nonce0 = uint64(0) + nonce1 = uint64(1) + nonce2 = uint64(2) + nonce3 = uint64(3) + originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" + originalStorageLocation = common.HexToHash("0") + originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes() + updatedStorageLocation = common.HexToHash("2") + updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes() + originalStorageValue = common.Hex2Bytes("01") + updatedStorageValue = common.Hex2Bytes("03") + + account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + bankAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(contractContractRoot), + }) + bankAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1BankBalance - balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(block2Account2Balance + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(newContractRoot), + }) + bankAccount3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce3, + Balance: big.NewInt(99989000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) ) +type arguments struct { + oldStateRoot common.Hash + newStateRoot common.Hash + blockNumber *big.Int + blockHash common.Hash +} + func TestBuilder(t *testing.T) { _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) defer chain.Stop() - block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") - block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") - block2Hash = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5") - block3Hash = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a") - block0 = blockMap[block0Hash] block1 = blockMap[block1Hash] block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] config := statediff.Config{ PathsAndProofs: true, - LeafsOnly: true, + AllNodes: false, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) - type arguments struct { - oldStateRoot common.Hash - newStateRoot common.Hash - blockNumber *big.Int - blockHash common.Hash - } - - var ( - balanceChange10000 = int64(10000) - balanceChange1000 = int64(1000) - block1BankBalance = int64(99990000) - block1Account1Balance = int64(10000) - block2Account2Balance = int64(1000) - nonce0 = uint64(0) - nonce1 = uint64(1) - nonce2 = uint64(2) - nonce3 = uint64(3) - originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" - newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" - originalStorageLocation = common.HexToHash("0") - originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes() - updatedStorageLocation = common.HexToHash("2") - updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes() - originalStorageValue = common.Hex2Bytes("01") - updatedStorageValue = common.Hex2Bytes("03") - account1, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(balanceChange10000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - burnAccount1, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(miningReward), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - bankAccount1, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce1, - Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - account2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(balanceChange1000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - contractAccount, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce1, - Balance: big.NewInt(0), - CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), - Root: common.HexToHash(contractContractRoot), - }) - bankAccount2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce2, - Balance: big.NewInt(block1BankBalance - balanceChange1000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - account3, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce2, - Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - burnAccount2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(miningReward + miningReward), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - account4, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(block2Account2Balance + miningReward), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - contractAccount2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce1, - Balance: big.NewInt(0), - CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), - Root: common.HexToHash(newContractRoot), - }) - bankAccount3, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce3, - Balance: big.NewInt(99989000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - ) - var tests = []struct { name string startingArguments arguments @@ -191,6 +188,7 @@ func TestBuilder(t *testing.T) { BlockHash: block1.Hash(), CreatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -199,6 +197,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: testhelpers.Account1LeafKey.Bytes(), Value: account1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -210,6 +209,7 @@ func TestBuilder(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -235,6 +235,7 @@ func TestBuilder(t *testing.T) { BlockHash: block2.Hash(), CreatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: contractLeafKey.Bytes(), Value: contractAccount, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -242,6 +243,7 @@ func TestBuilder(t *testing.T) { Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, Storage: []statediff.StorageDiff{ { + Leaf: true, Key: originalStorageKey, Value: originalStorageValue, Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}}, @@ -250,6 +252,7 @@ func TestBuilder(t *testing.T) { }, }, { + Leaf: true, Key: testhelpers.Account2LeafKey.Bytes(), Value: account2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -261,6 +264,7 @@ func TestBuilder(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -269,6 +273,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: burnLeafKey.Bytes(), Value: burnAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -277,6 +282,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: testhelpers.Account1LeafKey.Bytes(), Value: account3, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -304,6 +310,7 @@ func TestBuilder(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount3, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -312,6 +319,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: contractLeafKey.Bytes(), Value: contractAccount2, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -319,6 +327,7 @@ func TestBuilder(t *testing.T) { Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, Storage: []statediff.StorageDiff{ { + Leaf: true, Key: updatedStorageKey, Value: updatedStorageValue, Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, @@ -328,6 +337,7 @@ func TestBuilder(t *testing.T) { }, }, { + Leaf: true, Key: testhelpers.Account2LeafKey.Bytes(), Value: account4, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -363,6 +373,177 @@ func TestBuilder(t *testing.T) { } } +func TestBuilderWithWatchedAddressList(t *testing.T) { + _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + block2 = blockMap[block2Hash] + block3 = blockMap[block3Hash] + config := statediff.Config{ + PathsAndProofs: true, + AllNodes: false, + WatchedAddresses: []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()}, + } + builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) + + var tests = []struct { + name string + startingArguments arguments + expected *statediff.StateDiff + }{ + { + "testEmptyDiff", + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block0.Root(), + blockNumber: block0.Number(), + blockHash: block0Hash, + }, + &statediff.StateDiff{ + BlockNumber: block0.Number(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block1.Root(), + blockNumber: block1.Number(), + blockHash: block1Hash, + }, + &statediff.StateDiff{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + CreatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: []statediff.AccountDiff{}, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + arguments{ + oldStateRoot: block1.Root(), + newStateRoot: block2.Root(), + blockNumber: block2.Number(), + blockHash: block2Hash, + }, + &statediff.StateDiff{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + CreatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: contractLeafKey.Bytes(), + Value: contractAccount, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 130, 30, 37, 86, 162, 144, 200, 100, 5, 248, 22, 10, 45, 102, 32, 66, 164, 49, 186, 69, 107, 157, 178, 101, 199, 155, 184, 55, 192, 75, 229, 240, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Leaf: true, + Key: originalStorageKey, + Value: originalStorageValue, + Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}}, + Path: []byte{2, 9, 0, 13, 14, 12, 13, 9, 5, 4, 8, 11, 6, 2, 10, 8, 13, 6, 0, 3, 4, 5, 10, 9, 8, 8, 3, 8, 6, 15, 12, 8, 4, 11, 10, 6, 11, 12, 9, 5, 4, 8, 4, 0, 0, 8, 15, 6, 3, 6, 2, 15, 9, 3, 1, 6, 0, 14, 15, 3, 14, 5, 6, 3, 16}, + }, + }, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account3, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + arguments{ + oldStateRoot: block2.Root(), + newStateRoot: block3.Root(), + blockNumber: block3.Number(), + blockHash: block3.Hash(), + }, + &statediff.StateDiff{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + CreatedAccounts: []statediff.AccountDiff{}, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: contractLeafKey.Bytes(), + Value: contractAccount2, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 113, 224, 209, 75, 43, 147, 229, 199, 249, 116, 142, 105, 225, 254, 95, 23, 73, 138, 28, 58, 195, 206, 194, 159, 150, 175, 19, 215, 248, 164, 224, 112, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Leaf: true, + Key: updatedStorageKey, + Value: updatedStorageValue, + Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, + {226, 160, 48, 87, 135, 250, 18, 168, 35, 224, 242, 183, 99, 28, 196, 27, 59, 168, 130, 139, 51, 33, 202, 129, 17, 17, 250, 117, 205, 58, 163, 187, 90, 206, 3}}, + Path: []byte{4, 0, 5, 7, 8, 7, 15, 10, 1, 2, 10, 8, 2, 3, 14, 0, 15, 2, 11, 7, 6, 3, 1, 12, 12, 4, 1, 11, 3, 11, 10, 8, 8, 2, 8, 11, 3, 3, 2, 1, 12, 10, 8, 1, 1, 1, 1, 1, 15, 10, 7, 5, 12, 13, 3, 10, 10, 3, 11, 11, 5, 10, 12, 14, 16}, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + arguments := test.startingArguments + diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff rlp: %+v\nexpected state diff rlp: %+v", receivedStateDiffRlp, expectedStateDiffRlp) + } + } +} + /* contract test { diff --git a/statediff/config.go b/statediff/config.go index 4a568982f..c246cfc81 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -18,7 +18,8 @@ package statediff // Config is used to carry in parameters from CLI configuration type Config struct { - PathsAndProofs bool - LeafsOnly bool - WatchedAddress []string + StreamBlock bool + PathsAndProofs bool + AllNodes bool + WatchedAddresses []string } diff --git a/statediff/service.go b/statediff/service.go index 897032781..5e3f3e59f 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -66,6 +66,8 @@ type Service struct { Subscriptions map[rpc.ID]Subscription // Cache the last block so that we can avoid having to lookup the next block's parent lastBlock *types.Block + // Whether or not the block data is streamed alongside the state diff data in the subscription payload + streamBlock bool } // NewStateDiffService creates a new StateDiffingService @@ -76,6 +78,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Builder: NewBuilder(db, blockChain, config), QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]Subscription), + streamBlock: config.StreamBlock, }, nil } @@ -143,21 +146,23 @@ func (sds *Service) process(currentBlock, parentBlock *types.Block) error { if err != nil { return err } - - rlpBuff := new(bytes.Buffer) - currentBlock.EncodeRLP(rlpBuff) - blockRlp := rlpBuff.Bytes() stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) if err != nil { return err } payload := Payload{ - BlockRlp: blockRlp, StateDiffRlp: stateDiffRlp, Err: err, } + if sds.streamBlock { + rlpBuff := new(bytes.Buffer) + if err = currentBlock.EncodeRLP(rlpBuff); err != nil { + return err + } + payload.BlockRlp = rlpBuff.Bytes() + } - // If we have any websocket subscription listening in, send the data to them + // If we have any websocket subscriptions listening in, send the data to them sds.send(payload) return nil } diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index aaa0d0202..687a7c77d 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -42,6 +42,7 @@ type MockStateDiffService struct { ParentBlockChan chan *types.Block QuitChan chan bool Subscriptions map[rpc.ID]statediff.Subscription + streamBlock bool } // Protocols mock method @@ -76,25 +77,10 @@ func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { "current block number", currentBlock.Number()) continue } - - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) - if err != nil { + if err := sds.process(currentBlock, parentBlock); err != nil { + println(err.Error()) log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } - rlpBuff := new(bytes.Buffer) - currentBlock.EncodeRLP(rlpBuff) - blockRlp := rlpBuff.Bytes() - stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) - if err != nil { - log.Error("Error encoding statediff", "block number", currentBlock.Number(), "error", err) - } - payload := statediff.Payload{ - BlockRlp: blockRlp, - StateDiffRlp: stateDiffRlp, - Err: err, - } - // If we have any websocket subscription listening in, send the data to them - sds.send(payload) case <-sds.QuitChan: log.Debug("Quitting the statediff block channel") sds.close() @@ -103,6 +89,34 @@ func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { } } +// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions +func (sds *MockStateDiffService) process(currentBlock, parentBlock *types.Block) error { + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) + if err != nil { + return err + } + + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + return err + } + payload := statediff.Payload{ + StateDiffRlp: stateDiffRlp, + Err: err, + } + if sds.streamBlock { + rlpBuff := new(bytes.Buffer) + if err = currentBlock.EncodeRLP(rlpBuff); err != nil { + return err + } + payload.BlockRlp = rlpBuff.Bytes() + } + + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + return nil +} + // Subscribe mock method func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index 97e4893c6..4b4ac95a5 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -66,6 +66,7 @@ func TestAPI(t *testing.T) { serviceQuitChan := make(chan bool) config := statediff.Config{ PathsAndProofs: true, + AllNodes: false, } mockService := MockStateDiffService{ Mutex: sync.Mutex{}, @@ -74,6 +75,7 @@ func TestAPI(t *testing.T) { ParentBlockChan: parentBlockChain, QuitChan: serviceQuitChan, Subscriptions: make(map[rpc.ID]statediff.Subscription), + streamBlock: true, } mockService.Start(nil) id := rpc.NewID() @@ -88,6 +90,7 @@ func TestAPI(t *testing.T) { BlockHash: block1.Hash(), CreatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -96,6 +99,7 @@ func TestAPI(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: testhelpers.Account1LeafKey.Bytes(), Value: account1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -107,6 +111,7 @@ func TestAPI(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, diff --git a/statediff/types.go b/statediff/types.go index 3bcbed333..6df398a10 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -23,8 +23,9 @@ import ( "encoding/json" "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" + + "github.com/ethereum/go-ethereum/common" ) // Subscription struct holds our subscription channels @@ -40,18 +41,6 @@ type Payload struct { Err error `json:"error"` } -// AccountsMap is a mapping of keccak256(address) => accountWrapper -type AccountsMap map[common.Hash]accountWrapper - -// AccountWrapper is used to temporary associate the unpacked account with its raw values -type accountWrapper struct { - Account *state.Account - RawKey []byte - RawValue []byte - Proof [][]byte - Path []byte -} - // StateDiff is the final output structure from the builder type StateDiff struct { BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` @@ -84,6 +73,7 @@ func (sd *StateDiff) Encode() ([]byte, error) { // AccountDiff holds the data for a single state diff node type AccountDiff struct { + Leaf bool `json:"leaf" gencodec:"required"` Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` @@ -93,8 +83,22 @@ type AccountDiff struct { // StorageDiff holds the data for a single storage diff node type StorageDiff struct { + Leaf bool `json:"leaf" gencodec:"required"` Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` Path []byte `json:"path" gencodec:"required"` } + +// AccountsMap is a mapping of keccak256(address) => accountWrapper +type AccountsMap map[common.Hash]accountWrapper + +// AccountWrapper is used to temporary associate the unpacked account with its raw values +type accountWrapper struct { + Account *state.Account + Leaf bool + RawKey []byte + RawValue []byte + Proof [][]byte + Path []byte +} -- 2.45.2 From ba8156f63f4a1a20b1f7eb1aac9b587a02d6ce6f Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 17 May 2019 10:20:28 -0500 Subject: [PATCH 09/14] review fixes and fixes for issues ran into in integration --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 12 ++++++------ statediff/api.go | 14 ++++++++------ statediff/service.go | 12 ++++++------ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 13f22a7f6..9423648c2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -149,7 +149,7 @@ var ( utils.EVMInterpreterFlag, utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffAllNodeTypes, + utils.StateDiffIntermediateNodes, utils.StateDiffStreamBlock, utils.StateDiffWatchedAddresses, configFileFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index d6664d5c9..49bb0d32d 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -267,7 +267,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffAllNodeTypes, + utils.StateDiffIntermediateNodes, utils.StateDiffWatchedAddresses, utils.StateDiffStreamBlock, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index aeb876432..98bd3628c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -747,11 +747,11 @@ var ( } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", - Usage: "Set to true to generate paths and proof sets for diffed state and storage trie lead nodes", + Usage: "Set to true to generate paths and proof sets for diffed state and storage trie leaf nodes", } - StateDiffAllNodeTypes = cli.BoolFlag{ - Name: "statediff.allnodes", - Usage: "Set to true to consider all node types: leaf, branch, and extension; default (false) processes leaf nodes only", + StateDiffIntermediateNodes = cli.BoolFlag{ + Name: "statediff.intermediatenodes", + Usage: "Set to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only", } StateDiffWatchedAddresses = cli.StringSliceFlag{ Name: "statediff.watchedaddresses", @@ -759,7 +759,7 @@ var ( } StateDiffStreamBlock = cli.BoolFlag{ Name: "statediff.streamblock", - Usage: "Set to true to stream the block data alongside state diff data", + Usage: "Set to true to stream the block data alongside state diff data in the same subscription payload", } ) @@ -1619,7 +1619,7 @@ func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), - AllNodes: ctx.GlobalBool(StateDiffAllNodeTypes.Name), + AllNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name), WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { diff --git a/statediff/api.go b/statediff/api.go index 498c2f759..75f590daf 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -44,7 +44,7 @@ func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { } // Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created -func (api *PublicStateDiffAPI) Subscribe(ctx context.Context) (*rpc.Subscription, error) { +func (api *PublicStateDiffAPI) Subscribe(ctx context.Context, payloadChan chan Payload) (*rpc.Subscription, error) { // ensure that the RPC connection supports subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -56,19 +56,21 @@ func (api *PublicStateDiffAPI) Subscribe(ctx context.Context) (*rpc.Subscription go func() { // subscribe to events from the state diff service - payloadChannel := make(chan Payload) + payloadChannel := make(chan Payload, 10) quitChan := make(chan bool) api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) - - // loop and await state diff payloads and relay them to the subscriber with then notifier + // loop and await state diff payloads and relay them to the subscriber with the notifier for { select { case packet := <-payloadChannel: if err := notifier.Notify(rpcSub.ID, packet); err != nil { log.Error("Failed to send state diff packet", "err", err) } - case <-rpcSub.Err(): - err := api.sds.Unsubscribe(rpcSub.ID) + case err := <-rpcSub.Err(): + log.Error("State diff service rpcSub error", err) + println("err") + println(err.Error()) + err = api.sds.Unsubscribe(rpcSub.ID) if err != nil { log.Error("Failed to unsubscribe from the state diff service", err) } diff --git a/statediff/service.go b/statediff/service.go index 5e3f3e59f..49cebef5f 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -129,11 +129,11 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } case err := <-errCh: - log.Warn("Error from chain event subscription, breaking loop.", "error", err) + log.Warn("Error from chain event subscription, breaking loop", "error", err) sds.close() return case <-sds.QuitChan: - log.Info("Quitting the statediff block channel") + log.Info("Quitting the statediffing process") sds.close() return } @@ -214,9 +214,9 @@ func (sds *Service) send(payload Payload) { for id, sub := range sds.Subscriptions { select { case sub.PayloadChan <- payload: - log.Info("sending state diff payload to subscription %s", id) + log.Info(fmt.Sprintf("sending state diff payload to subscription %s", id)) default: - log.Info("unable to send payload to subscription %s; channel has no receiver", id) + log.Info(fmt.Sprintf("unable to send payload to subscription %s", id)) } } sds.Unlock() @@ -229,9 +229,9 @@ func (sds *Service) close() { select { case sub.QuitChan <- true: delete(sds.Subscriptions, id) - log.Info("closing subscription %s", id) + log.Info(fmt.Sprintf("closing subscription %s", id)) default: - log.Info("unable to close subscription %s; channel has no receiver", id) + log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) } } sds.Unlock() -- 2.45.2 From 073c340a51d215c07313a21d444eaf6be7bf0107 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 5 Jun 2019 13:10:04 -0500 Subject: [PATCH 10/14] review fixes; proper method signature for api; adjust service so that statediff processing is halted/paused until there is at least one subscriber listening for the results --- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 16 +++---- statediff/api.go | 6 +-- statediff/builder.go | 4 +- statediff/builder_test.go | 10 ++--- statediff/config.go | 8 ++-- statediff/service.go | 45 +++++++++++++++---- statediff/service_test.go | 13 +++++- .../testhelpers/mocks/{service.go => api.go} | 0 .../mocks/{service_test.go => api_test.go} | 4 +- statediff/testhelpers/mocks/publisher.go | 36 --------------- 11 files changed, 73 insertions(+), 71 deletions(-) rename statediff/testhelpers/mocks/{service.go => api.go} (100%) rename statediff/testhelpers/mocks/{service_test.go => api_test.go} (99%) delete mode 100644 statediff/testhelpers/mocks/publisher.go diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 49bb0d32d..d667120e7 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -268,8 +268,8 @@ var AppHelpFlagGroups = []flagGroup{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, utils.StateDiffIntermediateNodes, - utils.StateDiffWatchedAddresses, utils.StateDiffStreamBlock, + utils.StateDiffWatchedAddresses, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 98bd3628c..d216915ad 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -753,14 +753,14 @@ var ( Name: "statediff.intermediatenodes", Usage: "Set to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only", } - StateDiffWatchedAddresses = cli.StringSliceFlag{ - Name: "statediff.watchedaddresses", - Usage: "If provided, state diffing process is restricted to these addresses", - } StateDiffStreamBlock = cli.BoolFlag{ Name: "statediff.streamblock", Usage: "Set to true to stream the block data alongside state diff data in the same subscription payload", } + StateDiffWatchedAddresses = cli.StringSliceFlag{ + Name: "statediff.watchedaddresses", + Usage: "If provided, state diffing process is restricted to these addresses", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1617,10 +1617,10 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st // RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ - StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), - PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), - AllNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name), - WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), + PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), + IntermediateNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name), + StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), + WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum diff --git a/statediff/api.go b/statediff/api.go index 75f590daf..946be1146 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -43,8 +43,8 @@ func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { } } -// Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created -func (api *PublicStateDiffAPI) Subscribe(ctx context.Context, payloadChan chan Payload) (*rpc.Subscription, error) { +// Stream is the public method to setup a subscription that fires off state-diff payloads as they are created +func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, error) { // ensure that the RPC connection supports subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -68,8 +68,6 @@ func (api *PublicStateDiffAPI) Subscribe(ctx context.Context, payloadChan chan P } case err := <-rpcSub.Err(): log.Error("State diff service rpcSub error", err) - println("err") - println(err.Error()) err = api.sds.Unsubscribe(rpcSub.ID) if err != nil { log.Error("Failed to unsubscribe from the state diff service", err) diff --git a/statediff/builder.go b/statediff/builder.go index 7de9d8beb..765152e87 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -163,7 +163,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error // record account to diffs (creation if we are looking at new - old; deletion if old - new) log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) diffAccounts[leafKeyHash] = aw - } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + } else if sdb.config.IntermediateNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { nodeKey := it.Hash() node, err := sdb.stateCache.TrieDB().Node(nodeKey) if err != nil { @@ -297,7 +297,7 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDi sd.Path = leafPath } storageDiffs = append(storageDiffs, sd) - } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + } else if sdb.config.IntermediateNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { nodeKey := it.Hash() node, err := sdb.stateCache.TrieDB().Node(nodeKey) if err != nil { diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 7575b060a..2c9253de1 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -148,8 +148,8 @@ func TestBuilder(t *testing.T) { block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] config := statediff.Config{ - PathsAndProofs: true, - AllNodes: false, + PathsAndProofs: true, + IntermediateNodes: false, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) @@ -382,9 +382,9 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] config := statediff.Config{ - PathsAndProofs: true, - AllNodes: false, - WatchedAddresses: []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()}, + PathsAndProofs: true, + IntermediateNodes: false, + WatchedAddresses: []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()}, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) diff --git a/statediff/config.go b/statediff/config.go index c246cfc81..70f09a749 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -18,8 +18,8 @@ package statediff // Config is used to carry in parameters from CLI configuration type Config struct { - StreamBlock bool - PathsAndProofs bool - AllNodes bool - WatchedAddresses []string + PathsAndProofs bool + IntermediateNodes bool + StreamBlock bool + WatchedAddresses []string } diff --git a/statediff/service.go b/statediff/service.go index 49cebef5f..f3e47d16e 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -20,8 +20,7 @@ import ( "bytes" "fmt" "sync" - - "github.com/ethereum/go-ethereum/rlp" + "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -68,6 +68,8 @@ type Service struct { lastBlock *types.Block // Whether or not the block data is streamed alongside the state diff data in the subscription payload streamBlock bool + // Whether or not we have any subscribers; only if we do, do we processes state diffs + subscribers int32 } // NewStateDiffService creates a new StateDiffingService @@ -110,6 +112,11 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { //Notify chain event channel of events case chainEvent := <-chainEventCh: log.Debug("Event received from chainEventCh", "event", chainEvent) + // if we don't have any subscribers, do not process a statediff + if atomic.LoadInt32(&sds.subscribers) == 0 { + log.Debug("Currently no subscribers to the statediffing service; processing is halted") + continue + } currentBlock := chainEvent.Block parentHash := currentBlock.ParentHash() var parentBlock *types.Block @@ -125,7 +132,7 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { "current block number", currentBlock.Number()) continue } - if err := sds.process(currentBlock, parentBlock); err != nil { + if err := sds.processStateDiff(currentBlock, parentBlock); err != nil { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } case err := <-errCh: @@ -140,8 +147,8 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { } } -// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions -func (sds *Service) process(currentBlock, parentBlock *types.Block) error { +// processStateDiff method builds the state diff payload from the current and parent block and sends it to listening subscriptions +func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) error { stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { return err @@ -170,6 +177,9 @@ func (sds *Service) process(currentBlock, parentBlock *types.Block) error { // Subscribe is used by the API to subscribe to the StateDiffingService loop func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") + if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) { + log.Info("State diffing subscription received; beginning statediff processing") + } sds.Lock() sds.Subscriptions[id] = Subscription{ PayloadChan: sub, @@ -187,6 +197,11 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id) } delete(sds.Subscriptions, id) + if len(sds.Subscriptions) == 0 { + if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) { + log.Info("No more subscriptions; halting statediff processing") + } + } sds.Unlock() return nil } @@ -208,7 +223,7 @@ func (sds *Service) Stop() error { return nil } -// send is used to fan out and serve a payload to any subscriptions +// send is used to fan out and serve the statediff payload to all subscriptions func (sds *Service) send(payload Payload) { sds.Lock() for id, sub := range sds.Subscriptions { @@ -216,7 +231,21 @@ func (sds *Service) send(payload Payload) { case sub.PayloadChan <- payload: log.Info(fmt.Sprintf("sending state diff payload to subscription %s", id)) default: - log.Info(fmt.Sprintf("unable to send payload to subscription %s", id)) + log.Info(fmt.Sprintf("unable to send payload to subscription %s; channel has no receiver", id)) + // in this case, try to close the bad subscription and remove it + select { + case sub.QuitChan <- true: + log.Info(fmt.Sprintf("closing subscription %s", id)) + default: + log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) + } + delete(sds.Subscriptions, id) + } + } + // If after removing all bad subscriptions we have none left, halt processing + if len(sds.Subscriptions) == 0 { + if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) { + log.Info("No more subscriptions; halting statediff processing") } } sds.Unlock() @@ -228,11 +257,11 @@ func (sds *Service) close() { for id, sub := range sds.Subscriptions { select { case sub.QuitChan <- true: - delete(sds.Subscriptions, id) log.Info(fmt.Sprintf("closing subscription %s", id)) default: log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) } + delete(sds.Subscriptions, id) } sds.Unlock() } diff --git a/statediff/service_test.go b/statediff/service_test.go index d5edee04e..de3df3dd2 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -33,7 +33,7 @@ import ( func TestServiceLoop(t *testing.T) { testErrorInChainEventLoop(t) - testErrorInBlockLoop(t) + //testErrorInBlockLoop(t) } var ( @@ -76,10 +76,21 @@ func testErrorInChainEventLoop(t *testing.T) { QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]statediff.Subscription), } + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan) testRoot2 = common.HexToHash("0xTestRoot2") blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) + // Need to have listeners on the channels or the subscription will be closed and the processing halted + go func() { + select { + case <-payloadChan: + case <-quitChan: + } + }() service.Loop(eventsChannel) + if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { t.Error("Test failure:", t.Name()) t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/api.go similarity index 100% rename from statediff/testhelpers/mocks/service.go rename to statediff/testhelpers/mocks/api.go diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/api_test.go similarity index 99% rename from statediff/testhelpers/mocks/service_test.go rename to statediff/testhelpers/mocks/api_test.go index 4b4ac95a5..22971b4b6 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/api_test.go @@ -65,8 +65,8 @@ func TestAPI(t *testing.T) { parentBlockChain := make(chan *types.Block) serviceQuitChan := make(chan bool) config := statediff.Config{ - PathsAndProofs: true, - AllNodes: false, + PathsAndProofs: true, + IntermediateNodes: false, } mockService := MockStateDiffService{ Mutex: sync.Mutex{}, diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go deleted file mode 100644 index 6a6018746..000000000 --- a/statediff/testhelpers/mocks/publisher.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mocks - -import "github.com/ethereum/go-ethereum/statediff" - -// Publisher mock -type Publisher struct { - StateDiff *statediff.StateDiff - publisherError error -} - -// PublishStateDiff mock method -func (publisher *Publisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { - publisher.StateDiff = sd - return "", publisher.publisherError -} - -// SetPublisherError mock method -func (publisher *Publisher) SetPublisherError(err error) { - publisher.publisherError = err -} -- 2.45.2 From 9f5ff0fc0ecc2e6db904c76d3a74560b9f418b0a Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 6 Jun 2019 21:24:17 -0500 Subject: [PATCH 11/14] adjust buffering to improve stability; begin documentation; fix notifier err handling --- core/blockchain_test.go | 4 ++-- statediff/api.go | 23 +++++++++++++++-------- statediff/doc.go | 40 ++++++++++++++++++++++++++++++++++++++++ statediff/service.go | 10 +++++----- 4 files changed, 62 insertions(+), 15 deletions(-) create mode 100644 statediff/doc.go diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b6a44fe91..cd0fedb82 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2227,8 +2227,8 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) { func BenchmarkBlockChain_1x1000Executions(b *testing.B) { var ( - numTxs= 1000 - numBlocks= 1 + numTxs = 1000 + numBlocks = 1 ) b.StopTimer() b.ResetTimer() diff --git a/statediff/api.go b/statediff/api.go index 946be1146..a05ef5510 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -56,23 +56,30 @@ func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, e go func() { // subscribe to events from the state diff service - payloadChannel := make(chan Payload, 10) - quitChan := make(chan bool) + payloadChannel := make(chan Payload, chainEventChanSize) + quitChan := make(chan bool, 1) api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) // loop and await state diff payloads and relay them to the subscriber with the notifier for { select { case packet := <-payloadChannel: - if err := notifier.Notify(rpcSub.ID, packet); err != nil { - log.Error("Failed to send state diff packet", "err", err) + if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil { + log.Error("Failed to send state diff packet; error: " + notifyErr.Error()) + unSubErr := api.sds.Unsubscribe(rpcSub.ID) + if unSubErr != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + unSubErr.Error()) + } + return } case err := <-rpcSub.Err(): - log.Error("State diff service rpcSub error", err) - err = api.sds.Unsubscribe(rpcSub.ID) if err != nil { - log.Error("Failed to unsubscribe from the state diff service", err) + log.Error("State diff service rpcSub error: " + err.Error()) + err = api.sds.Unsubscribe(rpcSub.ID) + if err != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error()) + } + return } - return case <-quitChan: // don't need to unsubscribe, statediff service does so before sending the quit signal return diff --git a/statediff/doc.go b/statediff/doc.go new file mode 100644 index 000000000..c0cbb4353 --- /dev/null +++ b/statediff/doc.go @@ -0,0 +1,40 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go + +Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, +relaying the objects to any rpc subscriptions. + +Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, +with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". + +e.g. + +stateDiffPayloadChan := make(chan statediff.Payload, 20000) +rpcSub, err := Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"}) +for { + select { + case stateDiffPayload := <- stateDiffPayloadChan: + processPayload(stateDiffPayload) + case err := <= rpcSub.Err(): + log.Error(err) + } +} + +*/ +package statediff diff --git a/statediff/service.go b/statediff/service.go index f3e47d16e..e978fcb8f 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -34,6 +34,8 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) +const chainEventChanSize = 20000 + type blockChain interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription GetBlockByHash(hash common.Hash) *types.Block @@ -127,13 +129,11 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { } sds.lastBlock = currentBlock if parentBlock == nil { - log.Error("Parent block is nil, skipping this block", - "parent block hash", parentHash.String(), - "current block number", currentBlock.Number()) + log.Error(fmt.Sprintf("Parent block is nil, skipping this block (%d)", currentBlock.Number())) continue } if err := sds.processStateDiff(currentBlock, parentBlock); err != nil { - log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) + log.Error(fmt.Sprintf("Error building statediff for block %d; error: ", currentBlock.Number()) + err.Error()) } case err := <-errCh: log.Warn("Error from chain event subscription, breaking loop", "error", err) @@ -210,7 +210,7 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { func (sds *Service) Start(*p2p.Server) error { log.Info("Starting statediff service") - chainEventCh := make(chan core.ChainEvent, 10) + chainEventCh := make(chan core.ChainEvent, chainEventChanSize) go sds.Loop(chainEventCh) return nil -- 2.45.2 From 3dcb934614358e2ca362292ef93fee3df1cfecb8 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 11 Jul 2019 12:51:11 -0500 Subject: [PATCH 12/14] work on doc.go --- cmd/utils/flags.go | 2 +- statediff/doc.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d216915ad..dbe008646 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -743,7 +743,7 @@ var ( StateDiffFlag = cli.BoolFlag{ Name: "statediff", - Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", + Usage: "Enables the processing of state diffs between each block", } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", diff --git a/statediff/doc.go b/statediff/doc.go index c0cbb4353..0e6d5f3e1 100644 --- a/statediff/doc.go +++ b/statediff/doc.go @@ -20,13 +20,23 @@ This work is adapted from work by Charles Crain at https://github.com/jpmorganch Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, relaying the objects to any rpc subscriptions. +The service is spun up using the below CLI flags +--statediff: boolean flag, turns on the service +--statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs. +--statediff.intermediatenodes: boolean flag, tells service to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only. +--statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes. +--statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3 + +If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. + Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". e.g. +cli, _ := rpc.Dial("ipcPathOrWsURL") stateDiffPayloadChan := make(chan statediff.Payload, 20000) -rpcSub, err := Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"}) +rpcSub, err := cli.Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"}) for { select { case stateDiffPayload := <- stateDiffPayloadChan: @@ -35,6 +45,5 @@ for { log.Error(err) } } - */ package statediff -- 2.45.2 From 86949cc1c85567d7041afcd88c76445d53c52af5 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 18 Jul 2019 13:55:37 -0500 Subject: [PATCH 13/14] relay receipts with the rest of the data (don't know why I didn't realize to do this earlier) --- statediff/service.go | 21 ++++--- statediff/service_test.go | 73 ++++++++++++++++++++--- statediff/testhelpers/mocks/blockchain.go | 24 ++++++-- statediff/types.go | 7 ++- 4 files changed, 102 insertions(+), 23 deletions(-) diff --git a/statediff/service.go b/statediff/service.go index e978fcb8f..be3dc6cf9 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -40,6 +40,7 @@ type blockChain interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription GetBlockByHash(hash common.Hash) *types.Block AddToStateDiffProcessedCollection(hash common.Hash) + GetReceiptsByHash(hash common.Hash) types.Receipts } // IService is the state-diffing service interface @@ -69,7 +70,7 @@ type Service struct { // Cache the last block so that we can avoid having to lookup the next block's parent lastBlock *types.Block // Whether or not the block data is streamed alongside the state diff data in the subscription payload - streamBlock bool + StreamBlock bool // Whether or not we have any subscribers; only if we do, do we processes state diffs subscribers int32 } @@ -82,7 +83,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Builder: NewBuilder(db, blockChain, config), QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]Subscription), - streamBlock: config.StreamBlock, + StreamBlock: config.StreamBlock, }, nil } @@ -108,7 +109,6 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) defer chainEventSub.Unsubscribe() errCh := chainEventSub.Err() - for { select { //Notify chain event channel of events @@ -161,12 +161,19 @@ func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) err StateDiffRlp: stateDiffRlp, Err: err, } - if sds.streamBlock { - rlpBuff := new(bytes.Buffer) - if err = currentBlock.EncodeRLP(rlpBuff); err != nil { + if sds.StreamBlock { + blockBuff := new(bytes.Buffer) + if err = currentBlock.EncodeRLP(blockBuff); err != nil { return err } - payload.BlockRlp = rlpBuff.Bytes() + payload.BlockRlp = blockBuff.Bytes() + receiptBuff := new(bytes.Buffer) + receipts := sds.BlockChain.GetReceiptsByHash(currentBlock.Hash()) + if err = rlp.Encode(receiptBuff, receipts); err != nil { + println(err.Error()) + return err + } + payload.ReceiptsRlp = receiptBuff.Bytes() } // If we have any websocket subscriptions listening in, send the data to them diff --git a/statediff/service_test.go b/statediff/service_test.go index de3df3dd2..1508542b7 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -21,11 +21,13 @@ import ( "math/big" "math/rand" "reflect" + "sync" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" @@ -33,7 +35,7 @@ import ( func TestServiceLoop(t *testing.T) { testErrorInChainEventLoop(t) - //testErrorInBlockLoop(t) + testErrorInBlockLoop(t) } var ( @@ -61,6 +63,12 @@ var ( testBlock2 = types.NewBlock(&header2, nil, nil, nil) testBlock3 = types.NewBlock(&header3, nil, nil, nil) + receiptRoot1 = common.HexToHash("0x05") + receiptRoot2 = common.HexToHash("0x06") + receiptRoot3 = common.HexToHash("0x07") + testReceipts1 = []*types.Receipt{types.NewReceipt(receiptRoot1.Bytes(), false, 1000), types.NewReceipt(receiptRoot2.Bytes(), false, 2000)} + testReceipts2 = []*types.Receipt{types.NewReceipt(receiptRoot3.Bytes(), false, 3000)} + event1 = core.ChainEvent{Block: testBlock1} event2 = core.ChainEvent{Block: testBlock2} event3 = core.ChainEvent{Block: testBlock3} @@ -71,25 +79,61 @@ func testErrorInChainEventLoop(t *testing.T) { builder := mocks.Builder{} blockChain := mocks.BlockChain{} service := statediff.Service{ + Mutex: sync.Mutex{}, Builder: &builder, BlockChain: &blockChain, QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]statediff.Subscription), + StreamBlock: true, } - payloadChan := make(chan statediff.Payload) + payloadChan := make(chan statediff.Payload, 2) quitChan := make(chan bool) service.Subscribe(rpc.NewID(), payloadChan, quitChan) testRoot2 = common.HexToHash("0xTestRoot2") - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockMapping[parentBlock2.Hash()] = parentBlock2 + blockChain.SetParentBlocksToReturn(blockMapping) blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) - // Need to have listeners on the channels or the subscription will be closed and the processing halted + blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1) + blockChain.SetReceiptsForHash(testBlock2.Hash(), testReceipts2) + + payloads := make([]statediff.Payload, 0, 2) + wg := sync.WaitGroup{} go func() { - select { - case <-payloadChan: - case <-quitChan: + wg.Add(1) + for i := 0; i < 2; i++ { + select { + case payload := <-payloadChan: + payloads = append(payloads, payload) + case <-quitChan: + } } + wg.Done() }() + service.Loop(eventsChannel) + wg.Wait() + if len(payloads) != 2 { + t.Error("Test failure:", t.Name()) + t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads)) + } + + testReceipts1Rlp, err := rlp.EncodeToBytes(testReceipts1) + if err != nil { + t.Error(err) + } + testReceipts2Rlp, err := rlp.EncodeToBytes(testReceipts2) + if err != nil { + t.Error(err) + } + expectedReceiptsRlp := [][]byte{testReceipts1Rlp, testReceipts2Rlp, nil} + for i, payload := range payloads { + if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual reeipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) + } + } if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { t.Error("Test failure:", t.Name()) @@ -121,9 +165,20 @@ func testErrorInBlockLoop(t *testing.T) { QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]statediff.Subscription), } - - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan) + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockChain.SetParentBlocksToReturn(blockMapping) blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + // Need to have listeners on the channels or the subscription will be closed and the processing halted + go func() { + select { + case <-payloadChan: + case <-quitChan: + } + }() service.Loop(eventsChannel) if !bytes.Equal(builder.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index f2c097d38..508435236 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -30,16 +30,20 @@ import ( // BlockChain is a mock blockchain for testing type BlockChain struct { ParentHashesLookedUp []common.Hash - parentBlocksToReturn []*types.Block + parentBlocksToReturn map[common.Hash]*types.Block callCount int ChainEvents []core.ChainEvent + Receipts map[common.Hash]types.Receipts } // AddToStateDiffProcessedCollection mock method func (blockChain *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} // SetParentBlocksToReturn mock method -func (blockChain *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { +func (blockChain *BlockChain) SetParentBlocksToReturn(blocks map[common.Hash]*types.Block) { + if blockChain.parentBlocksToReturn == nil { + blockChain.parentBlocksToReturn = make(map[common.Hash]*types.Block) + } blockChain.parentBlocksToReturn = blocks } @@ -49,10 +53,9 @@ func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { var parentBlock *types.Block if len(blockChain.parentBlocksToReturn) > 0 { - parentBlock = blockChain.parentBlocksToReturn[blockChain.callCount] + parentBlock = blockChain.parentBlocksToReturn[hash] } - blockChain.callCount++ return parentBlock } @@ -84,3 +87,16 @@ func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) eve return subscription } + +// SetReceiptsForHash mock method +func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) { + if blockChain.Receipts == nil { + blockChain.Receipts = make(map[common.Hash]types.Receipts) + } + blockChain.Receipts[hash] = receipts +} + +// GetReceiptsByHash mock method +func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { + return blockChain.Receipts[hash] +} diff --git a/statediff/types.go b/statediff/types.go index 6df398a10..2db58d52b 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -36,15 +36,16 @@ type Subscription struct { // Payload packages the data to send to StateDiffingService subscriptions type Payload struct { - BlockRlp []byte `json:"blockRlp" gencodec:"required"` + BlockRlp []byte `json:"blockRlp"` + ReceiptsRlp []byte `json:"receiptsRlp"` StateDiffRlp []byte `json:"stateDiff" gencodec:"required"` Err error `json:"error"` } // StateDiff is the final output structure from the builder type StateDiff struct { - BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` CreatedAccounts []AccountDiff `json:"createdAccounts" gencodec:"required"` DeletedAccounts []AccountDiff `json:"deletedAccounts" gencodec:"required"` UpdatedAccounts []AccountDiff `json:"updatedAccounts" gencodec:"required"` -- 2.45.2 From a08be8596e732286095b9a515e7574e80beb4d14 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 29 Jul 2019 14:09:58 -0500 Subject: [PATCH 14/14] review fixes/changes --- core/blockchain.go | 4 ++-- eth/config.go | 2 -- statediff/builder.go | 13 ------------- statediff/doc.go | 17 +++++++++++++---- statediff/helpers.go | 31 +++++++------------------------ statediff/service.go | 2 +- statediff/service_test.go | 10 +++++----- 7 files changed, 28 insertions(+), 51 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index cf1a65cd5..0c56a13c9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1337,7 +1337,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. break } if bc.cacheConfig.ProcessingStateDiffs { - if !bc.allowedRootToBeDereferenced(root.(common.Hash)) { + if !bc.rootAllowedToBeDereferenced(root.(common.Hash)) { bc.triegc.Push(root, number) break } else { @@ -1406,7 +1406,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // since we need the state tries of the current block and its parent in-memory // in order to process statediffs, we should avoid dereferencing roots until // its statediff and its child have been processed -func (bc *BlockChain) allowedRootToBeDereferenced(root common.Hash) bool { +func (bc *BlockChain) rootAllowedToBeDereferenced(root common.Hash) bool { diffProcessedForSelfAndChildCount := 2 count := bc.stateDiffsProcessed[root] return count >= diffProcessedForSelfAndChildCount diff --git a/eth/config.go b/eth/config.go index 8ed014e24..e67e060cc 100644 --- a/eth/config.go +++ b/eth/config.go @@ -151,8 +151,6 @@ type Config struct { // RPCGasCap is the global gas cap for eth-call variants. RPCGasCap *big.Int `toml:",omitempty"` - StateDiff bool - // Checkpoint is a hardcoded checkpoint which can be nil. Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` diff --git a/statediff/builder.go b/statediff/builder.go index 765152e87..ad3acce52 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -25,7 +25,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" @@ -318,15 +317,3 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDi return storageDiffs, nil } - -func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { - log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...))) - addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)) - if err != nil { - log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err) - return nil, err - } - addr := common.BytesToAddress(addrBytes) - log.Debug("Address found", "Address", addr) - return &addr, nil -} diff --git a/statediff/doc.go b/statediff/doc.go index 0e6d5f3e1..35c48c02d 100644 --- a/statediff/doc.go +++ b/statediff/doc.go @@ -15,11 +15,11 @@ // along with the go-ethereum library. If not, see . /* -This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go - Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, relaying the objects to any rpc subscriptions. +This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go + The service is spun up using the below CLI flags --statediff: boolean flag, turns on the service --statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs. @@ -27,7 +27,16 @@ The service is spun up using the below CLI flags --statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes. --statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3 -If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. +If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default. + +The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node). + +e.g. + +$ ./geth --statediff --statediff.streamblock --ws --syncmode "full" + +This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service. +Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs. Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". @@ -41,7 +50,7 @@ for { select { case stateDiffPayload := <- stateDiffPayloadChan: processPayload(stateDiffPayload) - case err := <= rpcSub.Err(): + case err := <- rpcSub.Err(): log.Error(err) } } diff --git a/statediff/helpers.go b/statediff/helpers.go index 89852b55c..52ae1c392 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -37,7 +37,7 @@ func sortKeys(data AccountsMap) []string { return keys } -// BytesToNiblePath +// bytesToNiblePath converts the byte representation of a path to its string representation func bytesToNiblePath(path []byte) string { if hasTerm(path) { path = path[:len(path)-1] @@ -53,6 +53,8 @@ func bytesToNiblePath(path []byte) string { return nibblePath } +// findIntersection finds the set of strings from both arrays that are equivalent (same key as same index) +// this is used to find which keys have been both "deleted" and "created" i.e. they were "updated". func findIntersection(a, b []string) []string { lenA := len(a) lenB := len(b) @@ -63,13 +65,13 @@ func findIntersection(a, b []string) []string { } for { switch strings.Compare(a[iOfA], b[iOfB]) { - // a[iOfA] < b[iOfB] + // -1 when a[iOfA] < b[iOfB] case -1: iOfA++ if iOfA >= lenA { return updates } - // a[iOfA] == b[iOfB] + // 0 when a[iOfA] == b[iOfB] case 0: updates = append(updates, a[iOfA]) iOfA++ @@ -77,7 +79,7 @@ func findIntersection(a, b []string) []string { if iOfA >= lenA || iOfB >= lenB { return updates } - // a[iOfA] > b[iOfB] + // 1 when a[iOfA] > b[iOfB] case 1: iOfB++ if iOfB >= lenB { @@ -88,30 +90,11 @@ func findIntersection(a, b []string) []string { } +// pathToStr converts the NodeIterator path to a string representation func pathToStr(it trie.NodeIterator) string { return bytesToNiblePath(it.Path()) } -// Duplicated from trie/encoding.go -func hexToKeyBytes(hex []byte) []byte { - if hasTerm(hex) { - hex = hex[:len(hex)-1] - } - if len(hex)&1 != 0 { - panic("can't convert hex key of odd length") - } - key := make([]byte, (len(hex)+1)/2) - decodeNibbles(hex, key) - - return key -} - -func decodeNibbles(nibbles []byte, bytes []byte) { - for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { - bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] - } -} - // hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 diff --git a/statediff/service.go b/statediff/service.go index be3dc6cf9..658f4c8a8 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -176,7 +176,7 @@ func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) err payload.ReceiptsRlp = receiptBuff.Bytes() } - // If we have any websocket subscriptions listening in, send the data to them + // If we have any rpc subscriptions listening in, send the data to them sds.send(payload) return nil } diff --git a/statediff/service_test.go b/statediff/service_test.go index 1508542b7..6119f6ecb 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -131,27 +131,27 @@ func testErrorInChainEventLoop(t *testing.T) { for i, payload := range payloads { if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) { t.Error("Test failure:", t.Name()) - t.Logf("Actual reeipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) + t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) } } if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) } if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) + t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) } if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) + t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) } //look up the parent block from its hash expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) + t.Logf("Actual parent hash does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) } } -- 2.45.2