From 6932b03a214c89dc7051fcd49c7be2f8d18b9d3c Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 28 Jan 2019 15:31:01 -0600 Subject: [PATCH 01/17] Write state diff to CSV (#2) * port statediff from https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go; minor fixes * integrating state diff extracting, building, and persisting into geth processes * work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor * Add a state diff service * Remove diff extractor from blockchain * Update imports * Move statediff on/off check to geth cmd config * Update starting state diff service * Add debugging logs for creating diff * Add statediff extractor and builder tests and small refactoring * Start to write statediff to a CSV * Restructure statediff directory * Pull CSV publishing methods into their own file * Reformatting due to go fmt * Add gomega to vendor dir * Remove testing focuses * Update statediff tests to use golang test pkg instead of ginkgo - builder_test - extractor_test - publisher_test * Use hexutil.Encode instead of deprecated common.ToHex * Remove OldValue from DiffBigInt and DiffUint64 fields * Update builder test * Remove old storage value from updated accounts * Remove old values from created/deleted accounts * Update publisher to account for only storing current account values * Update service loop and fetching previous block * Update testing - remove statediff ginkgo test suite file - move mocks to their own dir * Updates per go fmt * Updates to tests * Pass statediff mode and path in through cli * Return filename from publisher * Remove some duplication in builder * Remove code field from state diff output this is the contract byte code, and it can still be obtained by querying the db by the codeHash * Consolidate acct diff structs for updated & updated/deleted accts * Include block number in csv filename * Clean up error logging * Cleanup formatting, spelling, etc * Address PR comments * Add contract address and storage value to csv * Refactor accumulating account row in csv publisher * Add DiffStorage struct * Add storage key to csv * Address PR comments * Fix publisher to include rows for accounts that don't have store updates * Update builder test after merging in release/1.8 * Update test contract to include storage on contract intialization - so that we're able to test that storage diffing works for created and deleted accounts (not just updated accounts). * Factor out a common trie iterator method in builder --- cmd/geth/config.go | 4 + cmd/geth/main.go | 3 + cmd/geth/usage.go | 8 + cmd/utils/flags.go | 47 ++ statediff/builder/builder.go | 272 +++++++++++ statediff/builder/builder_test.go | 350 ++++++++++++++ statediff/builder/helpers.go | 118 +++++ statediff/builder/struct.go | 78 ++++ statediff/config.go | 93 ++++ statediff/config_test.go | 23 + statediff/extractor/extractor.go | 51 +++ statediff/extractor/extractor_test.go | 122 +++++ statediff/publisher/csv.go | 137 ++++++ statediff/publisher/publisher.go | 48 ++ statediff/publisher/publisher_test.go | 315 +++++++++++++ statediff/service/service.go | 82 ++++ statediff/service/service_test.go | 76 ++++ statediff/testhelpers/helpers.go | 4 + statediff/testhelpers/mocks/blockchain.go | 34 ++ statediff/testhelpers/mocks/builder.go | 32 ++ statediff/testhelpers/mocks/error.go | 5 + statediff/testhelpers/mocks/extractor.go | 20 + statediff/testhelpers/mocks/publisher.go | 17 + statediff/testhelpers/test_data.go | 69 +++ trie/encoding.go | 4 +- trie/encoding_test.go | 6 +- trie/iterator.go | 2 +- vendor/github.com/onsi/gomega/CHANGELOG.md | 125 +++++ vendor/github.com/onsi/gomega/CONTRIBUTING.md | 14 + vendor/github.com/onsi/gomega/LICENSE | 20 + vendor/github.com/onsi/gomega/Makefile | 6 + vendor/github.com/onsi/gomega/README.md | 21 + vendor/github.com/onsi/gomega/RELEASING.md | 12 + .../github.com/onsi/gomega/format/format.go | 382 ++++++++++++++++ vendor/github.com/onsi/gomega/go.mod | 15 + vendor/github.com/onsi/gomega/go.sum | 24 + vendor/github.com/onsi/gomega/gomega_dsl.go | 429 ++++++++++++++++++ .../gomega/internal/assertion/assertion.go | 105 +++++ .../asyncassertion/async_assertion.go | 194 ++++++++ .../internal/oraclematcher/oracle_matcher.go | 25 + .../testingtsupport/testing_t_support.go | 60 +++ vendor/github.com/onsi/gomega/matchers.go | 427 +++++++++++++++++ vendor/github.com/onsi/gomega/matchers/and.go | 63 +++ .../matchers/assignable_to_type_of_matcher.go | 35 ++ .../onsi/gomega/matchers/attributes_slice.go | 14 + .../onsi/gomega/matchers/be_a_directory.go | 54 +++ .../onsi/gomega/matchers/be_a_regular_file.go | 54 +++ .../gomega/matchers/be_an_existing_file.go | 38 ++ .../onsi/gomega/matchers/be_closed_matcher.go | 46 ++ .../onsi/gomega/matchers/be_empty_matcher.go | 27 ++ .../matchers/be_equivalent_to_matcher.go | 34 ++ .../onsi/gomega/matchers/be_false_matcher.go | 26 ++ .../onsi/gomega/matchers/be_identical_to.go | 37 ++ .../onsi/gomega/matchers/be_nil_matcher.go | 18 + .../gomega/matchers/be_numerically_matcher.go | 132 ++++++ .../onsi/gomega/matchers/be_sent_matcher.go | 71 +++ .../gomega/matchers/be_temporally_matcher.go | 66 +++ .../onsi/gomega/matchers/be_true_matcher.go | 26 ++ .../onsi/gomega/matchers/be_zero_matcher.go | 28 ++ .../onsi/gomega/matchers/consist_of.go | 80 ++++ .../matchers/contain_element_matcher.go | 56 +++ .../matchers/contain_substring_matcher.go | 38 ++ .../onsi/gomega/matchers/equal_matcher.go | 42 ++ .../onsi/gomega/matchers/have_cap_matcher.go | 28 ++ .../onsi/gomega/matchers/have_key_matcher.go | 54 +++ .../matchers/have_key_with_value_matcher.go | 74 +++ .../onsi/gomega/matchers/have_len_matcher.go | 28 ++ .../gomega/matchers/have_occurred_matcher.go | 33 ++ .../gomega/matchers/have_prefix_matcher.go | 36 ++ .../gomega/matchers/have_suffix_matcher.go | 36 ++ .../gomega/matchers/match_error_matcher.go | 51 +++ .../gomega/matchers/match_json_matcher.go | 65 +++ .../gomega/matchers/match_regexp_matcher.go | 43 ++ .../onsi/gomega/matchers/match_xml_matcher.go | 134 ++++++ .../gomega/matchers/match_yaml_matcher.go | 76 ++++ vendor/github.com/onsi/gomega/matchers/not.go | 30 ++ vendor/github.com/onsi/gomega/matchers/or.go | 67 +++ .../onsi/gomega/matchers/panic_matcher.go | 46 ++ .../onsi/gomega/matchers/receive_matcher.go | 128 ++++++ .../matchers/semi_structured_data_support.go | 92 ++++ .../onsi/gomega/matchers/succeed_matcher.go | 33 ++ .../goraph/bipartitegraph/bipartitegraph.go | 41 ++ .../bipartitegraph/bipartitegraphmatching.go | 159 +++++++ .../matchers/support/goraph/edge/edge.go | 61 +++ .../matchers/support/goraph/node/node.go | 7 + .../matchers/support/goraph/util/util.go | 7 + .../onsi/gomega/matchers/type_support.go | 179 ++++++++ .../onsi/gomega/matchers/with_transform.go | 72 +++ vendor/github.com/onsi/gomega/types/types.go | 26 ++ 89 files changed, 6464 insertions(+), 6 deletions(-) create mode 100644 statediff/builder/builder.go create mode 100644 statediff/builder/builder_test.go create mode 100644 statediff/builder/helpers.go create mode 100644 statediff/builder/struct.go create mode 100644 statediff/config.go create mode 100644 statediff/config_test.go create mode 100644 statediff/extractor/extractor.go create mode 100644 statediff/extractor/extractor_test.go create mode 100644 statediff/publisher/csv.go create mode 100644 statediff/publisher/publisher.go create mode 100644 statediff/publisher/publisher_test.go create mode 100644 statediff/service/service.go create mode 100644 statediff/service/service_test.go create mode 100644 statediff/testhelpers/helpers.go create mode 100644 statediff/testhelpers/mocks/blockchain.go create mode 100644 statediff/testhelpers/mocks/builder.go create mode 100644 statediff/testhelpers/mocks/error.go create mode 100644 statediff/testhelpers/mocks/extractor.go create mode 100644 statediff/testhelpers/mocks/publisher.go create mode 100644 statediff/testhelpers/test_data.go create mode 100644 vendor/github.com/onsi/gomega/CHANGELOG.md create mode 100644 vendor/github.com/onsi/gomega/CONTRIBUTING.md create mode 100644 vendor/github.com/onsi/gomega/LICENSE create mode 100644 vendor/github.com/onsi/gomega/Makefile create mode 100644 vendor/github.com/onsi/gomega/README.md create mode 100644 vendor/github.com/onsi/gomega/RELEASING.md create mode 100644 vendor/github.com/onsi/gomega/format/format.go create mode 100644 vendor/github.com/onsi/gomega/go.mod create mode 100644 vendor/github.com/onsi/gomega/go.sum create mode 100644 vendor/github.com/onsi/gomega/gomega_dsl.go create mode 100644 vendor/github.com/onsi/gomega/internal/assertion/assertion.go create mode 100644 vendor/github.com/onsi/gomega/internal/asyncassertion/async_assertion.go create mode 100644 vendor/github.com/onsi/gomega/internal/oraclematcher/oracle_matcher.go create mode 100644 vendor/github.com/onsi/gomega/internal/testingtsupport/testing_t_support.go create mode 100644 vendor/github.com/onsi/gomega/matchers.go create mode 100644 vendor/github.com/onsi/gomega/matchers/and.go create mode 100644 vendor/github.com/onsi/gomega/matchers/assignable_to_type_of_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/attributes_slice.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_a_directory.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_a_regular_file.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_an_existing_file.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_closed_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_empty_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_equivalent_to_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_false_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_identical_to.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_nil_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_numerically_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_sent_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_temporally_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_true_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_zero_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/consist_of.go create mode 100644 vendor/github.com/onsi/gomega/matchers/contain_element_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/contain_substring_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/equal_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_cap_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_key_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_key_with_value_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_len_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_occurred_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_prefix_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_suffix_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/match_error_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/match_json_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/match_regexp_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/match_xml_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/match_yaml_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/not.go create mode 100644 vendor/github.com/onsi/gomega/matchers/or.go create mode 100644 vendor/github.com/onsi/gomega/matchers/panic_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/receive_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/semi_structured_data_support.go create mode 100644 vendor/github.com/onsi/gomega/matchers/succeed_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraph.go create mode 100644 vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraphmatching.go create mode 100644 vendor/github.com/onsi/gomega/matchers/support/goraph/edge/edge.go create mode 100644 vendor/github.com/onsi/gomega/matchers/support/goraph/node/node.go create mode 100644 vendor/github.com/onsi/gomega/matchers/support/goraph/util/util.go create mode 100644 vendor/github.com/onsi/gomega/matchers/type_support.go create mode 100644 vendor/github.com/onsi/gomega/matchers/with_transform.go create mode 100644 vendor/github.com/onsi/gomega/types/types.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index e2c107ffc..29210e549 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -182,6 +182,10 @@ func makeFullNode(ctx *cli.Context) *node.Node { if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) } + + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + utils.RegisterStateDiffService(stack, ctx) + } return stack } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 019eee605..eaeb3adfb 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -148,6 +148,9 @@ var ( utils.GpoPercentileFlag, utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, + utils.StateDiffFlag, + utils.StateDiffModeFlag, + utils.StateDiffPathFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index af195425b..ce6eab279 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -262,6 +262,14 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerLegacyExtraDataFlag, }, }, + { + Name: "STATE DIFF", + Flags: []cli.Flag{ + utils.StateDiffFlag, + utils.StateDiffModeFlag, + utils.StateDiffPathFlag, + }, + }, { Name: "MISC", }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 773207339..2beeb2900 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -65,6 +65,8 @@ import ( whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" pcsclite "github.com/gballet/go-libpcsclite" cli "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/statediff/service" + "github.com/ethereum/go-ethereum/statediff" ) var ( @@ -755,6 +757,23 @@ var ( Usage: "External EVM configuration (default = built-in interpreter)", Value: "", } + + StateDiffFlag = cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", + } + + StateDiffModeFlag = cli.StringFlag{ + Name: "statediff.mode", + Usage: "Enables the user to determine which persistence mode they'd like to store the state diffs in.", + Value: "csv", + } + + StateDiffPathFlag = cli.StringFlag{ + Name: "statediff.path", + Usage: "Enables the user to determine where to persist the state diffs.", + Value: ".", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1610,6 +1629,34 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st } } +func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { + //based on the context, if path and mode are set, update the config here + //otherwise pass in an empty config + + modeFlag := ctx.GlobalString(StateDiffModeFlag.Name) + mode, err := statediff.NewMode(modeFlag) + if err != nil { + Fatalf("Failed to register State Diff Service", err) + } + + path := ctx.GlobalString(StateDiffPathFlag.Name) + + config := statediff.Config{ + Mode: mode, + Path: path, + } + + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + var ethServ *eth.Ethereum + ctx.Service(ðServ) + chainDb := ethServ.ChainDb() + blockChain := ethServ.BlockChain() + return service.NewStateDiffService(chainDb, blockChain, config) + }); err != nil { + Fatalf("Failed to register State Diff Service", err) + } +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/statediff/builder/builder.go b/statediff/builder/builder.go new file mode 100644 index 000000000..825869150 --- /dev/null +++ b/statediff/builder/builder.go @@ -0,0 +1,272 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +type Builder interface { + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) +} + +type builder struct { + chainDB ethdb.Database + trieDB *trie.Database + cachedTrie *trie.Trie +} + +func NewBuilder(db ethdb.Database) *builder { + return &builder{ + chainDB: db, + trieDB: trie.NewDatabase(db), + } +} + +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { + // Generate tries for old and new states + oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) + if err != nil { + log.Error("Error creating trie for oldStateRoot", "error", err) + return nil, err + } + newTrie, err := trie.New(newStateRoot, sdb.trieDB) + if err != nil { + log.Error("Error creating trie for newStateRoot", "error", err) + return nil, err + } + + // Find created accounts + oldIt := oldTrie.NodeIterator([]byte{}) + newIt := newTrie.NodeIterator([]byte{}) + creations, err := sdb.collectDiffNodes(oldIt, newIt) + if err != nil { + log.Error("Error collecting creation diff nodes", "error", err) + return nil, err + } + + // Find deleted accounts + oldIt = oldTrie.NodeIterator([]byte{}) + newIt = newTrie.NodeIterator([]byte{}) + deletions, err := sdb.collectDiffNodes(newIt, oldIt) + if err != nil { + log.Error("Error collecting deletion diff nodes", "error", err) + return nil, err + } + + // Find all the diffed keys + createKeys := sortKeys(creations) + deleteKeys := sortKeys(deletions) + updatedKeys := findIntersection(createKeys, deleteKeys) + + // Build and return the statediff + updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) + if err != nil { + log.Error("Error building diff for updated accounts", "error", err) + return nil, err + } + createdAccounts, err := sdb.buildDiffEventual(creations) + if err != nil { + log.Error("Error building diff for created accounts", "error", err) + return nil, err + } + deletedAccounts, err := sdb.buildDiffEventual(deletions) + if err != nil { + log.Error("Error building diff for deleted accounts", "error", err) + return nil, err + } + + return &StateDiff{ + BlockNumber: blockNumber, + BlockHash: blockHash, + CreatedAccounts: createdAccounts, + DeletedAccounts: deletedAccounts, + UpdatedAccounts: updatedAccounts, + }, nil +} + +func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { + var diffAccounts = make(map[common.Address]*state.Account) + it, _ := trie.NewDifferenceIterator(a, b) + + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", common.Hash(it.Hash())) + if it.Leaf() { + + // lookup address + path := make([]byte, len(it.Path())-1) + copy(path, it.Path()) + addr, err := sdb.addressByPath(path) + if err != nil { + log.Error("Error looking up address via path", "path", path, "error", err) + return nil, err + } + + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { + log.Error("Error looking up account via address", "address", addr, "error", err) + return nil, err + } + + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", addr, "account", account) + diffAccounts[*addr] = &account + } + cont := it.Next(true) + if !cont { + break + } + } + + return diffAccounts, nil +} + +func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account) (map[common.Address]AccountDiff, error) { + accountDiffs := make(map[common.Address]AccountDiff) + for addr, val := range accounts { + sr := val.Root + storageDiffs, err := sdb.buildStorageDiffsEventual(sr) + if err != nil { + log.Error("Failed building eventual storage diffs", "Address", addr, "error", err) + return nil, err + } + + codeHash := hexutil.Encode(val.CodeHash) + hexRoot := val.Root.Hex() + nonce := DiffUint64{Value: &val.Nonce} + balance := DiffBigInt{Value: val.Balance} + contractRoot := DiffString{Value: &hexRoot} + accountDiffs[addr] = AccountDiff{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + } + + return accountDiffs, nil +} + +func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiff, error) { + updatedAccounts := make(map[common.Address]AccountDiff) + for _, val := range updatedKeys { + createdAcc := creations[common.HexToAddress(val)] + deletedAcc := deletions[common.HexToAddress(val)] + oldSR := deletedAcc.Root + newSR := createdAcc.Root + if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { + log.Error("Failed building storage diffs", "Address", val, "error", err) + return nil, err + } else { + nonce := DiffUint64{Value: &createdAcc.Nonce} + balance := DiffBigInt{Value: createdAcc.Balance} + codeHash := hexutil.Encode(createdAcc.CodeHash) + + nHexRoot := createdAcc.Root.Hex() + contractRoot := DiffString{Value: &nHexRoot} + + updatedAccounts[common.HexToAddress(val)] = AccountDiff{ + Nonce: nonce, + Balance: balance, + CodeHash: codeHash, + ContractRoot: contractRoot, + Storage: storageDiffs, + } + delete(creations, common.HexToAddress(val)) + delete(deletions, common.HexToAddress(val)) + } + } + return updatedAccounts, nil +} + +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffStorage, error) { + log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) + sTrie, err := trie.New(sr, sdb.trieDB) + if err != nil { + log.Info("error in build storage diff eventual", "error", err) + return nil, err + } + it := sTrie.NodeIterator(make([]byte, 0)) + storageDiffs := buildStorageDiffsFromTrie(it) + return storageDiffs, nil +} + +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffStorage, error) { + log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) + oldTrie, err := trie.New(oldSR, sdb.trieDB) + if err != nil { + return nil, err + } + newTrie, err := trie.New(newSR, sdb.trieDB) + if err != nil { + return nil, err + } + + oldIt := oldTrie.NodeIterator(make([]byte, 0)) + newIt := newTrie.NodeIterator(make([]byte, 0)) + it, _ := trie.NewDifferenceIterator(oldIt, newIt) + storageDiffs := buildStorageDiffsFromTrie(it) + + return storageDiffs, nil +} + +func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage { + storageDiffs := make(map[string]DiffStorage) + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + path := pathToStr(it) + storageKey:= hexutil.Encode(it.LeafKey()) + storageValue := hexutil.Encode(it.LeafBlob()) + storageDiffs[path] = DiffStorage{ + Key: &storageKey, + Value: &storageValue, + } + } + + cont := it.Next(true) + if !cont { + break + } + } + + return storageDiffs +} + +func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { + log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...))) + if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)); err != nil { + log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err) + return nil, err + } else { + addr := common.BytesToAddress(addrBytes) + log.Debug("Address found", "Address", addr) + return &addr, nil + } +} diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go new file mode 100644 index 000000000..72ebfff18 --- /dev/null +++ b/statediff/builder/builder_test.go @@ -0,0 +1,350 @@ +package builder_test + +import ( + "bytes" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" + b "github.com/ethereum/go-ethereum/statediff/builder" + "math/big" + "reflect" + "testing" +) + +var ( + testdb = ethdb.NewMemDatabase() + + testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + testBankFunds = big.NewInt(100000000) + genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) + + account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") + contractAddr common.Address + emptyAccountDiffEventualMap = make(map[common.Address]b.AccountDiff) + emptyAccountDiffIncrementalMap = make(map[common.Address]b.AccountDiff) + block0Hash, block1Hash, block2Hash, block3Hash common.Hash + block0, block1, block2, block3 *types.Block + builder b.Builder + miningReward = int64(2000000000000000000) + burnAddress = common.HexToAddress("0x0") +) + +func TestBuilder(t *testing.T) { + _, blockMap := makeChain(3, genesis) + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block2Hash = common.HexToHash("0xde75663f36a8497b4bdda2a4b52bd9540b705a2728c7391c59b8cb2cde5a2feb") + block3Hash = common.HexToHash("0x76c6d0e39285cee40d5e5fadc6141ca88c8ab8bd1a15d46717205af2efbb4a3c") + + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + block2 = blockMap[block2Hash] + block3 = blockMap[block3Hash] + builder = b.NewBuilder(testdb) + + type arguments struct { + oldStateRoot common.Hash + newStateRoot common.Hash + blockNumber int64 + blockHash common.Hash + } + + var ( + balanceChange10000 = int64(10000) + balanceChange1000 = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + block2Account2Balance = int64(1000) + nonce0 = uint64(0) + nonce1 = uint64(1) + nonce2 = uint64(2) + nonce3 = uint64(3) + originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" + originalStorageLocation = common.HexToHash("0") + originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).String() + updatedStorageLocation = common.HexToHash("2") + updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).String() + originalStorageValue = "0x01" + updatedStorageValue = "0x03" + ) + + var tests = []struct { + name string + startingArguments arguments + expected *b.StateDiff + }{ + { + "testEmptyDiff", + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block0.Root(), + blockNumber: block0.Number().Int64(), + blockHash: block0Hash, + }, + &b.StateDiff{ + BlockNumber: block0.Number().Int64(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block1.Root(), + blockNumber: block1.Number().Int64(), + blockHash: block1Hash, + }, + &b.StateDiff{ + BlockNumber: block1.Number().Int64(), + BlockHash: block1.Hash(), + CreatedAccounts: map[common.Address]b.AccountDiff{ + account1Addr: { + Nonce: b.DiffUint64{Value: &nonce0}, + Balance: b.DiffBigInt{Value: big.NewInt(balanceChange10000)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + burnAddress: { + Nonce: b.DiffUint64{Value: &nonce0}, + Balance: b.DiffBigInt{Value: big.NewInt(miningReward)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: map[common.Address]b.AccountDiff{ + testBankAddress: { + Nonce: b.DiffUint64{Value: &nonce1}, + Balance: b.DiffBigInt{Value: big.NewInt(testBankFunds.Int64() - balanceChange10000)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + }, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + arguments{ + oldStateRoot: block1.Root(), + newStateRoot: block2.Root(), + blockNumber: block2.Number().Int64(), + blockHash: block2Hash, + }, + &b.StateDiff{ + BlockNumber: block2.Number().Int64(), + BlockHash: block2.Hash(), + CreatedAccounts: map[common.Address]b.AccountDiff{ + account2Addr: { + Nonce: b.DiffUint64{Value: &nonce0}, + Balance: b.DiffBigInt{Value: big.NewInt(balanceChange1000)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + contractAddr: { + Nonce: b.DiffUint64{Value: &nonce1}, + Balance: b.DiffBigInt{Value: big.NewInt(0)}, + CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", + ContractRoot: b.DiffString{Value: &contractContractRoot}, + Storage: map[string]b.DiffStorage{ + originalStorageKey: { + Key: &originalStorageKey, + Value: &originalStorageValue}, + }, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: map[common.Address]b.AccountDiff{ + testBankAddress: { + Nonce: b.DiffUint64{Value: &nonce2}, + Balance: b.DiffBigInt{Value: big.NewInt(block1BankBalance - balanceChange1000)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + account1Addr: { + Nonce: b.DiffUint64{Value: &nonce2}, + Balance: b.DiffBigInt{Value: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + burnAddress: { + Nonce: b.DiffUint64{Value: &nonce0}, + Balance: b.DiffBigInt{Value: big.NewInt(miningReward + miningReward)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + arguments{ + oldStateRoot: block2.Root(), + newStateRoot: block3.Root(), + blockNumber: block3.Number().Int64(), + blockHash: block3.Hash(), + }, + &b.StateDiff{ + BlockNumber: block3.Number().Int64(), + BlockHash: block3.Hash(), + CreatedAccounts: map[common.Address]b.AccountDiff{}, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: map[common.Address]b.AccountDiff{ + account2Addr: { + Nonce: b.DiffUint64{Value: &nonce0}, + Balance: b.DiffBigInt{Value: big.NewInt(block2Account2Balance + miningReward)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + contractAddr: { + Nonce: b.DiffUint64{Value: &nonce1}, + Balance: b.DiffBigInt{Value: big.NewInt(0)}, + CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", + ContractRoot: b.DiffString{Value: &newContractRoot}, + Storage: map[string]b.DiffStorage{ + "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace": { + Key: &updatedStorageKey, + Value: &updatedStorageValue}, + }, + }, + testBankAddress: { + Nonce: b.DiffUint64{Value: &nonce3}, + Balance: b.DiffBigInt{Value: big.NewInt(99989000)}, + CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + ContractRoot: b.DiffString{Value: &originalContractRoot}, + Storage: map[string]b.DiffStorage{}, + }, + }, + }, + }, + } + + for _, test := range tests { + arguments := test.startingArguments + diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) + if err != nil { + t.Error(err) + } + + fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} + + for _, field := range fields { + reflectionOfDiff := reflect.ValueOf(diff) + diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) + + reflectionOfExpected := reflect.ValueOf(test.expected) + expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) + + diffValueInterface := diffValue.Interface() + expectedValueInterface := expectedValue.Interface() + + if !equals(diffValueInterface, expectedValueInterface) { + t.Logf("Test failed: %s", test.name) + t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) + } + } + } +} + +func equals(actual, expected interface{}) (success bool) { + if actualByteSlice, ok := actual.([]byte); ok { + if expectedByteSlice, ok := expected.([]byte); ok { + return bytes.Equal(actualByteSlice, expectedByteSlice) + } + } + + return reflect.DeepEqual(actual, expected) +} + +// makeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm +} + +func testChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // account1Addr passes it on to account #2. + // account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + nonce := block.TxNonce(account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key) + contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(account2Addr) + //get function: 60cd2685 + //put function: c16431b9 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) + block.AddTx(tx) + } +} + +/* +contract test { + + uint256[100] data; + + constructor() public { + data = [1]; + } + + function Put(uint256 addr, uint256 value) { + data[addr] = value; + } + + function Get(uint256 addr) constant returns (uint256 value) { + return data[addr]; + } +} +*/ diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go new file mode 100644 index 000000000..26602ebc9 --- /dev/null +++ b/statediff/builder/helpers.go @@ -0,0 +1,118 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder + +import ( + "sort" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/trie" +) + +func sortKeys(data map[common.Address]*state.Account) []string { + var keys []string + for key := range data { + keys = append(keys, key.Hex()) + } + sort.Strings(keys) + + return keys +} + +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + break + // a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + break + // a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + break + } + } + +} + +func pathToStr(it trie.NodeIterator) string { + path := it.Path() + if hasTerm(path) { + path = path[:len(path)-1] + } + nibblePath := "" + for i, v := range common.ToHex(path) { + if i%2 == 0 && i > 1 { + continue + } + nibblePath = nibblePath + string(v) + } + + return nibblePath +} + +// Duplicated from trie/encoding.go +func hexToKeyBytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go new file mode 100644 index 000000000..e3ad599f1 --- /dev/null +++ b/statediff/builder/struct.go @@ -0,0 +1,78 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package builder + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +type StateDiff struct { + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts map[common.Address]AccountDiff `json:"createdAccounts" gencodec:"required"` + DeletedAccounts map[common.Address]AccountDiff `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts map[common.Address]AccountDiff `json:"updatedAccounts" gencodec:"required"` + + encoded []byte + err error +} + +func (self *StateDiff) ensureEncoded() { + if self.encoded == nil && self.err == nil { + self.encoded, self.err = json.Marshal(self) + } +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Implement Encoder interface for StateDiff +func (sd *StateDiff) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +type AccountDiff struct { + Nonce DiffUint64 `json:"nonce" gencodec:"required"` + Balance DiffBigInt `json:"balance" gencodec:"required"` + CodeHash string `json:"codeHash" gencodec:"required"` + ContractRoot DiffString `json:"contractRoot" gencodec:"required"` + Storage map[string]DiffStorage `json:"storage" gencodec:"required"` +} + +type DiffStorage struct { + Key *string `json:"key" gencodec:"optional"` + Value *string `json:"value" gencodec:"optional"` +} +type DiffString struct { + Value *string `json:"value" gencodec:"optional"` +} +type DiffUint64 struct { + Value *uint64 `json:"value" gencodec:"optional"` +} +type DiffBigInt struct { + Value *big.Int `json:"value" gencodec:"optional"` +} diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000..7f5ec3c35 --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,93 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import "fmt" + +type Config struct { + Mode StateDiffMode // Mode for storing diffs + Path string // Path for storing diffs +} + +type StateDiffMode int + +const ( + CSV StateDiffMode = iota + IPLD + LDB + SQL +) + +func (mode StateDiffMode) IsValid() bool { + return mode >= IPLD && mode <= SQL +} + +// String implements the stringer interface. +func (mode StateDiffMode) String() string { + switch mode { + case CSV: + return "csv" + case IPLD: + return "ipfs" + case LDB: + return "ldb" + case SQL: + return "sql" + default: + return "unknown" + } +} + +func NewMode(mode string) (StateDiffMode, error) { + stateDiffMode := StateDiffMode(0) + err := stateDiffMode.UnmarshalText([]byte(mode)) + return stateDiffMode, err +} + +func (mode StateDiffMode) MarshalText() ([]byte, error) { + switch mode { + case CSV: + return []byte("ipfs"), nil + case IPLD: + return []byte("ipfs"), nil + case LDB: + return []byte("ldb"), nil + case SQL: + return []byte("sql"), nil + default: + return nil, fmt.Errorf("unknown state diff storage mode %d", mode) + } +} + +func (mode *StateDiffMode) UnmarshalText(text []byte) error { + switch string(text) { + case "csv": + *mode = CSV + case "ipfs": + *mode = IPLD + case "ldb": + *mode = LDB + case "sql": + *mode = SQL + default: + return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text) + } + return nil +} diff --git a/statediff/config_test.go b/statediff/config_test.go new file mode 100644 index 000000000..82f2d3a92 --- /dev/null +++ b/statediff/config_test.go @@ -0,0 +1,23 @@ +package statediff_test + +import ( + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + "testing" +) + +func TestNewMode(t *testing.T) { + mode, err := statediff.NewMode("csv") + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + if mode != statediff.CSV { + t.Error() + } + + _, err = statediff.NewMode("not a real mode") + if err == nil { + t.Error("Expected an error, and got nil.") + } +} diff --git a/statediff/extractor/extractor.go b/statediff/extractor/extractor.go new file mode 100644 index 000000000..770973c8d --- /dev/null +++ b/statediff/extractor/extractor.go @@ -0,0 +1,51 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package extractor + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/publisher" +) + +type Extractor interface { + ExtractStateDiff(parent, current types.Block) (string, error) +} + +type extractor struct { + Builder builder.Builder // Interface for building state diff objects from two blocks + Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) +} + +func NewExtractor(builder builder.Builder, publisher publisher.Publisher) *extractor { + return &extractor{ + Builder: builder, + Publisher: publisher, + } +} + +func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { + stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) + if err != nil { + return "", err + } + + return e.Publisher.PublishStateDiff(stateDiff) +} diff --git a/statediff/extractor/extractor_test.go b/statediff/extractor/extractor_test.go new file mode 100644 index 000000000..b75ea60b0 --- /dev/null +++ b/statediff/extractor/extractor_test.go @@ -0,0 +1,122 @@ +package extractor_test + +import ( + "bytes" + "github.com/ethereum/go-ethereum/core/types" + b "github.com/ethereum/go-ethereum/statediff/builder" + e "github.com/ethereum/go-ethereum/statediff/extractor" + "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" + "math/big" + "math/rand" + "reflect" + "testing" +) + +var publisher mocks.Publisher +var builder mocks.Builder +var currentBlockNumber *big.Int +var parentBlock, currentBlock *types.Block +var expectedStateDiff b.StateDiff +var extractor e.Extractor +var err error + +func TestExtractor(t *testing.T) { + publisher = mocks.Publisher{} + builder = mocks.Builder{} + extractor = e.NewExtractor(&builder, &publisher) + if err != nil { + t.Error(err) + } + + blockNumber := rand.Int63() + parentBlockNumber := big.NewInt(blockNumber - int64(1)) + currentBlockNumber = big.NewInt(blockNumber) + parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) + currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) + + expectedStateDiff = b.StateDiff{ + BlockNumber: blockNumber, + BlockHash: currentBlock.Hash(), + CreatedAccounts: nil, + DeletedAccounts: nil, + UpdatedAccounts: nil, + } + + testBuildStateDiffStruct(t) + testBuildStateDiffErrorHandling(t) + testPublishingStateDiff(t) + testPublisherErrorHandling(t) +} + +func testBuildStateDiffStruct(t *testing.T) { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + if err != nil { + t.Error(err) + } + + if !equals(builder.OldStateRoot, parentBlock.Root()) { + t.Error() + } + if !equals(builder.NewStateRoot, currentBlock.Root()) { + t.Error() + } + if !equals(builder.BlockNumber, currentBlockNumber.Int64()) { + t.Error() + } + if !equals(builder.BlockHash, currentBlock.Hash()) { + t.Error() + } +} + +func testBuildStateDiffErrorHandling(t *testing.T) { + builder.SetBuilderError(mocks.Error) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + if err == nil { + t.Error(err) + } + + if !equals(err, mocks.Error) { + t.Error() + } + builder.SetBuilderError(nil) +} + +func testPublishingStateDiff(t *testing.T) { + builder.SetStateDiffToBuild(&expectedStateDiff) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + if err != nil { + t.Error(err) + } + + if !equals(publisher.StateDiff, &expectedStateDiff) { + t.Error() + } +} + +func testPublisherErrorHandling(t *testing.T) { + publisher.SetPublisherError(mocks.Error) + + _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) + if err == nil { + t.Error("Expected an error, but it didn't occur.") + } + if !equals(err, mocks.Error) { + t.Error() + } + + publisher.SetPublisherError(nil) +} + +func equals(actual, expected interface{}) (success bool) { + if actualByteSlice, ok := actual.([]byte); ok { + if expectedByteSlice, ok := expected.([]byte); ok { + return bytes.Equal(actualByteSlice, expectedByteSlice) + } + } + + return reflect.DeepEqual(actual, expected) +} diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go new file mode 100644 index 000000000..82eb3e466 --- /dev/null +++ b/statediff/publisher/csv.go @@ -0,0 +1,137 @@ +package publisher + +import ( + "encoding/csv" + "github.com/ethereum/go-ethereum/statediff/builder" + "os" + "path/filepath" + "strconv" + "time" + "github.com/ethereum/go-ethereum/common" +) + +var ( + Headers = []string{ + "blockNumber", "blockHash", "accountAction", "codeHash", + "nonceValue", "balanceValue", "contractRoot", "storageDiffPaths", + "accountAddress", "storageKey", "storageValue", + } + + timeStampFormat = "20060102150405.00000" + deletedAccountAction = "deleted" + createdAccountAction = "created" + updatedAccountAction = "updated" +) + +func createCSVFilePath(path, blockNumber string) string { + now := time.Now() + timeStamp := now.Format(timeStampFormat) + suffix := timeStamp + "-" + blockNumber + filePath := filepath.Join(path, suffix) + filePath = filePath + ".csv" + return filePath +} + +func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) { + filePath := createCSVFilePath(p.Config.Path, strconv.FormatInt(sd.BlockNumber, 10)) + + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return "", err + } + defer file.Close() + + writer := csv.NewWriter(file) + defer writer.Flush() + + var data [][]string + data = append(data, Headers) + for _, row := range accumulateAccountRows(sd) { + data = append(data, row) + } + for _, value := range data { + err := writer.Write(value) + if err != nil { + return "", err + } + } + + return filePath, nil +} + +func accumulateAccountRows(sd builder.StateDiff) [][]string { + var accountRows [][]string + for accountAddr, accountDiff := range sd.CreatedAccounts { + formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction) + + for _, accountData := range formattedAccountData { + accountRows = append(accountRows, accountData) + } + } + + for accountAddr, accountDiff := range sd.UpdatedAccounts { + formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction) + + for _, accountData := range formattedAccountData { + accountRows = append(accountRows, accountData) + } + } + + for accountAddr, accountDiff := range sd.DeletedAccounts { + formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction) + + for _, accountData := range formattedAccountData { + accountRows = append(accountRows, accountData) + } + } + + return accountRows +} + +func formatAccountData(accountAddr common.Address, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { + blockNumberString := strconv.FormatInt(sd.BlockNumber, 10) + blockHash := sd.BlockHash.String() + codeHash := accountDiff.CodeHash + nonce := strconv.FormatUint(*accountDiff.Nonce.Value, 10) + balance := accountDiff.Balance.Value.String() + newContractRoot := accountDiff.ContractRoot.Value + address := accountAddr.String() + var result [][]string + + if len(accountDiff.Storage) > 0 { + for storagePath, storage := range accountDiff.Storage { + formattedAccountData := []string{ + blockNumberString, + blockHash, + accountAction, + codeHash, + nonce, + balance, + *newContractRoot, + storagePath, + address, + *storage.Key, + *storage.Value, + } + + result = append(result, formattedAccountData) + } + } else { + formattedAccountData := []string{ + blockNumberString, + blockHash, + accountAction, + codeHash, + nonce, + balance, + *newContractRoot, + "", + address, + "", + "", + } + result = append(result, formattedAccountData) + } + + return result +} diff --git a/statediff/publisher/publisher.go b/statediff/publisher/publisher.go new file mode 100644 index 000000000..ff1925513 --- /dev/null +++ b/statediff/publisher/publisher.go @@ -0,0 +1,48 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package publisher + +import ( + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" +) + +type Publisher interface { + PublishStateDiff(sd *builder.StateDiff) (string, error) +} + +type publisher struct { + Config statediff.Config +} + +func NewPublisher(config statediff.Config) (*publisher, error) { + return &publisher{ + Config: config, + }, nil +} + +func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { + switch p.Config.Mode { + case statediff.CSV: + return p.publishStateDiffToCSV(*sd) + default: + return p.publishStateDiffToCSV(*sd) + } +} diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go new file mode 100644 index 000000000..4319b7bd7 --- /dev/null +++ b/statediff/publisher/publisher_test.go @@ -0,0 +1,315 @@ +package publisher_test + +import ( + "bytes" + "encoding/csv" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + p "github.com/ethereum/go-ethereum/statediff/publisher" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + "github.com/pkg/errors" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +var ( + tempDir = os.TempDir() + testFilePrefix = "test-statediff" + publisher p.Publisher + dir string + err error +) + +var expectedCreatedAccountRow = []string{ + strconv.FormatInt(testhelpers.BlockNumber, 10), + testhelpers.BlockHash, + "created", + testhelpers.CodeHash, + strconv.FormatUint(testhelpers.NewNonceValue, 10), + strconv.FormatInt(testhelpers.NewBalanceValue, 10), + testhelpers.ContractRoot, + testhelpers.StoragePath, + testhelpers.ContractAddress, + "0000000000000000000000000000000000000000000000000000000000000001", + testhelpers.StorageValue, +} + +var expectedCreatedAccountWithoutStorageUpdateRow = []string{ + strconv.FormatInt(testhelpers.BlockNumber, 10), + testhelpers.BlockHash, + "created", + testhelpers.CodeHash, + strconv.FormatUint(testhelpers.NewNonceValue, 10), + strconv.FormatInt(testhelpers.NewBalanceValue, 10), + testhelpers.ContractRoot, + "", + testhelpers.AnotherContractAddress, + "", + "", +} + +var expectedUpdatedAccountRow = []string{ + strconv.FormatInt(testhelpers.BlockNumber, 10), + testhelpers.BlockHash, + "updated", + testhelpers.CodeHash, + strconv.FormatUint(testhelpers.NewNonceValue, 10), + strconv.FormatInt(testhelpers.NewBalanceValue, 10), + testhelpers.ContractRoot, + testhelpers.StoragePath, + testhelpers.ContractAddress, + "0000000000000000000000000000000000000000000000000000000000000001", + testhelpers.StorageValue, +} + +var expectedDeletedAccountRow = []string{ + strconv.FormatInt(testhelpers.BlockNumber, 10), + testhelpers.BlockHash, + "deleted", + testhelpers.CodeHash, + strconv.FormatUint(testhelpers.NewNonceValue, 10), + strconv.FormatInt(testhelpers.NewBalanceValue, 10), + testhelpers.ContractRoot, + testhelpers.StoragePath, + testhelpers.ContractAddress, + "0000000000000000000000000000000000000000000000000000000000000001", + testhelpers.StorageValue, +} + +func TestPublisher(t *testing.T) { + dir, err = ioutil.TempDir(tempDir, testFilePrefix) + if err != nil { + t.Error(err) + } + config := statediff.Config{ + Path: dir, + Mode: statediff.CSV, + } + publisher, err = p.NewPublisher(config) + if err != nil { + t.Error(err) + } + + type Test func(t *testing.T) + + var tests = []Test{ + testFileName, + testColumnHeaders, + testAccountDiffs, + testWhenNoDiff, + testDefaultPublisher, + testDefaultDirectory, + } + + for _, test := range tests { + test(t) + err := removeFilesFromDir(dir) + if err != nil { + t.Errorf("Error removing files from temp dir: %s", dir) + } + } +} + +func removeFilesFromDir(dir string) error { + files, err := filepath.Glob(filepath.Join(dir, "*")) + if err != nil { + return err + } + + for _, file := range files { + err = os.RemoveAll(file) + if err != nil { + return err + } + } + return nil +} + +func testFileName(t *testing.T) { + fileName, err := publisher.PublishStateDiff(&testhelpers.TestStateDiff) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + if !strings.HasPrefix(fileName, dir) { + t.Errorf(testhelpers.TestFailureFormatString, t.Name(), dir, fileName) + } + blockNumberWithFileExt := strconv.FormatInt(testhelpers.BlockNumber, 10) + ".csv" + if !strings.HasSuffix(fileName, blockNumberWithFileExt) { + t.Errorf(testhelpers.TestFailureFormatString, t.Name(), blockNumberWithFileExt, fileName) + } +} + +func testColumnHeaders(t *testing.T) { + _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + file, err := getTestDiffFile(dir) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + lines, err := csv.NewReader(file).ReadAll() + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if len(lines) < 1 { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[0], p.Headers) { + t.Error() + } +} + +func testAccountDiffs(t *testing.T) { + // it persists the created, updated and deleted account diffs to a CSV file + _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + file, err := getTestDiffFile(dir) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + lines, err := csv.NewReader(file).ReadAll() + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if len(lines) <= 3 { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[1], expectedCreatedAccountRow) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[2], expectedCreatedAccountWithoutStorageUpdateRow) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[3], expectedUpdatedAccountRow) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[4], expectedDeletedAccountRow) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } +} + +func testWhenNoDiff(t *testing.T) { + //it creates an empty CSV when there is no diff + emptyDiff := builder.StateDiff{} + _, err = publisher.PublishStateDiff(&emptyDiff) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + file, err := getTestDiffFile(dir) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + lines, err := csv.NewReader(file).ReadAll() + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + if !equals(len(lines), 1) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } +} + +func testDefaultPublisher(t *testing.T) { + //it defaults to publishing state diffs to a CSV file when no mode is configured + config := statediff.Config{Path: dir} + publisher, err = p.NewPublisher(config) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + file, err := getTestDiffFile(dir) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + lines, err := csv.NewReader(file).ReadAll() + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(len(lines), 5) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[0], p.Headers) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } +} + +func testDefaultDirectory(t *testing.T) { + //it defaults to publishing CSV files in the current directory when no path is configured + config := statediff.Config{} + publisher, err = p.NewPublisher(config) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + err := os.Chdir(dir) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + file, err := getTestDiffFile(dir) + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + + lines, err := csv.NewReader(file).ReadAll() + if err != nil { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(len(lines), 5) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } + if !equals(lines[0], p.Headers) { + t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) + } +} + +func getTestDiffFile(dir string) (*os.File, error) { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, errors.New("There are 0 files.") + } + + fileName := files[0].Name() + filePath := filepath.Join(dir, fileName) + + return os.Open(filePath) +} + +func equals(actual, expected interface{}) (success bool) { + if actualByteSlice, ok := actual.([]byte); ok { + if expectedByteSlice, ok := expected.([]byte); ok { + return bytes.Equal(actualByteSlice, expectedByteSlice) + } + } + + return reflect.DeepEqual(actual, expected) +} diff --git a/statediff/service/service.go b/statediff/service/service.go new file mode 100644 index 000000000..2b93a1dd1 --- /dev/null +++ b/statediff/service/service.go @@ -0,0 +1,82 @@ +package service + +import ( + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + b "github.com/ethereum/go-ethereum/statediff/builder" + e "github.com/ethereum/go-ethereum/statediff/extractor" + p "github.com/ethereum/go-ethereum/statediff/publisher" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" +) + +type BlockChain interface { + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + GetBlockByHash(hash common.Hash) *types.Block +} + +type StateDiffService struct { + Builder *b.Builder + Extractor e.Extractor + BlockChain BlockChain +} + +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config statediff.Config) (*StateDiffService, error) { + builder := b.NewBuilder(db) + publisher, err := p.NewPublisher(config) + if err != nil { + return nil, err + } + + extractor := e.NewExtractor(builder, publisher) + return &StateDiffService{ + BlockChain: blockChain, + Extractor: extractor, + }, nil +} + +func (StateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +func (StateDiffService) APIs() []rpc.API { + return []rpc.API{} +} + +func (sds *StateDiffService) Loop(events chan core.ChainEvent) { + for elem := range events { + currentBlock := elem.Block + parentHash := currentBlock.ParentHash() + parentBlock := sds.BlockChain.GetBlockByHash(parentHash) + + stateDiffLocation, err := sds.Extractor.ExtractStateDiff(*parentBlock, *currentBlock) + if err != nil { + log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err) + } else { + log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation) + } + } +} + +var eventsChannel chan core.ChainEvent + +func (sds *StateDiffService) Start(server *p2p.Server) error { + log.Info("Starting statediff service") + eventsChannel := make(chan core.ChainEvent, 10) + sds.BlockChain.SubscribeChainEvent(eventsChannel) + go sds.Loop(eventsChannel) + return nil +} + +func (StateDiffService) Stop() error { + log.Info("Stopping statediff service") + close(eventsChannel) + + return nil +} diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go new file mode 100644 index 000000000..0025239af --- /dev/null +++ b/statediff/service/service_test.go @@ -0,0 +1,76 @@ +package service_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + service2 "github.com/ethereum/go-ethereum/statediff/service" + "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" + "math/big" + "math/rand" + "reflect" + "testing" +) + +func TestServiceLoop(t *testing.T) { + testServiceLoop(t) +} + +var ( + eventsChannel = make(chan core.ChainEvent, 10) + + parentHeader1 = types.Header{Number: big.NewInt(rand.Int63())} + parentHeader2 = types.Header{Number: big.NewInt(rand.Int63())} + + parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil) + parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil) + + parentHash1 = parentBlock1.Hash() + parentHash2 = parentBlock2.Hash() + + header1 = types.Header{ParentHash: parentHash1} + header2 = types.Header{ParentHash: parentHash2} + + block1 = types.NewBlock(&header1, nil, nil, nil) + block2 = types.NewBlock(&header2, nil, nil, nil) + + event1 = core.ChainEvent{Block: block1} + event2 = core.ChainEvent{Block: block2} +) + +func testServiceLoop(t *testing.T) { + eventsChannel <- event1 + eventsChannel <- event2 + + extractor := mocks.Extractor{} + close(eventsChannel) + + blockChain := mocks.BlockChain{} + service := service2.StateDiffService{ + Builder: nil, + Extractor: &extractor, + BlockChain: &blockChain, + } + + blockChain.SetParentBlockToReturn([]*types.Block{parentBlock1, parentBlock2}) + service.Loop(eventsChannel) + + //parent and current blocks are passed to the extractor + expectedCurrentBlocks := []types.Block{*block1, *block2} + if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks) + } + expectedParentBlocks := []types.Block{*parentBlock1, *parentBlock2} + if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks) + } + + //look up the parent block from its hash + expectedHashes := []common.Hash{block1.ParentHash(), block2.ParentHash()} + if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) + } +} diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go new file mode 100644 index 000000000..0bfc53e5e --- /dev/null +++ b/statediff/testhelpers/helpers.go @@ -0,0 +1,4 @@ +package testhelpers + +var ErrorFormatString = "Error: %s, %+v" +var TestFailureFormatString = "Test failed: %s\nexpected %+v, got %+v\n" diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go new file mode 100644 index 000000000..baa7b3cec --- /dev/null +++ b/statediff/testhelpers/mocks/blockchain.go @@ -0,0 +1,34 @@ +package mocks + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" +) + +type BlockChain struct { + ParentHashesLookedUp []common.Hash + parentBlocksToReturn []*types.Block + callCount int +} + +func (mc *BlockChain) SetParentBlockToReturn(blocks []*types.Block) { + mc.parentBlocksToReturn = blocks +} + +func (mc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + mc.ParentHashesLookedUp = append(mc.ParentHashesLookedUp, hash) + + var parentBlock types.Block + if len(mc.parentBlocksToReturn) > 0 { + parentBlock = *mc.parentBlocksToReturn[mc.callCount] + } + + mc.callCount++ + return &parentBlock +} + +func (BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + panic("implement me") +} diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go new file mode 100644 index 000000000..ae9ff5ced --- /dev/null +++ b/statediff/testhelpers/mocks/builder.go @@ -0,0 +1,32 @@ +package mocks + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/builder" +) + +type Builder struct { + OldStateRoot common.Hash + NewStateRoot common.Hash + BlockNumber int64 + BlockHash common.Hash + stateDiff *builder.StateDiff + builderError error +} + +func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) { + builder.OldStateRoot = oldStateRoot + builder.NewStateRoot = newStateRoot + builder.BlockNumber = blockNumber + builder.BlockHash = blockHash + + return builder.stateDiff, builder.builderError +} + +func (builder *Builder) SetStateDiffToBuild(stateDiff *builder.StateDiff) { + builder.stateDiff = stateDiff +} + +func (builder *Builder) SetBuilderError(err error) { + builder.builderError = err +} diff --git a/statediff/testhelpers/mocks/error.go b/statediff/testhelpers/mocks/error.go new file mode 100644 index 000000000..7c40452ae --- /dev/null +++ b/statediff/testhelpers/mocks/error.go @@ -0,0 +1,5 @@ +package mocks + +import "errors" + +var Error = errors.New("mock error") diff --git a/statediff/testhelpers/mocks/extractor.go b/statediff/testhelpers/mocks/extractor.go new file mode 100644 index 000000000..067497646 --- /dev/null +++ b/statediff/testhelpers/mocks/extractor.go @@ -0,0 +1,20 @@ +package mocks + +import "github.com/ethereum/go-ethereum/core/types" + +type Extractor struct { + ParentBlocks []types.Block + CurrentBlocks []types.Block + extractError error +} + +func (me *Extractor) ExtractStateDiff(parent, current types.Block) (string, error) { + me.ParentBlocks = append(me.ParentBlocks, parent) + me.CurrentBlocks = append(me.CurrentBlocks, current) + + return "", me.extractError +} + +func (me *Extractor) SetExtractError(err error) { + me.extractError = err +} diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go new file mode 100644 index 000000000..efbe4e2ab --- /dev/null +++ b/statediff/testhelpers/mocks/publisher.go @@ -0,0 +1,17 @@ +package mocks + +import "github.com/ethereum/go-ethereum/statediff/builder" + +type Publisher struct { + StateDiff *builder.StateDiff + publisherError error +} + +func (publisher *Publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { + publisher.StateDiff = sd + return "", publisher.publisherError +} + +func (publisher *Publisher) SetPublisherError(err error) { + publisher.publisherError = err +} diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go new file mode 100644 index 000000000..e7d94b373 --- /dev/null +++ b/statediff/testhelpers/test_data.go @@ -0,0 +1,69 @@ +package testhelpers + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/builder" + "math/big" + "math/rand" +) + +var ( + BlockNumber = rand.Int63() + BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" + CodeHash = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + NewNonceValue = rand.Uint64() + NewBalanceValue = rand.Int63() + ContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + StoragePath = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + StorageKey = "0000000000000000000000000000000000000000000000000000000000000001" + StorageValue = "0x03" + storage = map[string]builder.DiffStorage{StoragePath: { + Key: &StorageKey, + Value: &StorageValue, + }} + emptyStorage = map[string]builder.DiffStorage{} + address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + anotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") + ContractAddress = address.String() + AnotherContractAddress = anotherAddress.String() + CreatedAccountDiffs = map[common.Address]builder.AccountDiff{ + address: { + Nonce: builder.DiffUint64{Value: &NewNonceValue}, + Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, + ContractRoot: builder.DiffString{Value: &ContractRoot}, + CodeHash: CodeHash, + Storage: storage, + }, + anotherAddress: { + Nonce: builder.DiffUint64{Value: &NewNonceValue}, + Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, + CodeHash: CodeHash, + ContractRoot: builder.DiffString{Value: &ContractRoot}, + Storage: emptyStorage, + }, + } + + UpdatedAccountDiffs = map[common.Address]builder.AccountDiff{address: { + Nonce: builder.DiffUint64{Value: &NewNonceValue}, + Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, + CodeHash: CodeHash, + ContractRoot: builder.DiffString{Value: &ContractRoot}, + Storage: storage, + }} + + DeletedAccountDiffs = map[common.Address]builder.AccountDiff{address: { + Nonce: builder.DiffUint64{Value: &NewNonceValue}, + Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, + ContractRoot: builder.DiffString{Value: &ContractRoot}, + CodeHash: CodeHash, + Storage: storage, + }} + + TestStateDiff = builder.StateDiff{ + BlockNumber: BlockNumber, + BlockHash: common.HexToHash(BlockHash), + CreatedAccounts: CreatedAccountDiffs, + DeletedAccounts: DeletedAccountDiffs, + UpdatedAccounts: UpdatedAccountDiffs, + } +) diff --git a/trie/encoding.go b/trie/encoding.go index 1955a3e66..425326db8 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -76,9 +76,9 @@ func keybytesToHex(str []byte) []byte { return nibbles } -// hexToKeybytes turns hex nibbles into key bytes. +// hexToKeyBytes turns hex nibbles into key bytes. // This can only be used for keys of even length. -func hexToKeybytes(hex []byte) []byte { +func hexToKeyBytes(hex []byte) []byte { if hasTerm(hex) { hex = hex[:len(hex)-1] } diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 97d8da136..add89afc8 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -69,8 +69,8 @@ func TestHexKeybytes(t *testing.T) { if h := keybytesToHex(test.key); !bytes.Equal(h, test.hexOut) { t.Errorf("keybytesToHex(%x) -> %x, want %x", test.key, h, test.hexOut) } - if k := hexToKeybytes(test.hexIn); !bytes.Equal(k, test.key) { - t.Errorf("hexToKeybytes(%x) -> %x, want %x", test.hexIn, k, test.key) + if k := hexToKeyBytes(test.hexIn); !bytes.Equal(k, test.key) { + t.Errorf("hexToKeyBytes(%x) -> %x, want %x", test.hexIn, k, test.key) } } } @@ -99,6 +99,6 @@ func BenchmarkKeybytesToHex(b *testing.B) { func BenchmarkHexToKeybytes(b *testing.B) { testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16} for i := 0; i < b.N; i++ { - hexToKeybytes(testBytes) + hexToKeyBytes(testBytes) } } diff --git a/trie/iterator.go b/trie/iterator.go index 8e84dee3b..ab2ea149d 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -164,7 +164,7 @@ func (it *nodeIterator) Leaf() bool { func (it *nodeIterator) LeafKey() []byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - return hexToKeybytes(it.path) + return hexToKeyBytes(it.path) } } panic("not at leaf") diff --git a/vendor/github.com/onsi/gomega/CHANGELOG.md b/vendor/github.com/onsi/gomega/CHANGELOG.md new file mode 100644 index 000000000..9153294f7 --- /dev/null +++ b/vendor/github.com/onsi/gomega/CHANGELOG.md @@ -0,0 +1,125 @@ +## 1.4.3 + +### Fixes: + +- ensure file name and line numbers are correctly reported for XUnit [6fff58f] +- Fixed matcher for content-type (#305) [69d9b43] + +## 1.4.2 + +### Fixes: + +- Add go.mod and go.sum files to define the gomega go module [f3de367, a085d30] +- Work around go vet issue with Go v1.11 (#300) [40dd6ad] +- Better output when using with go XUnit-style tests, fixes #255 (#297) [29a4b97] +- Fix MatchJSON fail to parse json.RawMessage (#298) [ae19f1b] +- show threshold in failure message of BeNumericallyMatcher (#293) [4bbecc8] + +## 1.4.1 + +### Fixes: + +- Update documentation formatting and examples (#289) [9be8410] +- allow 'Receive' matcher to be used with concrete types (#286) [41673fd] +- Fix data race in ghttp server (#283) [7ac6b01] +- Travis badge should only show master [cc102ab] + +## 1.4.0 + +### Features +- Make string pretty diff user configurable (#273) [eb112ce, 649b44d] + +### Fixes +- Use httputil.DumpRequest to pretty-print unhandled requests (#278) [a4ff0fc, b7d1a52] +- fix typo floa32 > float32 (#272) [041ae3b, 6e33911] +- Fix link to documentation on adding your own matchers (#270) [bb2c830, fcebc62] +- Use setters and getters to avoid race condition (#262) [13057c3, a9c79f1] +- Avoid sending a signal if the process is not alive (#259) [b8043e5, 4fc1762] +- Improve message from AssignableToTypeOf when expected value is nil (#281) [9c1fb20] + +## 1.3.0 + +Improvements: + +- The `Equal` matcher matches byte slices more performantly. +- Improved how `MatchError` matches error strings. +- `MatchXML` ignores the order of xml node attributes. +- Improve support for XUnit style golang tests. ([#254](https://github.com/onsi/gomega/issues/254)) + +Bug Fixes: + +- Diff generation now handles multi-byte sequences correctly. +- Multiple goroutines can now call `gexec.Build` concurrently. + +## 1.2.0 + +Improvements: + +- Added `BeSent` which attempts to send a value down a channel and fails if the attempt blocks. Can be paired with `Eventually` to safely send a value down a channel with a timeout. +- `Ω`, `Expect`, `Eventually`, and `Consistently` now immediately `panic` if there is no registered fail handler. This is always a mistake that can hide failing tests. +- `Receive()` no longer errors when passed a closed channel, it's perfectly fine to attempt to read from a closed channel so Ω(c).Should(Receive()) always fails and Ω(c).ShoudlNot(Receive()) always passes with a closed channel. +- Added `HavePrefix` and `HaveSuffix` matchers. +- `ghttp` can now handle concurrent requests. +- Added `Succeed` which allows one to write `Ω(MyFunction()).Should(Succeed())`. +- Improved `ghttp`'s behavior around failing assertions and panics: + - If a registered handler makes a failing assertion `ghttp` will return `500`. + - If a registered handler panics, `ghttp` will return `500` *and* fail the test. This is new behavior that may cause existing code to break. This code is almost certainly incorrect and creating a false positive. +- `ghttp` servers can take an `io.Writer`. `ghttp` will write a line to the writer when each request arrives. +- Added `WithTransform` matcher to allow munging input data before feeding into the relevant matcher +- Added boolean `And`, `Or`, and `Not` matchers to allow creating composite matchers +- Added `gbytes.TimeoutCloser`, `gbytes.TimeoutReader`, and `gbytes.TimeoutWriter` - these are convenience wrappers that timeout if the underlying Closer/Reader/Writer does not return within the alloted time. +- Added `gbytes.BufferReader` - this constructs a `gbytes.Buffer` that asynchronously reads the passed-in `io.Reader` into its buffer. + +Bug Fixes: +- gexec: `session.Wait` now uses `EventuallyWithOffset` to get the right line number in the failure. +- `ContainElement` no longer bails if a passed-in matcher errors. + +## 1.0 (8/2/2014) + +No changes. Dropping "beta" from the version number. + +## 1.0.0-beta (7/8/2014) +Breaking Changes: + +- Changed OmegaMatcher interface. Instead of having `Match` return failure messages, two new methods `FailureMessage` and `NegatedFailureMessage` are called instead. +- Moved and renamed OmegaFailHandler to types.GomegaFailHandler and OmegaMatcher to types.GomegaMatcher. Any references to OmegaMatcher in any custom matchers will need to be changed to point to types.GomegaMatcher + +New Test-Support Features: + +- `ghttp`: supports testing http clients + - Provides a flexible fake http server + - Provides a collection of chainable http handlers that perform assertions. +- `gbytes`: supports making ordered assertions against streams of data + - Provides a `gbytes.Buffer` + - Provides a `Say` matcher to perform ordered assertions against output data +- `gexec`: supports testing external processes + - Provides support for building Go binaries + - Wraps and starts `exec.Cmd` commands + - Makes it easy to assert against stdout and stderr + - Makes it easy to send signals and wait for processes to exit + - Provides an `Exit` matcher to assert against exit code. + +DSL Changes: + +- `Eventually` and `Consistently` can accept `time.Duration` interval and polling inputs. +- The default timeouts for `Eventually` and `Consistently` are now configurable. + +New Matchers: + +- `ConsistOf`: order-independent assertion against the elements of an array/slice or keys of a map. +- `BeTemporally`: like `BeNumerically` but for `time.Time` +- `HaveKeyWithValue`: asserts a map has a given key with the given value. + +Updated Matchers: + +- `Receive` matcher can take a matcher as an argument and passes only if the channel under test receives an objet that satisfies the passed-in matcher. +- Matchers that implement `MatchMayChangeInTheFuture(actual interface{}) bool` can inform `Eventually` and/or `Consistently` when a match has no chance of changing status in the future. For example, `Receive` returns `false` when a channel is closed. + +Misc: + +- Start using semantic versioning +- Start maintaining changelog + +Major refactor: + +- Pull out Gomega's internal to `internal` diff --git a/vendor/github.com/onsi/gomega/CONTRIBUTING.md b/vendor/github.com/onsi/gomega/CONTRIBUTING.md new file mode 100644 index 000000000..0d7a09928 --- /dev/null +++ b/vendor/github.com/onsi/gomega/CONTRIBUTING.md @@ -0,0 +1,14 @@ +# Contributing to Gomega + +Your contributions to Gomega are essential for its long-term maintenance and improvement. To make a contribution: + +- Please **open an issue first** - describe what problem you are trying to solve and give the community a forum for input and feedback ahead of investing time in writing code! +- Ensure adequate test coverage: + - Make sure to add appropriate unit tests + - Please run all tests locally (`ginkgo -r -p`) and make sure they go green before submitting the PR + - Please run following linter locally `go vet ./...` and make sure output does not contain any warnings +- Update the documentation. In addition to standard `godoc` comments Gomega has extensive documentation on the `gh-pages` branch. If relevant, please submit a docs PR to that branch alongside your code PR. + +If you're a committer, check out RELEASING.md to learn how to cut a release. + +Thanks for supporting Gomega! diff --git a/vendor/github.com/onsi/gomega/LICENSE b/vendor/github.com/onsi/gomega/LICENSE new file mode 100644 index 000000000..9415ee72c --- /dev/null +++ b/vendor/github.com/onsi/gomega/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2013-2014 Onsi Fakhouri + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/onsi/gomega/Makefile b/vendor/github.com/onsi/gomega/Makefile new file mode 100644 index 000000000..c92cd56e3 --- /dev/null +++ b/vendor/github.com/onsi/gomega/Makefile @@ -0,0 +1,6 @@ +test: + [ -z "`gofmt -s -w -l -e .`" ] + go vet + ginkgo -p -r --randomizeAllSpecs --failOnPending --randomizeSuites --race + +.PHONY: test diff --git a/vendor/github.com/onsi/gomega/README.md b/vendor/github.com/onsi/gomega/README.md new file mode 100644 index 000000000..76aa6b558 --- /dev/null +++ b/vendor/github.com/onsi/gomega/README.md @@ -0,0 +1,21 @@ +![Gomega: Ginkgo's Preferred Matcher Library](http://onsi.github.io/gomega/images/gomega.png) + +[![Build Status](https://travis-ci.org/onsi/gomega.svg?branch=master)](https://travis-ci.org/onsi/gomega) + +Jump straight to the [docs](http://onsi.github.io/gomega/) to learn about Gomega, including a list of [all available matchers](http://onsi.github.io/gomega/#provided-matchers). + +If you have a question, comment, bug report, feature request, etc. please open a GitHub issue. + +## [Ginkgo](http://github.com/onsi/ginkgo): a BDD Testing Framework for Golang + +Learn more about Ginkgo [here](http://onsi.github.io/ginkgo/) + +## Community Matchers + +A collection of community matchers is available on the [wiki](https://github.com/onsi/gomega/wiki). + +## License + +Gomega is MIT-Licensed + +The `ConsistOf` matcher uses [goraph](https://github.com/amitkgupta/goraph) which is embedded in the source to simplify distribution. goraph has an MIT license. diff --git a/vendor/github.com/onsi/gomega/RELEASING.md b/vendor/github.com/onsi/gomega/RELEASING.md new file mode 100644 index 000000000..998d64ee7 --- /dev/null +++ b/vendor/github.com/onsi/gomega/RELEASING.md @@ -0,0 +1,12 @@ +A Gomega release is a tagged sha and a GitHub release. To cut a release: + +1. Ensure CHANGELOG.md is up to date. + - Use `git log --pretty=format:'- %s [%h]' HEAD...vX.X.X` to list all the commits since the last release + - Categorize the changes into + - Breaking Changes (requires a major version) + - New Features (minor version) + - Fixes (fix version) + - Maintenance (which in general should not be mentioned in `CHANGELOG.md` as they have no user impact) +2. Update GOMEGA_VERSION in `gomega_dsl.go` +3. Push a commit with the version number as the commit message (e.g. `v1.3.0`) +4. Create a new [GitHub release](https://help.github.com/articles/creating-releases/) with the version number as the tag (e.g. `v1.3.0`). List the key changes in the release notes. diff --git a/vendor/github.com/onsi/gomega/format/format.go b/vendor/github.com/onsi/gomega/format/format.go new file mode 100644 index 000000000..6559525f1 --- /dev/null +++ b/vendor/github.com/onsi/gomega/format/format.go @@ -0,0 +1,382 @@ +/* +Gomega's format package pretty-prints objects. It explores input objects recursively and generates formatted, indented output with type information. +*/ +package format + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" +) + +// Use MaxDepth to set the maximum recursion depth when printing deeply nested objects +var MaxDepth = uint(10) + +/* +By default, all objects (even those that implement fmt.Stringer and fmt.GoStringer) are recursively inspected to generate output. + +Set UseStringerRepresentation = true to use GoString (for fmt.GoStringers) or String (for fmt.Stringer) instead. + +Note that GoString and String don't always have all the information you need to understand why a test failed! +*/ +var UseStringerRepresentation = false + +/* +Print the content of context objects. By default it will be suppressed. + +Set PrintContextObjects = true to enable printing of the context internals. +*/ +var PrintContextObjects = false + +// TruncatedDiff choose if we should display a truncated pretty diff or not +var TruncatedDiff = true + +// Ctx interface defined here to keep backwards compatability with go < 1.7 +// It matches the context.Context interface +type Ctx interface { + Deadline() (deadline time.Time, ok bool) + Done() <-chan struct{} + Err() error + Value(key interface{}) interface{} +} + +var contextType = reflect.TypeOf((*Ctx)(nil)).Elem() +var timeType = reflect.TypeOf(time.Time{}) + +//The default indentation string emitted by the format package +var Indent = " " + +var longFormThreshold = 20 + +/* +Generates a formatted matcher success/failure message of the form: + + Expected + + + + +If expected is omited, then the message looks like: + + Expected + + +*/ +func Message(actual interface{}, message string, expected ...interface{}) string { + if len(expected) == 0 { + return fmt.Sprintf("Expected\n%s\n%s", Object(actual, 1), message) + } + return fmt.Sprintf("Expected\n%s\n%s\n%s", Object(actual, 1), message, Object(expected[0], 1)) +} + +/* + +Generates a nicely formatted matcher success / failure message + +Much like Message(...), but it attempts to pretty print diffs in strings + +Expected + : "...aaaaabaaaaa..." +to equal | + : "...aaaaazaaaaa..." + +*/ + +func MessageWithDiff(actual, message, expected string) string { + if TruncatedDiff && len(actual) >= truncateThreshold && len(expected) >= truncateThreshold { + diffPoint := findFirstMismatch(actual, expected) + formattedActual := truncateAndFormat(actual, diffPoint) + formattedExpected := truncateAndFormat(expected, diffPoint) + + spacesBeforeFormattedMismatch := findFirstMismatch(formattedActual, formattedExpected) + + tabLength := 4 + spaceFromMessageToActual := tabLength + len(": ") - len(message) + padding := strings.Repeat(" ", spaceFromMessageToActual+spacesBeforeFormattedMismatch) + "|" + return Message(formattedActual, message+padding, formattedExpected) + } + return Message(actual, message, expected) +} + +func truncateAndFormat(str string, index int) string { + leftPadding := `...` + rightPadding := `...` + + start := index - charactersAroundMismatchToInclude + if start < 0 { + start = 0 + leftPadding = "" + } + + // slice index must include the mis-matched character + lengthOfMismatchedCharacter := 1 + end := index + charactersAroundMismatchToInclude + lengthOfMismatchedCharacter + if end > len(str) { + end = len(str) + rightPadding = "" + + } + return fmt.Sprintf("\"%s\"", leftPadding+str[start:end]+rightPadding) +} + +func findFirstMismatch(a, b string) int { + aSlice := strings.Split(a, "") + bSlice := strings.Split(b, "") + + for index, str := range aSlice { + if index > len(bSlice)-1 { + return index + } + if str != bSlice[index] { + return index + } + } + + if len(b) > len(a) { + return len(a) + 1 + } + + return 0 +} + +const ( + truncateThreshold = 50 + charactersAroundMismatchToInclude = 5 +) + +/* +Pretty prints the passed in object at the passed in indentation level. + +Object recurses into deeply nested objects emitting pretty-printed representations of their components. + +Modify format.MaxDepth to control how deep the recursion is allowed to go +Set format.UseStringerRepresentation to true to return object.GoString() or object.String() when available instead of +recursing into the object. + +Set PrintContextObjects to true to print the content of objects implementing context.Context +*/ +func Object(object interface{}, indentation uint) string { + indent := strings.Repeat(Indent, int(indentation)) + value := reflect.ValueOf(object) + return fmt.Sprintf("%s<%s>: %s", indent, formatType(object), formatValue(value, indentation)) +} + +/* +IndentString takes a string and indents each line by the specified amount. +*/ +func IndentString(s string, indentation uint) string { + components := strings.Split(s, "\n") + result := "" + indent := strings.Repeat(Indent, int(indentation)) + for i, component := range components { + result += indent + component + if i < len(components)-1 { + result += "\n" + } + } + + return result +} + +func formatType(object interface{}) string { + t := reflect.TypeOf(object) + if t == nil { + return "nil" + } + switch t.Kind() { + case reflect.Chan: + v := reflect.ValueOf(object) + return fmt.Sprintf("%T | len:%d, cap:%d", object, v.Len(), v.Cap()) + case reflect.Ptr: + return fmt.Sprintf("%T | %p", object, object) + case reflect.Slice: + v := reflect.ValueOf(object) + return fmt.Sprintf("%T | len:%d, cap:%d", object, v.Len(), v.Cap()) + case reflect.Map: + v := reflect.ValueOf(object) + return fmt.Sprintf("%T | len:%d", object, v.Len()) + default: + return fmt.Sprintf("%T", object) + } +} + +func formatValue(value reflect.Value, indentation uint) string { + if indentation > MaxDepth { + return "..." + } + + if isNilValue(value) { + return "nil" + } + + if UseStringerRepresentation { + if value.CanInterface() { + obj := value.Interface() + switch x := obj.(type) { + case fmt.GoStringer: + return x.GoString() + case fmt.Stringer: + return x.String() + } + } + } + + if !PrintContextObjects { + if value.Type().Implements(contextType) && indentation > 1 { + return "" + } + } + + switch value.Kind() { + case reflect.Bool: + return fmt.Sprintf("%v", value.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return fmt.Sprintf("%v", value.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return fmt.Sprintf("%v", value.Uint()) + case reflect.Uintptr: + return fmt.Sprintf("0x%x", value.Uint()) + case reflect.Float32, reflect.Float64: + return fmt.Sprintf("%v", value.Float()) + case reflect.Complex64, reflect.Complex128: + return fmt.Sprintf("%v", value.Complex()) + case reflect.Chan: + return fmt.Sprintf("0x%x", value.Pointer()) + case reflect.Func: + return fmt.Sprintf("0x%x", value.Pointer()) + case reflect.Ptr: + return formatValue(value.Elem(), indentation) + case reflect.Slice: + return formatSlice(value, indentation) + case reflect.String: + return formatString(value.String(), indentation) + case reflect.Array: + return formatSlice(value, indentation) + case reflect.Map: + return formatMap(value, indentation) + case reflect.Struct: + if value.Type() == timeType && value.CanInterface() { + t, _ := value.Interface().(time.Time) + return t.Format(time.RFC3339Nano) + } + return formatStruct(value, indentation) + case reflect.Interface: + return formatValue(value.Elem(), indentation) + default: + if value.CanInterface() { + return fmt.Sprintf("%#v", value.Interface()) + } + return fmt.Sprintf("%#v", value) + } +} + +func formatString(object interface{}, indentation uint) string { + if indentation == 1 { + s := fmt.Sprintf("%s", object) + components := strings.Split(s, "\n") + result := "" + for i, component := range components { + if i == 0 { + result += component + } else { + result += Indent + component + } + if i < len(components)-1 { + result += "\n" + } + } + + return fmt.Sprintf("%s", result) + } else { + return fmt.Sprintf("%q", object) + } +} + +func formatSlice(v reflect.Value, indentation uint) string { + if v.Kind() == reflect.Slice && v.Type().Elem().Kind() == reflect.Uint8 && isPrintableString(string(v.Bytes())) { + return formatString(v.Bytes(), indentation) + } + + l := v.Len() + result := make([]string, l) + longest := 0 + for i := 0; i < l; i++ { + result[i] = formatValue(v.Index(i), indentation+1) + if len(result[i]) > longest { + longest = len(result[i]) + } + } + + if longest > longFormThreshold { + indenter := strings.Repeat(Indent, int(indentation)) + return fmt.Sprintf("[\n%s%s,\n%s]", indenter+Indent, strings.Join(result, ",\n"+indenter+Indent), indenter) + } + return fmt.Sprintf("[%s]", strings.Join(result, ", ")) +} + +func formatMap(v reflect.Value, indentation uint) string { + l := v.Len() + result := make([]string, l) + + longest := 0 + for i, key := range v.MapKeys() { + value := v.MapIndex(key) + result[i] = fmt.Sprintf("%s: %s", formatValue(key, indentation+1), formatValue(value, indentation+1)) + if len(result[i]) > longest { + longest = len(result[i]) + } + } + + if longest > longFormThreshold { + indenter := strings.Repeat(Indent, int(indentation)) + return fmt.Sprintf("{\n%s%s,\n%s}", indenter+Indent, strings.Join(result, ",\n"+indenter+Indent), indenter) + } + return fmt.Sprintf("{%s}", strings.Join(result, ", ")) +} + +func formatStruct(v reflect.Value, indentation uint) string { + t := v.Type() + + l := v.NumField() + result := []string{} + longest := 0 + for i := 0; i < l; i++ { + structField := t.Field(i) + fieldEntry := v.Field(i) + representation := fmt.Sprintf("%s: %s", structField.Name, formatValue(fieldEntry, indentation+1)) + result = append(result, representation) + if len(representation) > longest { + longest = len(representation) + } + } + if longest > longFormThreshold { + indenter := strings.Repeat(Indent, int(indentation)) + return fmt.Sprintf("{\n%s%s,\n%s}", indenter+Indent, strings.Join(result, ",\n"+indenter+Indent), indenter) + } + return fmt.Sprintf("{%s}", strings.Join(result, ", ")) +} + +func isNilValue(a reflect.Value) bool { + switch a.Kind() { + case reflect.Invalid: + return true + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return a.IsNil() + } + + return false +} + +/* +Returns true when the string is entirely made of printable runes, false otherwise. +*/ +func isPrintableString(str string) bool { + for _, runeValue := range str { + if !strconv.IsPrint(runeValue) { + return false + } + } + return true +} diff --git a/vendor/github.com/onsi/gomega/go.mod b/vendor/github.com/onsi/gomega/go.mod new file mode 100644 index 000000000..65eedf696 --- /dev/null +++ b/vendor/github.com/onsi/gomega/go.mod @@ -0,0 +1,15 @@ +module github.com/onsi/gomega + +require ( + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/golang/protobuf v1.2.0 + github.com/hpcloud/tail v1.0.0 // indirect + github.com/onsi/ginkgo v1.6.0 + golang.org/x/net v0.0.0-20180906233101-161cd47e91fd + golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect + golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect + golang.org/x/text v0.3.0 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.2.1 +) diff --git a/vendor/github.com/onsi/gomega/go.sum b/vendor/github.com/onsi/gomega/go.sum new file mode 100644 index 000000000..b23f6ef02 --- /dev/null +++ b/vendor/github.com/onsi/gomega/go.sum @@ -0,0 +1,24 @@ +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/onsi/gomega/gomega_dsl.go b/vendor/github.com/onsi/gomega/gomega_dsl.go new file mode 100644 index 000000000..9456fbc1e --- /dev/null +++ b/vendor/github.com/onsi/gomega/gomega_dsl.go @@ -0,0 +1,429 @@ +/* +Gomega is the Ginkgo BDD-style testing framework's preferred matcher library. + +The godoc documentation describes Gomega's API. More comprehensive documentation (with examples!) is available at http://onsi.github.io/gomega/ + +Gomega on Github: http://github.com/onsi/gomega + +Learn more about Ginkgo online: http://onsi.github.io/ginkgo + +Ginkgo on Github: http://github.com/onsi/ginkgo + +Gomega is MIT-Licensed +*/ +package gomega + +import ( + "fmt" + "reflect" + "time" + + "github.com/onsi/gomega/internal/assertion" + "github.com/onsi/gomega/internal/asyncassertion" + "github.com/onsi/gomega/internal/testingtsupport" + "github.com/onsi/gomega/types" +) + +const GOMEGA_VERSION = "1.4.3" + +const nilFailHandlerPanic = `You are trying to make an assertion, but Gomega's fail handler is nil. +If you're using Ginkgo then you probably forgot to put your assertion in an It(). +Alternatively, you may have forgotten to register a fail handler with RegisterFailHandler() or RegisterTestingT(). +Depending on your vendoring solution you may be inadvertently importing gomega and subpackages (e.g. ghhtp, gexec,...) from different locations. +` + +var globalFailWrapper *types.GomegaFailWrapper + +var defaultEventuallyTimeout = time.Second +var defaultEventuallyPollingInterval = 10 * time.Millisecond +var defaultConsistentlyDuration = 100 * time.Millisecond +var defaultConsistentlyPollingInterval = 10 * time.Millisecond + +// RegisterFailHandler connects Ginkgo to Gomega. When a matcher fails +// the fail handler passed into RegisterFailHandler is called. +func RegisterFailHandler(handler types.GomegaFailHandler) { + RegisterFailHandlerWithT(testingtsupport.EmptyTWithHelper{}, handler) +} + +// RegisterFailHandlerWithT ensures that the given types.TWithHelper and fail handler +// are used globally. +func RegisterFailHandlerWithT(t types.TWithHelper, handler types.GomegaFailHandler) { + if handler == nil { + globalFailWrapper = nil + return + } + + globalFailWrapper = &types.GomegaFailWrapper{ + Fail: handler, + TWithHelper: t, + } +} + +// RegisterTestingT connects Gomega to Golang's XUnit style +// Testing.T tests. It is now deprecated and you should use NewWithT() instead. +// +// Legacy Documentation: +// +// You'll need to call this at the top of each XUnit style test: +// +// func TestFarmHasCow(t *testing.T) { +// RegisterTestingT(t) +// +// f := farm.New([]string{"Cow", "Horse"}) +// Expect(f.HasCow()).To(BeTrue(), "Farm should have cow") +// } +// +// Note that this *testing.T is registered *globally* by Gomega (this is why you don't have to +// pass `t` down to the matcher itself). This means that you cannot run the XUnit style tests +// in parallel as the global fail handler cannot point to more than one testing.T at a time. +// +// NewWithT() does not have this limitation +// +// (As an aside: Ginkgo gets around this limitation by running parallel tests in different *processes*). +func RegisterTestingT(t types.GomegaTestingT) { + tWithHelper, hasHelper := t.(types.TWithHelper) + if !hasHelper { + RegisterFailHandler(testingtsupport.BuildTestingTGomegaFailWrapper(t).Fail) + return + } + RegisterFailHandlerWithT(tWithHelper, testingtsupport.BuildTestingTGomegaFailWrapper(t).Fail) +} + +// InterceptGomegaFailures runs a given callback and returns an array of +// failure messages generated by any Gomega assertions within the callback. +// +// This is accomplished by temporarily replacing the *global* fail handler +// with a fail handler that simply annotates failures. The original fail handler +// is reset when InterceptGomegaFailures returns. +// +// This is most useful when testing custom matchers, but can also be used to check +// on a value using a Gomega assertion without causing a test failure. +func InterceptGomegaFailures(f func()) []string { + originalHandler := globalFailWrapper.Fail + failures := []string{} + RegisterFailHandler(func(message string, callerSkip ...int) { + failures = append(failures, message) + }) + f() + RegisterFailHandler(originalHandler) + return failures +} + +// Ω wraps an actual value allowing assertions to be made on it: +// Ω("foo").Should(Equal("foo")) +// +// If Ω is passed more than one argument it will pass the *first* argument to the matcher. +// All subsequent arguments will be required to be nil/zero. +// +// This is convenient if you want to make an assertion on a method/function that returns +// a value and an error - a common patter in Go. +// +// For example, given a function with signature: +// func MyAmazingThing() (int, error) +// +// Then: +// Ω(MyAmazingThing()).Should(Equal(3)) +// Will succeed only if `MyAmazingThing()` returns `(3, nil)` +// +// Ω and Expect are identical +func Ω(actual interface{}, extra ...interface{}) Assertion { + return ExpectWithOffset(0, actual, extra...) +} + +// Expect wraps an actual value allowing assertions to be made on it: +// Expect("foo").To(Equal("foo")) +// +// If Expect is passed more than one argument it will pass the *first* argument to the matcher. +// All subsequent arguments will be required to be nil/zero. +// +// This is convenient if you want to make an assertion on a method/function that returns +// a value and an error - a common patter in Go. +// +// For example, given a function with signature: +// func MyAmazingThing() (int, error) +// +// Then: +// Expect(MyAmazingThing()).Should(Equal(3)) +// Will succeed only if `MyAmazingThing()` returns `(3, nil)` +// +// Expect and Ω are identical +func Expect(actual interface{}, extra ...interface{}) Assertion { + return ExpectWithOffset(0, actual, extra...) +} + +// ExpectWithOffset wraps an actual value allowing assertions to be made on it: +// ExpectWithOffset(1, "foo").To(Equal("foo")) +// +// Unlike `Expect` and `Ω`, `ExpectWithOffset` takes an additional integer argument +// this is used to modify the call-stack offset when computing line numbers. +// +// This is most useful in helper functions that make assertions. If you want Gomega's +// error message to refer to the calling line in the test (as opposed to the line in the helper function) +// set the first argument of `ExpectWithOffset` appropriately. +func ExpectWithOffset(offset int, actual interface{}, extra ...interface{}) Assertion { + if globalFailWrapper == nil { + panic(nilFailHandlerPanic) + } + return assertion.New(actual, globalFailWrapper, offset, extra...) +} + +// Eventually wraps an actual value allowing assertions to be made on it. +// The assertion is tried periodically until it passes or a timeout occurs. +// +// Both the timeout and polling interval are configurable as optional arguments: +// The first optional argument is the timeout +// The second optional argument is the polling interval +// +// Both intervals can either be specified as time.Duration, parsable duration strings or as floats/integers. In the +// last case they are interpreted as seconds. +// +// If Eventually is passed an actual that is a function taking no arguments and returning at least one value, +// then Eventually will call the function periodically and try the matcher against the function's first return value. +// +// Example: +// +// Eventually(func() int { +// return thingImPolling.Count() +// }).Should(BeNumerically(">=", 17)) +// +// Note that this example could be rewritten: +// +// Eventually(thingImPolling.Count).Should(BeNumerically(">=", 17)) +// +// If the function returns more than one value, then Eventually will pass the first value to the matcher and +// assert that all other values are nil/zero. +// This allows you to pass Eventually a function that returns a value and an error - a common pattern in Go. +// +// For example, consider a method that returns a value and an error: +// func FetchFromDB() (string, error) +// +// Then +// Eventually(FetchFromDB).Should(Equal("hasselhoff")) +// +// Will pass only if the the returned error is nil and the returned string passes the matcher. +// +// Eventually's default timeout is 1 second, and its default polling interval is 10ms +func Eventually(actual interface{}, intervals ...interface{}) AsyncAssertion { + return EventuallyWithOffset(0, actual, intervals...) +} + +// EventuallyWithOffset operates like Eventually but takes an additional +// initial argument to indicate an offset in the call stack. This is useful when building helper +// functions that contain matchers. To learn more, read about `ExpectWithOffset`. +func EventuallyWithOffset(offset int, actual interface{}, intervals ...interface{}) AsyncAssertion { + if globalFailWrapper == nil { + panic(nilFailHandlerPanic) + } + timeoutInterval := defaultEventuallyTimeout + pollingInterval := defaultEventuallyPollingInterval + if len(intervals) > 0 { + timeoutInterval = toDuration(intervals[0]) + } + if len(intervals) > 1 { + pollingInterval = toDuration(intervals[1]) + } + return asyncassertion.New(asyncassertion.AsyncAssertionTypeEventually, actual, globalFailWrapper, timeoutInterval, pollingInterval, offset) +} + +// Consistently wraps an actual value allowing assertions to be made on it. +// The assertion is tried periodically and is required to pass for a period of time. +// +// Both the total time and polling interval are configurable as optional arguments: +// The first optional argument is the duration that Consistently will run for +// The second optional argument is the polling interval +// +// Both intervals can either be specified as time.Duration, parsable duration strings or as floats/integers. In the +// last case they are interpreted as seconds. +// +// If Consistently is passed an actual that is a function taking no arguments and returning at least one value, +// then Consistently will call the function periodically and try the matcher against the function's first return value. +// +// If the function returns more than one value, then Consistently will pass the first value to the matcher and +// assert that all other values are nil/zero. +// This allows you to pass Consistently a function that returns a value and an error - a common pattern in Go. +// +// Consistently is useful in cases where you want to assert that something *does not happen* over a period of tiem. +// For example, you want to assert that a goroutine does *not* send data down a channel. In this case, you could: +// +// Consistently(channel).ShouldNot(Receive()) +// +// Consistently's default duration is 100ms, and its default polling interval is 10ms +func Consistently(actual interface{}, intervals ...interface{}) AsyncAssertion { + return ConsistentlyWithOffset(0, actual, intervals...) +} + +// ConsistentlyWithOffset operates like Consistnetly but takes an additional +// initial argument to indicate an offset in the call stack. This is useful when building helper +// functions that contain matchers. To learn more, read about `ExpectWithOffset`. +func ConsistentlyWithOffset(offset int, actual interface{}, intervals ...interface{}) AsyncAssertion { + if globalFailWrapper == nil { + panic(nilFailHandlerPanic) + } + timeoutInterval := defaultConsistentlyDuration + pollingInterval := defaultConsistentlyPollingInterval + if len(intervals) > 0 { + timeoutInterval = toDuration(intervals[0]) + } + if len(intervals) > 1 { + pollingInterval = toDuration(intervals[1]) + } + return asyncassertion.New(asyncassertion.AsyncAssertionTypeConsistently, actual, globalFailWrapper, timeoutInterval, pollingInterval, offset) +} + +// SetDefaultEventuallyTimeout sets the default timeout duration for Eventually. Eventually will repeatedly poll your condition until it succeeds, or until this timeout elapses. +func SetDefaultEventuallyTimeout(t time.Duration) { + defaultEventuallyTimeout = t +} + +// SetDefaultEventuallyPollingInterval sets the default polling interval for Eventually. +func SetDefaultEventuallyPollingInterval(t time.Duration) { + defaultEventuallyPollingInterval = t +} + +// SetDefaultConsistentlyDuration sets the default duration for Consistently. Consistently will verify that your condition is satsified for this long. +func SetDefaultConsistentlyDuration(t time.Duration) { + defaultConsistentlyDuration = t +} + +// SetDefaultConsistentlyPollingInterval sets the default polling interval for Consistently. +func SetDefaultConsistentlyPollingInterval(t time.Duration) { + defaultConsistentlyPollingInterval = t +} + +// AsyncAssertion is returned by Eventually and Consistently and polls the actual value passed into Eventually against +// the matcher passed to the Should and ShouldNot methods. +// +// Both Should and ShouldNot take a variadic optionalDescription argument. This is passed on to +// fmt.Sprintf() and is used to annotate failure messages. This allows you to make your failure messages more +// descriptive. +// +// Both Should and ShouldNot return a boolean that is true if the assertion passed and false if it failed. +// +// Example: +// +// Eventually(myChannel).Should(Receive(), "Something should have come down the pipe.") +// Consistently(myChannel).ShouldNot(Receive(), "Nothing should have come down the pipe.") +type AsyncAssertion interface { + Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool + ShouldNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool +} + +// GomegaAsyncAssertion is deprecated in favor of AsyncAssertion, which does not stutter. +type GomegaAsyncAssertion = AsyncAssertion + +// Assertion is returned by Ω and Expect and compares the actual value to the matcher +// passed to the Should/ShouldNot and To/ToNot/NotTo methods. +// +// Typically Should/ShouldNot are used with Ω and To/ToNot/NotTo are used with Expect +// though this is not enforced. +// +// All methods take a variadic optionalDescription argument. This is passed on to fmt.Sprintf() +// and is used to annotate failure messages. +// +// All methods return a bool that is true if hte assertion passed and false if it failed. +// +// Example: +// +// Ω(farm.HasCow()).Should(BeTrue(), "Farm %v should have a cow", farm) +type Assertion interface { + Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool + ShouldNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool + + To(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool + ToNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool + NotTo(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool +} + +// GomegaAssertion is deprecated in favor of Assertion, which does not stutter. +type GomegaAssertion = Assertion + +// OmegaMatcher is deprecated in favor of the better-named and better-organized types.GomegaMatcher but sticks around to support existing code that uses it +type OmegaMatcher types.GomegaMatcher + +// WithT wraps a *testing.T and provides `Expect`, `Eventually`, and `Consistently` methods. This allows you to leverage +// Gomega's rich ecosystem of matchers in standard `testing` test suites. +// +// Use `NewWithT` to instantiate a `WithT` +type WithT struct { + t types.GomegaTestingT +} + +// GomegaWithT is deprecated in favor of gomega.WithT, which does not stutter. +type GomegaWithT = WithT + +// NewWithT takes a *testing.T and returngs a `gomega.WithT` allowing you to use `Expect`, `Eventually`, and `Consistently` along with +// Gomega's rich ecosystem of matchers in standard `testing` test suits. +// +// func TestFarmHasCow(t *testing.T) { +// g := gomega.NewWithT(t) +// +// f := farm.New([]string{"Cow", "Horse"}) +// g.Expect(f.HasCow()).To(BeTrue(), "Farm should have cow") +// } +func NewWithT(t types.GomegaTestingT) *WithT { + return &WithT{ + t: t, + } +} + +// NewGomegaWithT is deprecated in favor of gomega.NewWithT, which does not stutter. +func NewGomegaWithT(t types.GomegaTestingT) *GomegaWithT { + return NewWithT(t) +} + +// Expect is used to make assertions. See documentation for Expect. +func (g *WithT) Expect(actual interface{}, extra ...interface{}) Assertion { + return assertion.New(actual, testingtsupport.BuildTestingTGomegaFailWrapper(g.t), 0, extra...) +} + +// Eventually is used to make asynchronous assertions. See documentation for Eventually. +func (g *WithT) Eventually(actual interface{}, intervals ...interface{}) AsyncAssertion { + timeoutInterval := defaultEventuallyTimeout + pollingInterval := defaultEventuallyPollingInterval + if len(intervals) > 0 { + timeoutInterval = toDuration(intervals[0]) + } + if len(intervals) > 1 { + pollingInterval = toDuration(intervals[1]) + } + return asyncassertion.New(asyncassertion.AsyncAssertionTypeEventually, actual, testingtsupport.BuildTestingTGomegaFailWrapper(g.t), timeoutInterval, pollingInterval, 0) +} + +// Consistently is used to make asynchronous assertions. See documentation for Consistently. +func (g *WithT) Consistently(actual interface{}, intervals ...interface{}) AsyncAssertion { + timeoutInterval := defaultConsistentlyDuration + pollingInterval := defaultConsistentlyPollingInterval + if len(intervals) > 0 { + timeoutInterval = toDuration(intervals[0]) + } + if len(intervals) > 1 { + pollingInterval = toDuration(intervals[1]) + } + return asyncassertion.New(asyncassertion.AsyncAssertionTypeConsistently, actual, testingtsupport.BuildTestingTGomegaFailWrapper(g.t), timeoutInterval, pollingInterval, 0) +} + +func toDuration(input interface{}) time.Duration { + duration, ok := input.(time.Duration) + if ok { + return duration + } + + value := reflect.ValueOf(input) + kind := reflect.TypeOf(input).Kind() + + if reflect.Int <= kind && kind <= reflect.Int64 { + return time.Duration(value.Int()) * time.Second + } else if reflect.Uint <= kind && kind <= reflect.Uint64 { + return time.Duration(value.Uint()) * time.Second + } else if reflect.Float32 <= kind && kind <= reflect.Float64 { + return time.Duration(value.Float() * float64(time.Second)) + } else if reflect.String == kind { + duration, err := time.ParseDuration(value.String()) + if err != nil { + panic(fmt.Sprintf("%#v is not a valid parsable duration string.", input)) + } + return duration + } + + panic(fmt.Sprintf("%v is not a valid interval. Must be time.Duration, parsable duration string or a number.", input)) +} diff --git a/vendor/github.com/onsi/gomega/internal/assertion/assertion.go b/vendor/github.com/onsi/gomega/internal/assertion/assertion.go new file mode 100644 index 000000000..00197b67a --- /dev/null +++ b/vendor/github.com/onsi/gomega/internal/assertion/assertion.go @@ -0,0 +1,105 @@ +package assertion + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/types" +) + +type Assertion struct { + actualInput interface{} + failWrapper *types.GomegaFailWrapper + offset int + extra []interface{} +} + +func New(actualInput interface{}, failWrapper *types.GomegaFailWrapper, offset int, extra ...interface{}) *Assertion { + return &Assertion{ + actualInput: actualInput, + failWrapper: failWrapper, + offset: offset, + extra: extra, + } +} + +func (assertion *Assertion) Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.vetExtras(optionalDescription...) && assertion.match(matcher, true, optionalDescription...) +} + +func (assertion *Assertion) ShouldNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.vetExtras(optionalDescription...) && assertion.match(matcher, false, optionalDescription...) +} + +func (assertion *Assertion) To(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.vetExtras(optionalDescription...) && assertion.match(matcher, true, optionalDescription...) +} + +func (assertion *Assertion) ToNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.vetExtras(optionalDescription...) && assertion.match(matcher, false, optionalDescription...) +} + +func (assertion *Assertion) NotTo(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.vetExtras(optionalDescription...) && assertion.match(matcher, false, optionalDescription...) +} + +func (assertion *Assertion) buildDescription(optionalDescription ...interface{}) string { + switch len(optionalDescription) { + case 0: + return "" + default: + return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...) + "\n" + } +} + +func (assertion *Assertion) match(matcher types.GomegaMatcher, desiredMatch bool, optionalDescription ...interface{}) bool { + matches, err := matcher.Match(assertion.actualInput) + description := assertion.buildDescription(optionalDescription...) + assertion.failWrapper.TWithHelper.Helper() + if err != nil { + assertion.failWrapper.Fail(description+err.Error(), 2+assertion.offset) + return false + } + if matches != desiredMatch { + var message string + if desiredMatch { + message = matcher.FailureMessage(assertion.actualInput) + } else { + message = matcher.NegatedFailureMessage(assertion.actualInput) + } + assertion.failWrapper.Fail(description+message, 2+assertion.offset) + return false + } + + return true +} + +func (assertion *Assertion) vetExtras(optionalDescription ...interface{}) bool { + success, message := vetExtras(assertion.extra) + if success { + return true + } + + description := assertion.buildDescription(optionalDescription...) + assertion.failWrapper.TWithHelper.Helper() + assertion.failWrapper.Fail(description+message, 2+assertion.offset) + return false +} + +func vetExtras(extras []interface{}) (bool, string) { + for i, extra := range extras { + if extra != nil { + zeroValue := reflect.Zero(reflect.TypeOf(extra)).Interface() + if !reflect.DeepEqual(zeroValue, extra) { + message := fmt.Sprintf("Unexpected non-nil/non-zero extra argument at index %d:\n\t<%T>: %#v", i+1, extra, extra) + return false, message + } + } + } + return true, "" +} diff --git a/vendor/github.com/onsi/gomega/internal/asyncassertion/async_assertion.go b/vendor/github.com/onsi/gomega/internal/asyncassertion/async_assertion.go new file mode 100644 index 000000000..cdab233eb --- /dev/null +++ b/vendor/github.com/onsi/gomega/internal/asyncassertion/async_assertion.go @@ -0,0 +1,194 @@ +package asyncassertion + +import ( + "errors" + "fmt" + "reflect" + "time" + + "github.com/onsi/gomega/internal/oraclematcher" + "github.com/onsi/gomega/types" +) + +type AsyncAssertionType uint + +const ( + AsyncAssertionTypeEventually AsyncAssertionType = iota + AsyncAssertionTypeConsistently +) + +type AsyncAssertion struct { + asyncType AsyncAssertionType + actualInput interface{} + timeoutInterval time.Duration + pollingInterval time.Duration + failWrapper *types.GomegaFailWrapper + offset int +} + +func New(asyncType AsyncAssertionType, actualInput interface{}, failWrapper *types.GomegaFailWrapper, timeoutInterval time.Duration, pollingInterval time.Duration, offset int) *AsyncAssertion { + actualType := reflect.TypeOf(actualInput) + if actualType.Kind() == reflect.Func { + if actualType.NumIn() != 0 || actualType.NumOut() == 0 { + panic("Expected a function with no arguments and one or more return values.") + } + } + + return &AsyncAssertion{ + asyncType: asyncType, + actualInput: actualInput, + failWrapper: failWrapper, + timeoutInterval: timeoutInterval, + pollingInterval: pollingInterval, + offset: offset, + } +} + +func (assertion *AsyncAssertion) Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.match(matcher, true, optionalDescription...) +} + +func (assertion *AsyncAssertion) ShouldNot(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { + assertion.failWrapper.TWithHelper.Helper() + return assertion.match(matcher, false, optionalDescription...) +} + +func (assertion *AsyncAssertion) buildDescription(optionalDescription ...interface{}) string { + switch len(optionalDescription) { + case 0: + return "" + default: + return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...) + "\n" + } +} + +func (assertion *AsyncAssertion) actualInputIsAFunction() bool { + actualType := reflect.TypeOf(assertion.actualInput) + return actualType.Kind() == reflect.Func && actualType.NumIn() == 0 && actualType.NumOut() > 0 +} + +func (assertion *AsyncAssertion) pollActual() (interface{}, error) { + if assertion.actualInputIsAFunction() { + values := reflect.ValueOf(assertion.actualInput).Call([]reflect.Value{}) + + extras := []interface{}{} + for _, value := range values[1:] { + extras = append(extras, value.Interface()) + } + + success, message := vetExtras(extras) + + if !success { + return nil, errors.New(message) + } + + return values[0].Interface(), nil + } + + return assertion.actualInput, nil +} + +func (assertion *AsyncAssertion) matcherMayChange(matcher types.GomegaMatcher, value interface{}) bool { + if assertion.actualInputIsAFunction() { + return true + } + + return oraclematcher.MatchMayChangeInTheFuture(matcher, value) +} + +func (assertion *AsyncAssertion) match(matcher types.GomegaMatcher, desiredMatch bool, optionalDescription ...interface{}) bool { + timer := time.Now() + timeout := time.After(assertion.timeoutInterval) + + description := assertion.buildDescription(optionalDescription...) + + var matches bool + var err error + mayChange := true + value, err := assertion.pollActual() + if err == nil { + mayChange = assertion.matcherMayChange(matcher, value) + matches, err = matcher.Match(value) + } + + assertion.failWrapper.TWithHelper.Helper() + + fail := func(preamble string) { + errMsg := "" + message := "" + if err != nil { + errMsg = "Error: " + err.Error() + } else { + if desiredMatch { + message = matcher.FailureMessage(value) + } else { + message = matcher.NegatedFailureMessage(value) + } + } + assertion.failWrapper.TWithHelper.Helper() + assertion.failWrapper.Fail(fmt.Sprintf("%s after %.3fs.\n%s%s%s", preamble, time.Since(timer).Seconds(), description, message, errMsg), 3+assertion.offset) + } + + if assertion.asyncType == AsyncAssertionTypeEventually { + for { + if err == nil && matches == desiredMatch { + return true + } + + if !mayChange { + fail("No future change is possible. Bailing out early") + return false + } + + select { + case <-time.After(assertion.pollingInterval): + value, err = assertion.pollActual() + if err == nil { + mayChange = assertion.matcherMayChange(matcher, value) + matches, err = matcher.Match(value) + } + case <-timeout: + fail("Timed out") + return false + } + } + } else if assertion.asyncType == AsyncAssertionTypeConsistently { + for { + if !(err == nil && matches == desiredMatch) { + fail("Failed") + return false + } + + if !mayChange { + return true + } + + select { + case <-time.After(assertion.pollingInterval): + value, err = assertion.pollActual() + if err == nil { + mayChange = assertion.matcherMayChange(matcher, value) + matches, err = matcher.Match(value) + } + case <-timeout: + return true + } + } + } + + return false +} + +func vetExtras(extras []interface{}) (bool, string) { + for i, extra := range extras { + if extra != nil { + zeroValue := reflect.Zero(reflect.TypeOf(extra)).Interface() + if !reflect.DeepEqual(zeroValue, extra) { + message := fmt.Sprintf("Unexpected non-nil/non-zero extra argument at index %d:\n\t<%T>: %#v", i+1, extra, extra) + return false, message + } + } + } + return true, "" +} diff --git a/vendor/github.com/onsi/gomega/internal/oraclematcher/oracle_matcher.go b/vendor/github.com/onsi/gomega/internal/oraclematcher/oracle_matcher.go new file mode 100644 index 000000000..66cad88a1 --- /dev/null +++ b/vendor/github.com/onsi/gomega/internal/oraclematcher/oracle_matcher.go @@ -0,0 +1,25 @@ +package oraclematcher + +import "github.com/onsi/gomega/types" + +/* +GomegaMatchers that also match the OracleMatcher interface can convey information about +whether or not their result will change upon future attempts. + +This allows `Eventually` and `Consistently` to short circuit if success becomes impossible. + +For example, a process' exit code can never change. So, gexec's Exit matcher returns `true` +for `MatchMayChangeInTheFuture` until the process exits, at which point it returns `false` forevermore. +*/ +type OracleMatcher interface { + MatchMayChangeInTheFuture(actual interface{}) bool +} + +func MatchMayChangeInTheFuture(matcher types.GomegaMatcher, value interface{}) bool { + oracleMatcher, ok := matcher.(OracleMatcher) + if !ok { + return true + } + + return oracleMatcher.MatchMayChangeInTheFuture(value) +} diff --git a/vendor/github.com/onsi/gomega/internal/testingtsupport/testing_t_support.go b/vendor/github.com/onsi/gomega/internal/testingtsupport/testing_t_support.go new file mode 100644 index 000000000..bb27032f6 --- /dev/null +++ b/vendor/github.com/onsi/gomega/internal/testingtsupport/testing_t_support.go @@ -0,0 +1,60 @@ +package testingtsupport + +import ( + "regexp" + "runtime/debug" + "strings" + + "github.com/onsi/gomega/types" +) + +var StackTracePruneRE = regexp.MustCompile(`\/gomega\/|\/ginkgo\/|\/pkg\/testing\/|\/pkg\/runtime\/`) + +type EmptyTWithHelper struct{} + +func (e EmptyTWithHelper) Helper() {} + +type gomegaTestingT interface { + Fatalf(format string, args ...interface{}) +} + +func BuildTestingTGomegaFailWrapper(t gomegaTestingT) *types.GomegaFailWrapper { + tWithHelper, hasHelper := t.(types.TWithHelper) + if !hasHelper { + tWithHelper = EmptyTWithHelper{} + } + + fail := func(message string, callerSkip ...int) { + if hasHelper { + tWithHelper.Helper() + t.Fatalf("\n%s", message) + } else { + skip := 2 + if len(callerSkip) > 0 { + skip += callerSkip[0] + } + stackTrace := pruneStack(string(debug.Stack()), skip) + t.Fatalf("\n%s\n%s\n", stackTrace, message) + } + } + + return &types.GomegaFailWrapper{ + Fail: fail, + TWithHelper: tWithHelper, + } +} + +func pruneStack(fullStackTrace string, skip int) string { + stack := strings.Split(fullStackTrace, "\n")[1:] + if len(stack) > 2*skip { + stack = stack[2*skip:] + } + prunedStack := []string{} + for i := 0; i < len(stack)/2; i++ { + if !StackTracePruneRE.Match([]byte(stack[i*2])) { + prunedStack = append(prunedStack, stack[i*2]) + prunedStack = append(prunedStack, stack[i*2+1]) + } + } + return strings.Join(prunedStack, "\n") +} diff --git a/vendor/github.com/onsi/gomega/matchers.go b/vendor/github.com/onsi/gomega/matchers.go new file mode 100644 index 000000000..c3a326dd4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers.go @@ -0,0 +1,427 @@ +package gomega + +import ( + "time" + + "github.com/onsi/gomega/matchers" + "github.com/onsi/gomega/types" +) + +//Equal uses reflect.DeepEqual to compare actual with expected. Equal is strict about +//types when performing comparisons. +//It is an error for both actual and expected to be nil. Use BeNil() instead. +func Equal(expected interface{}) types.GomegaMatcher { + return &matchers.EqualMatcher{ + Expected: expected, + } +} + +//BeEquivalentTo is more lax than Equal, allowing equality between different types. +//This is done by converting actual to have the type of expected before +//attempting equality with reflect.DeepEqual. +//It is an error for actual and expected to be nil. Use BeNil() instead. +func BeEquivalentTo(expected interface{}) types.GomegaMatcher { + return &matchers.BeEquivalentToMatcher{ + Expected: expected, + } +} + +//BeIdenticalTo uses the == operator to compare actual with expected. +//BeIdenticalTo is strict about types when performing comparisons. +//It is an error for both actual and expected to be nil. Use BeNil() instead. +func BeIdenticalTo(expected interface{}) types.GomegaMatcher { + return &matchers.BeIdenticalToMatcher{ + Expected: expected, + } +} + +//BeNil succeeds if actual is nil +func BeNil() types.GomegaMatcher { + return &matchers.BeNilMatcher{} +} + +//BeTrue succeeds if actual is true +func BeTrue() types.GomegaMatcher { + return &matchers.BeTrueMatcher{} +} + +//BeFalse succeeds if actual is false +func BeFalse() types.GomegaMatcher { + return &matchers.BeFalseMatcher{} +} + +//HaveOccurred succeeds if actual is a non-nil error +//The typical Go error checking pattern looks like: +// err := SomethingThatMightFail() +// Expect(err).ShouldNot(HaveOccurred()) +func HaveOccurred() types.GomegaMatcher { + return &matchers.HaveOccurredMatcher{} +} + +//Succeed passes if actual is a nil error +//Succeed is intended to be used with functions that return a single error value. Instead of +// err := SomethingThatMightFail() +// Expect(err).ShouldNot(HaveOccurred()) +// +//You can write: +// Expect(SomethingThatMightFail()).Should(Succeed()) +// +//It is a mistake to use Succeed with a function that has multiple return values. Gomega's Ω and Expect +//functions automatically trigger failure if any return values after the first return value are non-zero/non-nil. +//This means that Ω(MultiReturnFunc()).ShouldNot(Succeed()) can never pass. +func Succeed() types.GomegaMatcher { + return &matchers.SucceedMatcher{} +} + +//MatchError succeeds if actual is a non-nil error that matches the passed in string/error. +// +//These are valid use-cases: +// Expect(err).Should(MatchError("an error")) //asserts that err.Error() == "an error" +// Expect(err).Should(MatchError(SomeError)) //asserts that err == SomeError (via reflect.DeepEqual) +// +//It is an error for err to be nil or an object that does not implement the Error interface +func MatchError(expected interface{}) types.GomegaMatcher { + return &matchers.MatchErrorMatcher{ + Expected: expected, + } +} + +//BeClosed succeeds if actual is a closed channel. +//It is an error to pass a non-channel to BeClosed, it is also an error to pass nil +// +//In order to check whether or not the channel is closed, Gomega must try to read from the channel +//(even in the `ShouldNot(BeClosed())` case). You should keep this in mind if you wish to make subsequent assertions about +//values coming down the channel. +// +//Also, if you are testing that a *buffered* channel is closed you must first read all values out of the channel before +//asserting that it is closed (it is not possible to detect that a buffered-channel has been closed until all its buffered values are read). +// +//Finally, as a corollary: it is an error to check whether or not a send-only channel is closed. +func BeClosed() types.GomegaMatcher { + return &matchers.BeClosedMatcher{} +} + +//Receive succeeds if there is a value to be received on actual. +//Actual must be a channel (and cannot be a send-only channel) -- anything else is an error. +// +//Receive returns immediately and never blocks: +// +//- If there is nothing on the channel `c` then Expect(c).Should(Receive()) will fail and Ω(c).ShouldNot(Receive()) will pass. +// +//- If the channel `c` is closed then Expect(c).Should(Receive()) will fail and Ω(c).ShouldNot(Receive()) will pass. +// +//- If there is something on the channel `c` ready to be read, then Expect(c).Should(Receive()) will pass and Ω(c).ShouldNot(Receive()) will fail. +// +//If you have a go-routine running in the background that will write to channel `c` you can: +// Eventually(c).Should(Receive()) +// +//This will timeout if nothing gets sent to `c` (you can modify the timeout interval as you normally do with `Eventually`) +// +//A similar use-case is to assert that no go-routine writes to a channel (for a period of time). You can do this with `Consistently`: +// Consistently(c).ShouldNot(Receive()) +// +//You can pass `Receive` a matcher. If you do so, it will match the received object against the matcher. For example: +// Expect(c).Should(Receive(Equal("foo"))) +// +//When given a matcher, `Receive` will always fail if there is nothing to be received on the channel. +// +//Passing Receive a matcher is especially useful when paired with Eventually: +// +// Eventually(c).Should(Receive(ContainSubstring("bar"))) +// +//will repeatedly attempt to pull values out of `c` until a value matching "bar" is received. +// +//Finally, if you want to have a reference to the value *sent* to the channel you can pass the `Receive` matcher a pointer to a variable of the appropriate type: +// var myThing thing +// Eventually(thingChan).Should(Receive(&myThing)) +// Expect(myThing.Sprocket).Should(Equal("foo")) +// Expect(myThing.IsValid()).Should(BeTrue()) +func Receive(args ...interface{}) types.GomegaMatcher { + var arg interface{} + if len(args) > 0 { + arg = args[0] + } + + return &matchers.ReceiveMatcher{ + Arg: arg, + } +} + +//BeSent succeeds if a value can be sent to actual. +//Actual must be a channel (and cannot be a receive-only channel) that can sent the type of the value passed into BeSent -- anything else is an error. +//In addition, actual must not be closed. +// +//BeSent never blocks: +// +//- If the channel `c` is not ready to receive then Expect(c).Should(BeSent("foo")) will fail immediately +//- If the channel `c` is eventually ready to receive then Eventually(c).Should(BeSent("foo")) will succeed.. presuming the channel becomes ready to receive before Eventually's timeout +//- If the channel `c` is closed then Expect(c).Should(BeSent("foo")) and Ω(c).ShouldNot(BeSent("foo")) will both fail immediately +// +//Of course, the value is actually sent to the channel. The point of `BeSent` is less to make an assertion about the availability of the channel (which is typically an implementation detail that your test should not be concerned with). +//Rather, the point of `BeSent` is to make it possible to easily and expressively write tests that can timeout on blocked channel sends. +func BeSent(arg interface{}) types.GomegaMatcher { + return &matchers.BeSentMatcher{ + Arg: arg, + } +} + +//MatchRegexp succeeds if actual is a string or stringer that matches the +//passed-in regexp. Optional arguments can be provided to construct a regexp +//via fmt.Sprintf(). +func MatchRegexp(regexp string, args ...interface{}) types.GomegaMatcher { + return &matchers.MatchRegexpMatcher{ + Regexp: regexp, + Args: args, + } +} + +//ContainSubstring succeeds if actual is a string or stringer that contains the +//passed-in substring. Optional arguments can be provided to construct the substring +//via fmt.Sprintf(). +func ContainSubstring(substr string, args ...interface{}) types.GomegaMatcher { + return &matchers.ContainSubstringMatcher{ + Substr: substr, + Args: args, + } +} + +//HavePrefix succeeds if actual is a string or stringer that contains the +//passed-in string as a prefix. Optional arguments can be provided to construct +//via fmt.Sprintf(). +func HavePrefix(prefix string, args ...interface{}) types.GomegaMatcher { + return &matchers.HavePrefixMatcher{ + Prefix: prefix, + Args: args, + } +} + +//HaveSuffix succeeds if actual is a string or stringer that contains the +//passed-in string as a suffix. Optional arguments can be provided to construct +//via fmt.Sprintf(). +func HaveSuffix(suffix string, args ...interface{}) types.GomegaMatcher { + return &matchers.HaveSuffixMatcher{ + Suffix: suffix, + Args: args, + } +} + +//MatchJSON succeeds if actual is a string or stringer of JSON that matches +//the expected JSON. The JSONs are decoded and the resulting objects are compared via +//reflect.DeepEqual so things like key-ordering and whitespace shouldn't matter. +func MatchJSON(json interface{}) types.GomegaMatcher { + return &matchers.MatchJSONMatcher{ + JSONToMatch: json, + } +} + +//MatchXML succeeds if actual is a string or stringer of XML that matches +//the expected XML. The XMLs are decoded and the resulting objects are compared via +//reflect.DeepEqual so things like whitespaces shouldn't matter. +func MatchXML(xml interface{}) types.GomegaMatcher { + return &matchers.MatchXMLMatcher{ + XMLToMatch: xml, + } +} + +//MatchYAML succeeds if actual is a string or stringer of YAML that matches +//the expected YAML. The YAML's are decoded and the resulting objects are compared via +//reflect.DeepEqual so things like key-ordering and whitespace shouldn't matter. +func MatchYAML(yaml interface{}) types.GomegaMatcher { + return &matchers.MatchYAMLMatcher{ + YAMLToMatch: yaml, + } +} + +//BeEmpty succeeds if actual is empty. Actual must be of type string, array, map, chan, or slice. +func BeEmpty() types.GomegaMatcher { + return &matchers.BeEmptyMatcher{} +} + +//HaveLen succeeds if actual has the passed-in length. Actual must be of type string, array, map, chan, or slice. +func HaveLen(count int) types.GomegaMatcher { + return &matchers.HaveLenMatcher{ + Count: count, + } +} + +//HaveCap succeeds if actual has the passed-in capacity. Actual must be of type array, chan, or slice. +func HaveCap(count int) types.GomegaMatcher { + return &matchers.HaveCapMatcher{ + Count: count, + } +} + +//BeZero succeeds if actual is the zero value for its type or if actual is nil. +func BeZero() types.GomegaMatcher { + return &matchers.BeZeroMatcher{} +} + +//ContainElement succeeds if actual contains the passed in element. +//By default ContainElement() uses Equal() to perform the match, however a +//matcher can be passed in instead: +// Expect([]string{"Foo", "FooBar"}).Should(ContainElement(ContainSubstring("Bar"))) +// +//Actual must be an array, slice or map. +//For maps, ContainElement searches through the map's values. +func ContainElement(element interface{}) types.GomegaMatcher { + return &matchers.ContainElementMatcher{ + Element: element, + } +} + +//ConsistOf succeeds if actual contains precisely the elements passed into the matcher. The ordering of the elements does not matter. +//By default ConsistOf() uses Equal() to match the elements, however custom matchers can be passed in instead. Here are some examples: +// +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf("FooBar", "Foo")) +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf(ContainSubstring("Bar"), "Foo")) +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf(ContainSubstring("Foo"), ContainSubstring("Foo"))) +// +//Actual must be an array, slice or map. For maps, ConsistOf matches against the map's values. +// +//You typically pass variadic arguments to ConsistOf (as in the examples above). However, if you need to pass in a slice you can provided that it +//is the only element passed in to ConsistOf: +// +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf([]string{"FooBar", "Foo"})) +// +//Note that Go's type system does not allow you to write this as ConsistOf([]string{"FooBar", "Foo"}...) as []string and []interface{} are different types - hence the need for this special rule. +func ConsistOf(elements ...interface{}) types.GomegaMatcher { + return &matchers.ConsistOfMatcher{ + Elements: elements, + } +} + +//HaveKey succeeds if actual is a map with the passed in key. +//By default HaveKey uses Equal() to perform the match, however a +//matcher can be passed in instead: +// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKey(MatchRegexp(`.+Foo$`))) +func HaveKey(key interface{}) types.GomegaMatcher { + return &matchers.HaveKeyMatcher{ + Key: key, + } +} + +//HaveKeyWithValue succeeds if actual is a map with the passed in key and value. +//By default HaveKeyWithValue uses Equal() to perform the match, however a +//matcher can be passed in instead: +// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKeyWithValue("Foo", "Bar")) +// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKeyWithValue(MatchRegexp(`.+Foo$`), "Bar")) +func HaveKeyWithValue(key interface{}, value interface{}) types.GomegaMatcher { + return &matchers.HaveKeyWithValueMatcher{ + Key: key, + Value: value, + } +} + +//BeNumerically performs numerical assertions in a type-agnostic way. +//Actual and expected should be numbers, though the specific type of +//number is irrelevant (float32, float64, uint8, etc...). +// +//There are six, self-explanatory, supported comparators: +// Expect(1.0).Should(BeNumerically("==", 1)) +// Expect(1.0).Should(BeNumerically("~", 0.999, 0.01)) +// Expect(1.0).Should(BeNumerically(">", 0.9)) +// Expect(1.0).Should(BeNumerically(">=", 1.0)) +// Expect(1.0).Should(BeNumerically("<", 3)) +// Expect(1.0).Should(BeNumerically("<=", 1.0)) +func BeNumerically(comparator string, compareTo ...interface{}) types.GomegaMatcher { + return &matchers.BeNumericallyMatcher{ + Comparator: comparator, + CompareTo: compareTo, + } +} + +//BeTemporally compares time.Time's like BeNumerically +//Actual and expected must be time.Time. The comparators are the same as for BeNumerically +// Expect(time.Now()).Should(BeTemporally(">", time.Time{})) +// Expect(time.Now()).Should(BeTemporally("~", time.Now(), time.Second)) +func BeTemporally(comparator string, compareTo time.Time, threshold ...time.Duration) types.GomegaMatcher { + return &matchers.BeTemporallyMatcher{ + Comparator: comparator, + CompareTo: compareTo, + Threshold: threshold, + } +} + +//BeAssignableToTypeOf succeeds if actual is assignable to the type of expected. +//It will return an error when one of the values is nil. +// Expect(0).Should(BeAssignableToTypeOf(0)) // Same values +// Expect(5).Should(BeAssignableToTypeOf(-1)) // different values same type +// Expect("foo").Should(BeAssignableToTypeOf("bar")) // different values same type +// Expect(struct{ Foo string }{}).Should(BeAssignableToTypeOf(struct{ Foo string }{})) +func BeAssignableToTypeOf(expected interface{}) types.GomegaMatcher { + return &matchers.AssignableToTypeOfMatcher{ + Expected: expected, + } +} + +//Panic succeeds if actual is a function that, when invoked, panics. +//Actual must be a function that takes no arguments and returns no results. +func Panic() types.GomegaMatcher { + return &matchers.PanicMatcher{} +} + +//BeAnExistingFile succeeds if a file exists. +//Actual must be a string representing the abs path to the file being checked. +func BeAnExistingFile() types.GomegaMatcher { + return &matchers.BeAnExistingFileMatcher{} +} + +//BeARegularFile succeeds if a file exists and is a regular file. +//Actual must be a string representing the abs path to the file being checked. +func BeARegularFile() types.GomegaMatcher { + return &matchers.BeARegularFileMatcher{} +} + +//BeADirectory succeeds if a file exists and is a directory. +//Actual must be a string representing the abs path to the file being checked. +func BeADirectory() types.GomegaMatcher { + return &matchers.BeADirectoryMatcher{} +} + +//And succeeds only if all of the given matchers succeed. +//The matchers are tried in order, and will fail-fast if one doesn't succeed. +// Expect("hi").To(And(HaveLen(2), Equal("hi")) +// +//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +func And(ms ...types.GomegaMatcher) types.GomegaMatcher { + return &matchers.AndMatcher{Matchers: ms} +} + +//SatisfyAll is an alias for And(). +// Expect("hi").Should(SatisfyAll(HaveLen(2), Equal("hi"))) +func SatisfyAll(matchers ...types.GomegaMatcher) types.GomegaMatcher { + return And(matchers...) +} + +//Or succeeds if any of the given matchers succeed. +//The matchers are tried in order and will return immediately upon the first successful match. +// Expect("hi").To(Or(HaveLen(3), HaveLen(2)) +// +//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +func Or(ms ...types.GomegaMatcher) types.GomegaMatcher { + return &matchers.OrMatcher{Matchers: ms} +} + +//SatisfyAny is an alias for Or(). +// Expect("hi").SatisfyAny(Or(HaveLen(3), HaveLen(2)) +func SatisfyAny(matchers ...types.GomegaMatcher) types.GomegaMatcher { + return Or(matchers...) +} + +//Not negates the given matcher; it succeeds if the given matcher fails. +// Expect(1).To(Not(Equal(2)) +// +//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +func Not(matcher types.GomegaMatcher) types.GomegaMatcher { + return &matchers.NotMatcher{Matcher: matcher} +} + +//WithTransform applies the `transform` to the actual value and matches it against `matcher`. +//The given transform must be a function of one parameter that returns one value. +// var plus1 = func(i int) int { return i + 1 } +// Expect(1).To(WithTransform(plus1, Equal(2)) +// +//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +func WithTransform(transform interface{}, matcher types.GomegaMatcher) types.GomegaMatcher { + return matchers.NewWithTransformMatcher(transform, matcher) +} diff --git a/vendor/github.com/onsi/gomega/matchers/and.go b/vendor/github.com/onsi/gomega/matchers/and.go new file mode 100644 index 000000000..d83a29164 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/and.go @@ -0,0 +1,63 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/internal/oraclematcher" + "github.com/onsi/gomega/types" +) + +type AndMatcher struct { + Matchers []types.GomegaMatcher + + // state + firstFailedMatcher types.GomegaMatcher +} + +func (m *AndMatcher) Match(actual interface{}) (success bool, err error) { + m.firstFailedMatcher = nil + for _, matcher := range m.Matchers { + success, err := matcher.Match(actual) + if !success || err != nil { + m.firstFailedMatcher = matcher + return false, err + } + } + return true, nil +} + +func (m *AndMatcher) FailureMessage(actual interface{}) (message string) { + return m.firstFailedMatcher.FailureMessage(actual) +} + +func (m *AndMatcher) NegatedFailureMessage(actual interface{}) (message string) { + // not the most beautiful list of matchers, but not bad either... + return format.Message(actual, fmt.Sprintf("To not satisfy all of these matchers: %s", m.Matchers)) +} + +func (m *AndMatcher) MatchMayChangeInTheFuture(actual interface{}) bool { + /* + Example with 3 matchers: A, B, C + + Match evaluates them: T, F, => F + So match is currently F, what should MatchMayChangeInTheFuture() return? + Seems like it only depends on B, since currently B MUST change to allow the result to become T + + Match eval: T, T, T => T + So match is currently T, what should MatchMayChangeInTheFuture() return? + Seems to depend on ANY of them being able to change to F. + */ + + if m.firstFailedMatcher == nil { + // so all matchers succeeded.. Any one of them changing would change the result. + for _, matcher := range m.Matchers { + if oraclematcher.MatchMayChangeInTheFuture(matcher, actual) { + return true + } + } + return false // none of were going to change + } + // one of the matchers failed.. it must be able to change in order to affect the result + return oraclematcher.MatchMayChangeInTheFuture(m.firstFailedMatcher, actual) +} diff --git a/vendor/github.com/onsi/gomega/matchers/assignable_to_type_of_matcher.go b/vendor/github.com/onsi/gomega/matchers/assignable_to_type_of_matcher.go new file mode 100644 index 000000000..51f8be6ae --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/assignable_to_type_of_matcher.go @@ -0,0 +1,35 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type AssignableToTypeOfMatcher struct { + Expected interface{} +} + +func (matcher *AssignableToTypeOfMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil && matcher.Expected == nil { + return false, fmt.Errorf("Refusing to compare to .\nBe explicit and use BeNil() instead. This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.") + } else if matcher.Expected == nil { + return false, fmt.Errorf("Refusing to compare type to .\nBe explicit and use BeNil() instead. This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.") + } else if actual == nil { + return false, nil + } + + actualType := reflect.TypeOf(actual) + expectedType := reflect.TypeOf(matcher.Expected) + + return actualType.AssignableTo(expectedType), nil +} + +func (matcher *AssignableToTypeOfMatcher) FailureMessage(actual interface{}) string { + return format.Message(actual, fmt.Sprintf("to be assignable to the type: %T", matcher.Expected)) +} + +func (matcher *AssignableToTypeOfMatcher) NegatedFailureMessage(actual interface{}) string { + return format.Message(actual, fmt.Sprintf("not to be assignable to the type: %T", matcher.Expected)) +} diff --git a/vendor/github.com/onsi/gomega/matchers/attributes_slice.go b/vendor/github.com/onsi/gomega/matchers/attributes_slice.go new file mode 100644 index 000000000..355b362f4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/attributes_slice.go @@ -0,0 +1,14 @@ +package matchers + +import ( + "encoding/xml" + "strings" +) + +type attributesSlice []xml.Attr + +func (attrs attributesSlice) Len() int { return len(attrs) } +func (attrs attributesSlice) Less(i, j int) bool { + return strings.Compare(attrs[i].Name.Local, attrs[j].Name.Local) == -1 +} +func (attrs attributesSlice) Swap(i, j int) { attrs[i], attrs[j] = attrs[j], attrs[i] } diff --git a/vendor/github.com/onsi/gomega/matchers/be_a_directory.go b/vendor/github.com/onsi/gomega/matchers/be_a_directory.go new file mode 100644 index 000000000..7b6975e41 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_a_directory.go @@ -0,0 +1,54 @@ +package matchers + +import ( + "fmt" + "os" + + "github.com/onsi/gomega/format" +) + +type notADirectoryError struct { + os.FileInfo +} + +func (t notADirectoryError) Error() string { + fileInfo := os.FileInfo(t) + switch { + case fileInfo.Mode().IsRegular(): + return "file is a regular file" + default: + return fmt.Sprintf("file mode is: %s", fileInfo.Mode().String()) + } +} + +type BeADirectoryMatcher struct { + expected interface{} + err error +} + +func (matcher *BeADirectoryMatcher) Match(actual interface{}) (success bool, err error) { + actualFilename, ok := actual.(string) + if !ok { + return false, fmt.Errorf("BeADirectoryMatcher matcher expects a file path") + } + + fileInfo, err := os.Stat(actualFilename) + if err != nil { + matcher.err = err + return false, nil + } + + if !fileInfo.Mode().IsDir() { + matcher.err = notADirectoryError{fileInfo} + return false, nil + } + return true, nil +} + +func (matcher *BeADirectoryMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("to be a directory: %s", matcher.err)) +} + +func (matcher *BeADirectoryMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not be a directory")) +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_a_regular_file.go b/vendor/github.com/onsi/gomega/matchers/be_a_regular_file.go new file mode 100644 index 000000000..e239131fb --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_a_regular_file.go @@ -0,0 +1,54 @@ +package matchers + +import ( + "fmt" + "os" + + "github.com/onsi/gomega/format" +) + +type notARegularFileError struct { + os.FileInfo +} + +func (t notARegularFileError) Error() string { + fileInfo := os.FileInfo(t) + switch { + case fileInfo.IsDir(): + return "file is a directory" + default: + return fmt.Sprintf("file mode is: %s", fileInfo.Mode().String()) + } +} + +type BeARegularFileMatcher struct { + expected interface{} + err error +} + +func (matcher *BeARegularFileMatcher) Match(actual interface{}) (success bool, err error) { + actualFilename, ok := actual.(string) + if !ok { + return false, fmt.Errorf("BeARegularFileMatcher matcher expects a file path") + } + + fileInfo, err := os.Stat(actualFilename) + if err != nil { + matcher.err = err + return false, nil + } + + if !fileInfo.Mode().IsRegular() { + matcher.err = notARegularFileError{fileInfo} + return false, nil + } + return true, nil +} + +func (matcher *BeARegularFileMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("to be a regular file: %s", matcher.err)) +} + +func (matcher *BeARegularFileMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not be a regular file")) +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_an_existing_file.go b/vendor/github.com/onsi/gomega/matchers/be_an_existing_file.go new file mode 100644 index 000000000..d42eba223 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_an_existing_file.go @@ -0,0 +1,38 @@ +package matchers + +import ( + "fmt" + "os" + + "github.com/onsi/gomega/format" +) + +type BeAnExistingFileMatcher struct { + expected interface{} +} + +func (matcher *BeAnExistingFileMatcher) Match(actual interface{}) (success bool, err error) { + actualFilename, ok := actual.(string) + if !ok { + return false, fmt.Errorf("BeAnExistingFileMatcher matcher expects a file path") + } + + if _, err = os.Stat(actualFilename); err != nil { + switch { + case os.IsNotExist(err): + return false, nil + default: + return false, err + } + } + + return true, nil +} + +func (matcher *BeAnExistingFileMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("to exist")) +} + +func (matcher *BeAnExistingFileMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not to exist")) +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_closed_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_closed_matcher.go new file mode 100644 index 000000000..80c9c8bb1 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_closed_matcher.go @@ -0,0 +1,46 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type BeClosedMatcher struct { +} + +func (matcher *BeClosedMatcher) Match(actual interface{}) (success bool, err error) { + if !isChan(actual) { + return false, fmt.Errorf("BeClosed matcher expects a channel. Got:\n%s", format.Object(actual, 1)) + } + + channelType := reflect.TypeOf(actual) + channelValue := reflect.ValueOf(actual) + + if channelType.ChanDir() == reflect.SendDir { + return false, fmt.Errorf("BeClosed matcher cannot determine if a send-only channel is closed or open. Got:\n%s", format.Object(actual, 1)) + } + + winnerIndex, _, open := reflect.Select([]reflect.SelectCase{ + {Dir: reflect.SelectRecv, Chan: channelValue}, + {Dir: reflect.SelectDefault}, + }) + + var closed bool + if winnerIndex == 0 { + closed = !open + } else if winnerIndex == 1 { + closed = false + } + + return closed, nil +} + +func (matcher *BeClosedMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be closed") +} + +func (matcher *BeClosedMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be open") +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_empty_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_empty_matcher.go new file mode 100644 index 000000000..8b00311b0 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_empty_matcher.go @@ -0,0 +1,27 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type BeEmptyMatcher struct { +} + +func (matcher *BeEmptyMatcher) Match(actual interface{}) (success bool, err error) { + length, ok := lengthOf(actual) + if !ok { + return false, fmt.Errorf("BeEmpty matcher expects a string/array/map/channel/slice. Got:\n%s", format.Object(actual, 1)) + } + + return length == 0, nil +} + +func (matcher *BeEmptyMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be empty") +} + +func (matcher *BeEmptyMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be empty") +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_equivalent_to_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_equivalent_to_matcher.go new file mode 100644 index 000000000..97ab20a4e --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_equivalent_to_matcher.go @@ -0,0 +1,34 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type BeEquivalentToMatcher struct { + Expected interface{} +} + +func (matcher *BeEquivalentToMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil && matcher.Expected == nil { + return false, fmt.Errorf("Both actual and expected must not be nil.") + } + + convertedActual := actual + + if actual != nil && matcher.Expected != nil && reflect.TypeOf(actual).ConvertibleTo(reflect.TypeOf(matcher.Expected)) { + convertedActual = reflect.ValueOf(actual).Convert(reflect.TypeOf(matcher.Expected)).Interface() + } + + return reflect.DeepEqual(convertedActual, matcher.Expected), nil +} + +func (matcher *BeEquivalentToMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be equivalent to", matcher.Expected) +} + +func (matcher *BeEquivalentToMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be equivalent to", matcher.Expected) +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_false_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_false_matcher.go new file mode 100644 index 000000000..91d3b779e --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_false_matcher.go @@ -0,0 +1,26 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type BeFalseMatcher struct { +} + +func (matcher *BeFalseMatcher) Match(actual interface{}) (success bool, err error) { + if !isBool(actual) { + return false, fmt.Errorf("Expected a boolean. Got:\n%s", format.Object(actual, 1)) + } + + return actual == false, nil +} + +func (matcher *BeFalseMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be false") +} + +func (matcher *BeFalseMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be false") +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_identical_to.go b/vendor/github.com/onsi/gomega/matchers/be_identical_to.go new file mode 100644 index 000000000..fdcda4d1f --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_identical_to.go @@ -0,0 +1,37 @@ +package matchers + +import ( + "fmt" + "runtime" + + "github.com/onsi/gomega/format" +) + +type BeIdenticalToMatcher struct { + Expected interface{} +} + +func (matcher *BeIdenticalToMatcher) Match(actual interface{}) (success bool, matchErr error) { + if actual == nil && matcher.Expected == nil { + return false, fmt.Errorf("Refusing to compare to .\nBe explicit and use BeNil() instead. This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.") + } + + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + success = false + matchErr = nil + } + } + }() + + return actual == matcher.Expected, nil +} + +func (matcher *BeIdenticalToMatcher) FailureMessage(actual interface{}) string { + return format.Message(actual, "to be identical to", matcher.Expected) +} + +func (matcher *BeIdenticalToMatcher) NegatedFailureMessage(actual interface{}) string { + return format.Message(actual, "not to be identical to", matcher.Expected) +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_nil_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_nil_matcher.go new file mode 100644 index 000000000..7ee84fe1b --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_nil_matcher.go @@ -0,0 +1,18 @@ +package matchers + +import "github.com/onsi/gomega/format" + +type BeNilMatcher struct { +} + +func (matcher *BeNilMatcher) Match(actual interface{}) (success bool, err error) { + return isNil(actual), nil +} + +func (matcher *BeNilMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be nil") +} + +func (matcher *BeNilMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be nil") +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_numerically_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_numerically_matcher.go new file mode 100644 index 000000000..9f4f77eec --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_numerically_matcher.go @@ -0,0 +1,132 @@ +package matchers + +import ( + "fmt" + "math" + + "github.com/onsi/gomega/format" +) + +type BeNumericallyMatcher struct { + Comparator string + CompareTo []interface{} +} + +func (matcher *BeNumericallyMatcher) FailureMessage(actual interface{}) (message string) { + return matcher.FormatFailureMessage(actual, false) +} + +func (matcher *BeNumericallyMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return matcher.FormatFailureMessage(actual, true) +} + +func (matcher *BeNumericallyMatcher) FormatFailureMessage(actual interface{}, negated bool) (message string) { + if len(matcher.CompareTo) == 1 { + message = fmt.Sprintf("to be %s", matcher.Comparator) + } else { + message = fmt.Sprintf("to be within %v of %s", matcher.CompareTo[1], matcher.Comparator) + } + if negated { + message = "not " + message + } + return format.Message(actual, message, matcher.CompareTo[0]) +} + +func (matcher *BeNumericallyMatcher) Match(actual interface{}) (success bool, err error) { + if len(matcher.CompareTo) == 0 || len(matcher.CompareTo) > 2 { + return false, fmt.Errorf("BeNumerically requires 1 or 2 CompareTo arguments. Got:\n%s", format.Object(matcher.CompareTo, 1)) + } + if !isNumber(actual) { + return false, fmt.Errorf("Expected a number. Got:\n%s", format.Object(actual, 1)) + } + if !isNumber(matcher.CompareTo[0]) { + return false, fmt.Errorf("Expected a number. Got:\n%s", format.Object(matcher.CompareTo[0], 1)) + } + if len(matcher.CompareTo) == 2 && !isNumber(matcher.CompareTo[1]) { + return false, fmt.Errorf("Expected a number. Got:\n%s", format.Object(matcher.CompareTo[0], 1)) + } + + switch matcher.Comparator { + case "==", "~", ">", ">=", "<", "<=": + default: + return false, fmt.Errorf("Unknown comparator: %s", matcher.Comparator) + } + + if isFloat(actual) || isFloat(matcher.CompareTo[0]) { + var secondOperand float64 = 1e-8 + if len(matcher.CompareTo) == 2 { + secondOperand = toFloat(matcher.CompareTo[1]) + } + success = matcher.matchFloats(toFloat(actual), toFloat(matcher.CompareTo[0]), secondOperand) + } else if isInteger(actual) { + var secondOperand int64 = 0 + if len(matcher.CompareTo) == 2 { + secondOperand = toInteger(matcher.CompareTo[1]) + } + success = matcher.matchIntegers(toInteger(actual), toInteger(matcher.CompareTo[0]), secondOperand) + } else if isUnsignedInteger(actual) { + var secondOperand uint64 = 0 + if len(matcher.CompareTo) == 2 { + secondOperand = toUnsignedInteger(matcher.CompareTo[1]) + } + success = matcher.matchUnsignedIntegers(toUnsignedInteger(actual), toUnsignedInteger(matcher.CompareTo[0]), secondOperand) + } else { + return false, fmt.Errorf("Failed to compare:\n%s\n%s:\n%s", format.Object(actual, 1), matcher.Comparator, format.Object(matcher.CompareTo[0], 1)) + } + + return success, nil +} + +func (matcher *BeNumericallyMatcher) matchIntegers(actual, compareTo, threshold int64) (success bool) { + switch matcher.Comparator { + case "==", "~": + diff := actual - compareTo + return -threshold <= diff && diff <= threshold + case ">": + return (actual > compareTo) + case ">=": + return (actual >= compareTo) + case "<": + return (actual < compareTo) + case "<=": + return (actual <= compareTo) + } + return false +} + +func (matcher *BeNumericallyMatcher) matchUnsignedIntegers(actual, compareTo, threshold uint64) (success bool) { + switch matcher.Comparator { + case "==", "~": + if actual < compareTo { + actual, compareTo = compareTo, actual + } + return actual-compareTo <= threshold + case ">": + return (actual > compareTo) + case ">=": + return (actual >= compareTo) + case "<": + return (actual < compareTo) + case "<=": + return (actual <= compareTo) + } + return false +} + +func (matcher *BeNumericallyMatcher) matchFloats(actual, compareTo, threshold float64) (success bool) { + switch matcher.Comparator { + case "~": + return math.Abs(actual-compareTo) <= threshold + case "==": + return (actual == compareTo) + case ">": + return (actual > compareTo) + case ">=": + return (actual >= compareTo) + case "<": + return (actual < compareTo) + case "<=": + return (actual <= compareTo) + } + return false +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_sent_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_sent_matcher.go new file mode 100644 index 000000000..302dd1a0a --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_sent_matcher.go @@ -0,0 +1,71 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type BeSentMatcher struct { + Arg interface{} + channelClosed bool +} + +func (matcher *BeSentMatcher) Match(actual interface{}) (success bool, err error) { + if !isChan(actual) { + return false, fmt.Errorf("BeSent expects a channel. Got:\n%s", format.Object(actual, 1)) + } + + channelType := reflect.TypeOf(actual) + channelValue := reflect.ValueOf(actual) + + if channelType.ChanDir() == reflect.RecvDir { + return false, fmt.Errorf("BeSent matcher cannot be passed a receive-only channel. Got:\n%s", format.Object(actual, 1)) + } + + argType := reflect.TypeOf(matcher.Arg) + assignable := argType.AssignableTo(channelType.Elem()) + + if !assignable { + return false, fmt.Errorf("Cannot pass:\n%s to the channel:\n%s\nThe types don't match.", format.Object(matcher.Arg, 1), format.Object(actual, 1)) + } + + argValue := reflect.ValueOf(matcher.Arg) + + defer func() { + if e := recover(); e != nil { + success = false + err = fmt.Errorf("Cannot send to a closed channel") + matcher.channelClosed = true + } + }() + + winnerIndex, _, _ := reflect.Select([]reflect.SelectCase{ + {Dir: reflect.SelectSend, Chan: channelValue, Send: argValue}, + {Dir: reflect.SelectDefault}, + }) + + var didSend bool + if winnerIndex == 0 { + didSend = true + } + + return didSend, nil +} + +func (matcher *BeSentMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to send:", matcher.Arg) +} + +func (matcher *BeSentMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to send:", matcher.Arg) +} + +func (matcher *BeSentMatcher) MatchMayChangeInTheFuture(actual interface{}) bool { + if !isChan(actual) { + return false + } + + return !matcher.channelClosed +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_temporally_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_temporally_matcher.go new file mode 100644 index 000000000..cb7c038ef --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_temporally_matcher.go @@ -0,0 +1,66 @@ +package matchers + +import ( + "fmt" + "time" + + "github.com/onsi/gomega/format" +) + +type BeTemporallyMatcher struct { + Comparator string + CompareTo time.Time + Threshold []time.Duration +} + +func (matcher *BeTemporallyMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("to be %s", matcher.Comparator), matcher.CompareTo) +} + +func (matcher *BeTemporallyMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not to be %s", matcher.Comparator), matcher.CompareTo) +} + +func (matcher *BeTemporallyMatcher) Match(actual interface{}) (bool, error) { + // predicate to test for time.Time type + isTime := func(t interface{}) bool { + _, ok := t.(time.Time) + return ok + } + + if !isTime(actual) { + return false, fmt.Errorf("Expected a time.Time. Got:\n%s", format.Object(actual, 1)) + } + + switch matcher.Comparator { + case "==", "~", ">", ">=", "<", "<=": + default: + return false, fmt.Errorf("Unknown comparator: %s", matcher.Comparator) + } + + var threshold = time.Millisecond + if len(matcher.Threshold) == 1 { + threshold = matcher.Threshold[0] + } + + return matcher.matchTimes(actual.(time.Time), matcher.CompareTo, threshold), nil +} + +func (matcher *BeTemporallyMatcher) matchTimes(actual, compareTo time.Time, threshold time.Duration) (success bool) { + switch matcher.Comparator { + case "==": + return actual.Equal(compareTo) + case "~": + diff := actual.Sub(compareTo) + return -threshold <= diff && diff <= threshold + case ">": + return actual.After(compareTo) + case ">=": + return !actual.Before(compareTo) + case "<": + return actual.Before(compareTo) + case "<=": + return !actual.After(compareTo) + } + return false +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_true_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_true_matcher.go new file mode 100644 index 000000000..ec57c5db4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_true_matcher.go @@ -0,0 +1,26 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type BeTrueMatcher struct { +} + +func (matcher *BeTrueMatcher) Match(actual interface{}) (success bool, err error) { + if !isBool(actual) { + return false, fmt.Errorf("Expected a boolean. Got:\n%s", format.Object(actual, 1)) + } + + return actual.(bool), nil +} + +func (matcher *BeTrueMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be true") +} + +func (matcher *BeTrueMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be true") +} diff --git a/vendor/github.com/onsi/gomega/matchers/be_zero_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_zero_matcher.go new file mode 100644 index 000000000..26196f168 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_zero_matcher.go @@ -0,0 +1,28 @@ +package matchers + +import ( + "reflect" + + "github.com/onsi/gomega/format" +) + +type BeZeroMatcher struct { +} + +func (matcher *BeZeroMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil { + return true, nil + } + zeroValue := reflect.Zero(reflect.TypeOf(actual)).Interface() + + return reflect.DeepEqual(zeroValue, actual), nil + +} + +func (matcher *BeZeroMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be zero-valued") +} + +func (matcher *BeZeroMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be zero-valued") +} diff --git a/vendor/github.com/onsi/gomega/matchers/consist_of.go b/vendor/github.com/onsi/gomega/matchers/consist_of.go new file mode 100644 index 000000000..7b0e08868 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/consist_of.go @@ -0,0 +1,80 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/matchers/support/goraph/bipartitegraph" +) + +type ConsistOfMatcher struct { + Elements []interface{} +} + +func (matcher *ConsistOfMatcher) Match(actual interface{}) (success bool, err error) { + if !isArrayOrSlice(actual) && !isMap(actual) { + return false, fmt.Errorf("ConsistOf matcher expects an array/slice/map. Got:\n%s", format.Object(actual, 1)) + } + + elements := matcher.Elements + if len(matcher.Elements) == 1 && isArrayOrSlice(matcher.Elements[0]) { + elements = []interface{}{} + value := reflect.ValueOf(matcher.Elements[0]) + for i := 0; i < value.Len(); i++ { + elements = append(elements, value.Index(i).Interface()) + } + } + + matchers := []interface{}{} + for _, element := range elements { + matcher, isMatcher := element.(omegaMatcher) + if !isMatcher { + matcher = &EqualMatcher{Expected: element} + } + matchers = append(matchers, matcher) + } + + values := matcher.valuesOf(actual) + + if len(values) != len(matchers) { + return false, nil + } + + neighbours := func(v, m interface{}) (bool, error) { + match, err := m.(omegaMatcher).Match(v) + return match && err == nil, nil + } + + bipartiteGraph, err := bipartitegraph.NewBipartiteGraph(values, matchers, neighbours) + if err != nil { + return false, err + } + + return len(bipartiteGraph.LargestMatching()) == len(values), nil +} + +func (matcher *ConsistOfMatcher) valuesOf(actual interface{}) []interface{} { + value := reflect.ValueOf(actual) + values := []interface{}{} + if isMap(actual) { + keys := value.MapKeys() + for i := 0; i < value.Len(); i++ { + values = append(values, value.MapIndex(keys[i]).Interface()) + } + } else { + for i := 0; i < value.Len(); i++ { + values = append(values, value.Index(i).Interface()) + } + } + + return values +} + +func (matcher *ConsistOfMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to consist of", matcher.Elements) +} + +func (matcher *ConsistOfMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to consist of", matcher.Elements) +} diff --git a/vendor/github.com/onsi/gomega/matchers/contain_element_matcher.go b/vendor/github.com/onsi/gomega/matchers/contain_element_matcher.go new file mode 100644 index 000000000..4159335d0 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/contain_element_matcher.go @@ -0,0 +1,56 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type ContainElementMatcher struct { + Element interface{} +} + +func (matcher *ContainElementMatcher) Match(actual interface{}) (success bool, err error) { + if !isArrayOrSlice(actual) && !isMap(actual) { + return false, fmt.Errorf("ContainElement matcher expects an array/slice/map. Got:\n%s", format.Object(actual, 1)) + } + + elemMatcher, elementIsMatcher := matcher.Element.(omegaMatcher) + if !elementIsMatcher { + elemMatcher = &EqualMatcher{Expected: matcher.Element} + } + + value := reflect.ValueOf(actual) + var keys []reflect.Value + if isMap(actual) { + keys = value.MapKeys() + } + var lastError error + for i := 0; i < value.Len(); i++ { + var success bool + var err error + if isMap(actual) { + success, err = elemMatcher.Match(value.MapIndex(keys[i]).Interface()) + } else { + success, err = elemMatcher.Match(value.Index(i).Interface()) + } + if err != nil { + lastError = err + continue + } + if success { + return true, nil + } + } + + return false, lastError +} + +func (matcher *ContainElementMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to contain element matching", matcher.Element) +} + +func (matcher *ContainElementMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to contain element matching", matcher.Element) +} diff --git a/vendor/github.com/onsi/gomega/matchers/contain_substring_matcher.go b/vendor/github.com/onsi/gomega/matchers/contain_substring_matcher.go new file mode 100644 index 000000000..f8dc41e74 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/contain_substring_matcher.go @@ -0,0 +1,38 @@ +package matchers + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/format" +) + +type ContainSubstringMatcher struct { + Substr string + Args []interface{} +} + +func (matcher *ContainSubstringMatcher) Match(actual interface{}) (success bool, err error) { + actualString, ok := toString(actual) + if !ok { + return false, fmt.Errorf("ContainSubstring matcher requires a string or stringer. Got:\n%s", format.Object(actual, 1)) + } + + return strings.Contains(actualString, matcher.stringToMatch()), nil +} + +func (matcher *ContainSubstringMatcher) stringToMatch() string { + stringToMatch := matcher.Substr + if len(matcher.Args) > 0 { + stringToMatch = fmt.Sprintf(matcher.Substr, matcher.Args...) + } + return stringToMatch +} + +func (matcher *ContainSubstringMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to contain substring", matcher.stringToMatch()) +} + +func (matcher *ContainSubstringMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to contain substring", matcher.stringToMatch()) +} diff --git a/vendor/github.com/onsi/gomega/matchers/equal_matcher.go b/vendor/github.com/onsi/gomega/matchers/equal_matcher.go new file mode 100644 index 000000000..befb7bdfd --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/equal_matcher.go @@ -0,0 +1,42 @@ +package matchers + +import ( + "bytes" + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type EqualMatcher struct { + Expected interface{} +} + +func (matcher *EqualMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil && matcher.Expected == nil { + return false, fmt.Errorf("Refusing to compare to .\nBe explicit and use BeNil() instead. This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.") + } + // Shortcut for byte slices. + // Comparing long byte slices with reflect.DeepEqual is very slow, + // so use bytes.Equal if actual and expected are both byte slices. + if actualByteSlice, ok := actual.([]byte); ok { + if expectedByteSlice, ok := matcher.Expected.([]byte); ok { + return bytes.Equal(actualByteSlice, expectedByteSlice), nil + } + } + return reflect.DeepEqual(actual, matcher.Expected), nil +} + +func (matcher *EqualMatcher) FailureMessage(actual interface{}) (message string) { + actualString, actualOK := actual.(string) + expectedString, expectedOK := matcher.Expected.(string) + if actualOK && expectedOK { + return format.MessageWithDiff(actualString, "to equal", expectedString) + } + + return format.Message(actual, "to equal", matcher.Expected) +} + +func (matcher *EqualMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to equal", matcher.Expected) +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_cap_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_cap_matcher.go new file mode 100644 index 000000000..7ace93dc3 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_cap_matcher.go @@ -0,0 +1,28 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type HaveCapMatcher struct { + Count int +} + +func (matcher *HaveCapMatcher) Match(actual interface{}) (success bool, err error) { + length, ok := capOf(actual) + if !ok { + return false, fmt.Errorf("HaveCap matcher expects a array/channel/slice. Got:\n%s", format.Object(actual, 1)) + } + + return length == matcher.Count, nil +} + +func (matcher *HaveCapMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected\n%s\nto have capacity %d", format.Object(actual, 1), matcher.Count) +} + +func (matcher *HaveCapMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected\n%s\nnot to have capacity %d", format.Object(actual, 1), matcher.Count) +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_key_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_key_matcher.go new file mode 100644 index 000000000..ea5b92336 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_key_matcher.go @@ -0,0 +1,54 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type HaveKeyMatcher struct { + Key interface{} +} + +func (matcher *HaveKeyMatcher) Match(actual interface{}) (success bool, err error) { + if !isMap(actual) { + return false, fmt.Errorf("HaveKey matcher expects a map. Got:%s", format.Object(actual, 1)) + } + + keyMatcher, keyIsMatcher := matcher.Key.(omegaMatcher) + if !keyIsMatcher { + keyMatcher = &EqualMatcher{Expected: matcher.Key} + } + + keys := reflect.ValueOf(actual).MapKeys() + for i := 0; i < len(keys); i++ { + success, err := keyMatcher.Match(keys[i].Interface()) + if err != nil { + return false, fmt.Errorf("HaveKey's key matcher failed with:\n%s%s", format.Indent, err.Error()) + } + if success { + return true, nil + } + } + + return false, nil +} + +func (matcher *HaveKeyMatcher) FailureMessage(actual interface{}) (message string) { + switch matcher.Key.(type) { + case omegaMatcher: + return format.Message(actual, "to have key matching", matcher.Key) + default: + return format.Message(actual, "to have key", matcher.Key) + } +} + +func (matcher *HaveKeyMatcher) NegatedFailureMessage(actual interface{}) (message string) { + switch matcher.Key.(type) { + case omegaMatcher: + return format.Message(actual, "not to have key matching", matcher.Key) + default: + return format.Message(actual, "not to have key", matcher.Key) + } +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_key_with_value_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_key_with_value_matcher.go new file mode 100644 index 000000000..06355b1e9 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_key_with_value_matcher.go @@ -0,0 +1,74 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type HaveKeyWithValueMatcher struct { + Key interface{} + Value interface{} +} + +func (matcher *HaveKeyWithValueMatcher) Match(actual interface{}) (success bool, err error) { + if !isMap(actual) { + return false, fmt.Errorf("HaveKeyWithValue matcher expects a map. Got:%s", format.Object(actual, 1)) + } + + keyMatcher, keyIsMatcher := matcher.Key.(omegaMatcher) + if !keyIsMatcher { + keyMatcher = &EqualMatcher{Expected: matcher.Key} + } + + valueMatcher, valueIsMatcher := matcher.Value.(omegaMatcher) + if !valueIsMatcher { + valueMatcher = &EqualMatcher{Expected: matcher.Value} + } + + keys := reflect.ValueOf(actual).MapKeys() + for i := 0; i < len(keys); i++ { + success, err := keyMatcher.Match(keys[i].Interface()) + if err != nil { + return false, fmt.Errorf("HaveKeyWithValue's key matcher failed with:\n%s%s", format.Indent, err.Error()) + } + if success { + actualValue := reflect.ValueOf(actual).MapIndex(keys[i]) + success, err := valueMatcher.Match(actualValue.Interface()) + if err != nil { + return false, fmt.Errorf("HaveKeyWithValue's value matcher failed with:\n%s%s", format.Indent, err.Error()) + } + return success, nil + } + } + + return false, nil +} + +func (matcher *HaveKeyWithValueMatcher) FailureMessage(actual interface{}) (message string) { + str := "to have {key: value}" + if _, ok := matcher.Key.(omegaMatcher); ok { + str += " matching" + } else if _, ok := matcher.Value.(omegaMatcher); ok { + str += " matching" + } + + expect := make(map[interface{}]interface{}, 1) + expect[matcher.Key] = matcher.Value + return format.Message(actual, str, expect) +} + +func (matcher *HaveKeyWithValueMatcher) NegatedFailureMessage(actual interface{}) (message string) { + kStr := "not to have key" + if _, ok := matcher.Key.(omegaMatcher); ok { + kStr = "not to have key matching" + } + + vStr := "or that key's value not be" + if _, ok := matcher.Value.(omegaMatcher); ok { + vStr = "or to have that key's value not matching" + } + + return format.Message(actual, kStr, matcher.Key, vStr, matcher.Value) +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_len_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_len_matcher.go new file mode 100644 index 000000000..ee4276189 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_len_matcher.go @@ -0,0 +1,28 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type HaveLenMatcher struct { + Count int +} + +func (matcher *HaveLenMatcher) Match(actual interface{}) (success bool, err error) { + length, ok := lengthOf(actual) + if !ok { + return false, fmt.Errorf("HaveLen matcher expects a string/array/map/channel/slice. Got:\n%s", format.Object(actual, 1)) + } + + return length == matcher.Count, nil +} + +func (matcher *HaveLenMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected\n%s\nto have length %d", format.Object(actual, 1), matcher.Count) +} + +func (matcher *HaveLenMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected\n%s\nnot to have length %d", format.Object(actual, 1), matcher.Count) +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_occurred_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_occurred_matcher.go new file mode 100644 index 000000000..bef00ae21 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_occurred_matcher.go @@ -0,0 +1,33 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type HaveOccurredMatcher struct { +} + +func (matcher *HaveOccurredMatcher) Match(actual interface{}) (success bool, err error) { + // is purely nil? + if actual == nil { + return false, nil + } + + // must be an 'error' type + if !isError(actual) { + return false, fmt.Errorf("Expected an error-type. Got:\n%s", format.Object(actual, 1)) + } + + // must be non-nil (or a pointer to a non-nil) + return !isNil(actual), nil +} + +func (matcher *HaveOccurredMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected an error to have occurred. Got:\n%s", format.Object(actual, 1)) +} + +func (matcher *HaveOccurredMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Unexpected error:\n%s\n%s\n%s", format.Object(actual, 1), format.IndentString(actual.(error).Error(), 1), "occurred") +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_prefix_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_prefix_matcher.go new file mode 100644 index 000000000..1d8e80270 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_prefix_matcher.go @@ -0,0 +1,36 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type HavePrefixMatcher struct { + Prefix string + Args []interface{} +} + +func (matcher *HavePrefixMatcher) Match(actual interface{}) (success bool, err error) { + actualString, ok := toString(actual) + if !ok { + return false, fmt.Errorf("HavePrefix matcher requires a string or stringer. Got:\n%s", format.Object(actual, 1)) + } + prefix := matcher.prefix() + return len(actualString) >= len(prefix) && actualString[0:len(prefix)] == prefix, nil +} + +func (matcher *HavePrefixMatcher) prefix() string { + if len(matcher.Args) > 0 { + return fmt.Sprintf(matcher.Prefix, matcher.Args...) + } + return matcher.Prefix +} + +func (matcher *HavePrefixMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to have prefix", matcher.prefix()) +} + +func (matcher *HavePrefixMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to have prefix", matcher.prefix()) +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_suffix_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_suffix_matcher.go new file mode 100644 index 000000000..40a3526eb --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_suffix_matcher.go @@ -0,0 +1,36 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type HaveSuffixMatcher struct { + Suffix string + Args []interface{} +} + +func (matcher *HaveSuffixMatcher) Match(actual interface{}) (success bool, err error) { + actualString, ok := toString(actual) + if !ok { + return false, fmt.Errorf("HaveSuffix matcher requires a string or stringer. Got:\n%s", format.Object(actual, 1)) + } + suffix := matcher.suffix() + return len(actualString) >= len(suffix) && actualString[len(actualString)-len(suffix):] == suffix, nil +} + +func (matcher *HaveSuffixMatcher) suffix() string { + if len(matcher.Args) > 0 { + return fmt.Sprintf(matcher.Suffix, matcher.Args...) + } + return matcher.Suffix +} + +func (matcher *HaveSuffixMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to have suffix", matcher.suffix()) +} + +func (matcher *HaveSuffixMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to have suffix", matcher.suffix()) +} diff --git a/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go b/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go new file mode 100644 index 000000000..07499ac95 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go @@ -0,0 +1,51 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type MatchErrorMatcher struct { + Expected interface{} +} + +func (matcher *MatchErrorMatcher) Match(actual interface{}) (success bool, err error) { + if isNil(actual) { + return false, fmt.Errorf("Expected an error, got nil") + } + + if !isError(actual) { + return false, fmt.Errorf("Expected an error. Got:\n%s", format.Object(actual, 1)) + } + + actualErr := actual.(error) + + if isError(matcher.Expected) { + return reflect.DeepEqual(actualErr, matcher.Expected), nil + } + + if isString(matcher.Expected) { + return actualErr.Error() == matcher.Expected, nil + } + + var subMatcher omegaMatcher + var hasSubMatcher bool + if matcher.Expected != nil { + subMatcher, hasSubMatcher = (matcher.Expected).(omegaMatcher) + if hasSubMatcher { + return subMatcher.Match(actualErr.Error()) + } + } + + return false, fmt.Errorf("MatchError must be passed an error, string, or Matcher that can match on strings. Got:\n%s", format.Object(matcher.Expected, 1)) +} + +func (matcher *MatchErrorMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to match error", matcher.Expected) +} + +func (matcher *MatchErrorMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to match error", matcher.Expected) +} diff --git a/vendor/github.com/onsi/gomega/matchers/match_json_matcher.go b/vendor/github.com/onsi/gomega/matchers/match_json_matcher.go new file mode 100644 index 000000000..f962f139f --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/match_json_matcher.go @@ -0,0 +1,65 @@ +package matchers + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/onsi/gomega/format" +) + +type MatchJSONMatcher struct { + JSONToMatch interface{} + firstFailurePath []interface{} +} + +func (matcher *MatchJSONMatcher) Match(actual interface{}) (success bool, err error) { + actualString, expectedString, err := matcher.prettyPrint(actual) + if err != nil { + return false, err + } + + var aval interface{} + var eval interface{} + + // this is guarded by prettyPrint + json.Unmarshal([]byte(actualString), &aval) + json.Unmarshal([]byte(expectedString), &eval) + var equal bool + equal, matcher.firstFailurePath = deepEqual(aval, eval) + return equal, nil +} + +func (matcher *MatchJSONMatcher) FailureMessage(actual interface{}) (message string) { + actualString, expectedString, _ := matcher.prettyPrint(actual) + return formattedMessage(format.Message(actualString, "to match JSON of", expectedString), matcher.firstFailurePath) +} + +func (matcher *MatchJSONMatcher) NegatedFailureMessage(actual interface{}) (message string) { + actualString, expectedString, _ := matcher.prettyPrint(actual) + return formattedMessage(format.Message(actualString, "not to match JSON of", expectedString), matcher.firstFailurePath) +} + +func (matcher *MatchJSONMatcher) prettyPrint(actual interface{}) (actualFormatted, expectedFormatted string, err error) { + actualString, ok := toString(actual) + if !ok { + return "", "", fmt.Errorf("MatchJSONMatcher matcher requires a string, stringer, or []byte. Got actual:\n%s", format.Object(actual, 1)) + } + expectedString, ok := toString(matcher.JSONToMatch) + if !ok { + return "", "", fmt.Errorf("MatchJSONMatcher matcher requires a string, stringer, or []byte. Got expected:\n%s", format.Object(matcher.JSONToMatch, 1)) + } + + abuf := new(bytes.Buffer) + ebuf := new(bytes.Buffer) + + if err := json.Indent(abuf, []byte(actualString), "", " "); err != nil { + return "", "", fmt.Errorf("Actual '%s' should be valid JSON, but it is not.\nUnderlying error:%s", actualString, err) + } + + if err := json.Indent(ebuf, []byte(expectedString), "", " "); err != nil { + return "", "", fmt.Errorf("Expected '%s' should be valid JSON, but it is not.\nUnderlying error:%s", expectedString, err) + } + + return abuf.String(), ebuf.String(), nil +} diff --git a/vendor/github.com/onsi/gomega/matchers/match_regexp_matcher.go b/vendor/github.com/onsi/gomega/matchers/match_regexp_matcher.go new file mode 100644 index 000000000..adac5db6b --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/match_regexp_matcher.go @@ -0,0 +1,43 @@ +package matchers + +import ( + "fmt" + "regexp" + + "github.com/onsi/gomega/format" +) + +type MatchRegexpMatcher struct { + Regexp string + Args []interface{} +} + +func (matcher *MatchRegexpMatcher) Match(actual interface{}) (success bool, err error) { + actualString, ok := toString(actual) + if !ok { + return false, fmt.Errorf("RegExp matcher requires a string or stringer.\nGot:%s", format.Object(actual, 1)) + } + + match, err := regexp.Match(matcher.regexp(), []byte(actualString)) + if err != nil { + return false, fmt.Errorf("RegExp match failed to compile with error:\n\t%s", err.Error()) + } + + return match, nil +} + +func (matcher *MatchRegexpMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to match regular expression", matcher.regexp()) +} + +func (matcher *MatchRegexpMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to match regular expression", matcher.regexp()) +} + +func (matcher *MatchRegexpMatcher) regexp() string { + re := matcher.Regexp + if len(matcher.Args) > 0 { + re = fmt.Sprintf(matcher.Regexp, matcher.Args...) + } + return re +} diff --git a/vendor/github.com/onsi/gomega/matchers/match_xml_matcher.go b/vendor/github.com/onsi/gomega/matchers/match_xml_matcher.go new file mode 100644 index 000000000..3b412ce81 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/match_xml_matcher.go @@ -0,0 +1,134 @@ +package matchers + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "reflect" + "sort" + "strings" + + "github.com/onsi/gomega/format" + "golang.org/x/net/html/charset" +) + +type MatchXMLMatcher struct { + XMLToMatch interface{} +} + +func (matcher *MatchXMLMatcher) Match(actual interface{}) (success bool, err error) { + actualString, expectedString, err := matcher.formattedPrint(actual) + if err != nil { + return false, err + } + + aval, err := parseXmlContent(actualString) + if err != nil { + return false, fmt.Errorf("Actual '%s' should be valid XML, but it is not.\nUnderlying error:%s", actualString, err) + } + + eval, err := parseXmlContent(expectedString) + if err != nil { + return false, fmt.Errorf("Expected '%s' should be valid XML, but it is not.\nUnderlying error:%s", expectedString, err) + } + + return reflect.DeepEqual(aval, eval), nil +} + +func (matcher *MatchXMLMatcher) FailureMessage(actual interface{}) (message string) { + actualString, expectedString, _ := matcher.formattedPrint(actual) + return fmt.Sprintf("Expected\n%s\nto match XML of\n%s", actualString, expectedString) +} + +func (matcher *MatchXMLMatcher) NegatedFailureMessage(actual interface{}) (message string) { + actualString, expectedString, _ := matcher.formattedPrint(actual) + return fmt.Sprintf("Expected\n%s\nnot to match XML of\n%s", actualString, expectedString) +} + +func (matcher *MatchXMLMatcher) formattedPrint(actual interface{}) (actualString, expectedString string, err error) { + var ok bool + actualString, ok = toString(actual) + if !ok { + return "", "", fmt.Errorf("MatchXMLMatcher matcher requires a string, stringer, or []byte. Got actual:\n%s", format.Object(actual, 1)) + } + expectedString, ok = toString(matcher.XMLToMatch) + if !ok { + return "", "", fmt.Errorf("MatchXMLMatcher matcher requires a string, stringer, or []byte. Got expected:\n%s", format.Object(matcher.XMLToMatch, 1)) + } + return actualString, expectedString, nil +} + +func parseXmlContent(content string) (*xmlNode, error) { + allNodes := []*xmlNode{} + + dec := newXmlDecoder(strings.NewReader(content)) + for { + tok, err := dec.Token() + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("failed to decode next token: %v", err) + } + + lastNodeIndex := len(allNodes) - 1 + var lastNode *xmlNode + if len(allNodes) > 0 { + lastNode = allNodes[lastNodeIndex] + } else { + lastNode = &xmlNode{} + } + + switch tok := tok.(type) { + case xml.StartElement: + attrs := attributesSlice(tok.Attr) + sort.Sort(attrs) + allNodes = append(allNodes, &xmlNode{XMLName: tok.Name, XMLAttr: tok.Attr}) + case xml.EndElement: + if len(allNodes) > 1 { + allNodes[lastNodeIndex-1].Nodes = append(allNodes[lastNodeIndex-1].Nodes, lastNode) + allNodes = allNodes[:lastNodeIndex] + } + case xml.CharData: + lastNode.Content = append(lastNode.Content, tok.Copy()...) + case xml.Comment: + lastNode.Comments = append(lastNode.Comments, tok.Copy()) + case xml.ProcInst: + lastNode.ProcInsts = append(lastNode.ProcInsts, tok.Copy()) + } + } + + if len(allNodes) == 0 { + return nil, errors.New("found no nodes") + } + firstNode := allNodes[0] + trimParentNodesContentSpaces(firstNode) + + return firstNode, nil +} + +func newXmlDecoder(reader io.Reader) *xml.Decoder { + dec := xml.NewDecoder(reader) + dec.CharsetReader = charset.NewReaderLabel + return dec +} + +func trimParentNodesContentSpaces(node *xmlNode) { + if len(node.Nodes) > 0 { + node.Content = bytes.TrimSpace(node.Content) + for _, childNode := range node.Nodes { + trimParentNodesContentSpaces(childNode) + } + } +} + +type xmlNode struct { + XMLName xml.Name + Comments []xml.Comment + ProcInsts []xml.ProcInst + XMLAttr []xml.Attr + Content []byte + Nodes []*xmlNode +} diff --git a/vendor/github.com/onsi/gomega/matchers/match_yaml_matcher.go b/vendor/github.com/onsi/gomega/matchers/match_yaml_matcher.go new file mode 100644 index 000000000..0c83c2b63 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/match_yaml_matcher.go @@ -0,0 +1,76 @@ +package matchers + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/format" + "gopkg.in/yaml.v2" +) + +type MatchYAMLMatcher struct { + YAMLToMatch interface{} + firstFailurePath []interface{} +} + +func (matcher *MatchYAMLMatcher) Match(actual interface{}) (success bool, err error) { + actualString, expectedString, err := matcher.toStrings(actual) + if err != nil { + return false, err + } + + var aval interface{} + var eval interface{} + + if err := yaml.Unmarshal([]byte(actualString), &aval); err != nil { + return false, fmt.Errorf("Actual '%s' should be valid YAML, but it is not.\nUnderlying error:%s", actualString, err) + } + if err := yaml.Unmarshal([]byte(expectedString), &eval); err != nil { + return false, fmt.Errorf("Expected '%s' should be valid YAML, but it is not.\nUnderlying error:%s", expectedString, err) + } + + var equal bool + equal, matcher.firstFailurePath = deepEqual(aval, eval) + return equal, nil +} + +func (matcher *MatchYAMLMatcher) FailureMessage(actual interface{}) (message string) { + actualString, expectedString, _ := matcher.toNormalisedStrings(actual) + return formattedMessage(format.Message(actualString, "to match YAML of", expectedString), matcher.firstFailurePath) +} + +func (matcher *MatchYAMLMatcher) NegatedFailureMessage(actual interface{}) (message string) { + actualString, expectedString, _ := matcher.toNormalisedStrings(actual) + return formattedMessage(format.Message(actualString, "not to match YAML of", expectedString), matcher.firstFailurePath) +} + +func (matcher *MatchYAMLMatcher) toNormalisedStrings(actual interface{}) (actualFormatted, expectedFormatted string, err error) { + actualString, expectedString, err := matcher.toStrings(actual) + return normalise(actualString), normalise(expectedString), err +} + +func normalise(input string) string { + var val interface{} + err := yaml.Unmarshal([]byte(input), &val) + if err != nil { + panic(err) // unreachable since Match already calls Unmarshal + } + output, err := yaml.Marshal(val) + if err != nil { + panic(err) // untested section, unreachable since we Unmarshal above + } + return strings.TrimSpace(string(output)) +} + +func (matcher *MatchYAMLMatcher) toStrings(actual interface{}) (actualFormatted, expectedFormatted string, err error) { + actualString, ok := toString(actual) + if !ok { + return "", "", fmt.Errorf("MatchYAMLMatcher matcher requires a string, stringer, or []byte. Got actual:\n%s", format.Object(actual, 1)) + } + expectedString, ok := toString(matcher.YAMLToMatch) + if !ok { + return "", "", fmt.Errorf("MatchYAMLMatcher matcher requires a string, stringer, or []byte. Got expected:\n%s", format.Object(matcher.YAMLToMatch, 1)) + } + + return actualString, expectedString, nil +} diff --git a/vendor/github.com/onsi/gomega/matchers/not.go b/vendor/github.com/onsi/gomega/matchers/not.go new file mode 100644 index 000000000..2c91670bd --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/not.go @@ -0,0 +1,30 @@ +package matchers + +import ( + "github.com/onsi/gomega/internal/oraclematcher" + "github.com/onsi/gomega/types" +) + +type NotMatcher struct { + Matcher types.GomegaMatcher +} + +func (m *NotMatcher) Match(actual interface{}) (bool, error) { + success, err := m.Matcher.Match(actual) + if err != nil { + return false, err + } + return !success, nil +} + +func (m *NotMatcher) FailureMessage(actual interface{}) (message string) { + return m.Matcher.NegatedFailureMessage(actual) // works beautifully +} + +func (m *NotMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return m.Matcher.FailureMessage(actual) // works beautifully +} + +func (m *NotMatcher) MatchMayChangeInTheFuture(actual interface{}) bool { + return oraclematcher.MatchMayChangeInTheFuture(m.Matcher, actual) // just return m.Matcher's value +} diff --git a/vendor/github.com/onsi/gomega/matchers/or.go b/vendor/github.com/onsi/gomega/matchers/or.go new file mode 100644 index 000000000..3bf799800 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/or.go @@ -0,0 +1,67 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/internal/oraclematcher" + "github.com/onsi/gomega/types" +) + +type OrMatcher struct { + Matchers []types.GomegaMatcher + + // state + firstSuccessfulMatcher types.GomegaMatcher +} + +func (m *OrMatcher) Match(actual interface{}) (success bool, err error) { + m.firstSuccessfulMatcher = nil + for _, matcher := range m.Matchers { + success, err := matcher.Match(actual) + if err != nil { + return false, err + } + if success { + m.firstSuccessfulMatcher = matcher + return true, nil + } + } + return false, nil +} + +func (m *OrMatcher) FailureMessage(actual interface{}) (message string) { + // not the most beautiful list of matchers, but not bad either... + return format.Message(actual, fmt.Sprintf("To satisfy at least one of these matchers: %s", m.Matchers)) +} + +func (m *OrMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return m.firstSuccessfulMatcher.NegatedFailureMessage(actual) +} + +func (m *OrMatcher) MatchMayChangeInTheFuture(actual interface{}) bool { + /* + Example with 3 matchers: A, B, C + + Match evaluates them: F, T, => T + So match is currently T, what should MatchMayChangeInTheFuture() return? + Seems like it only depends on B, since currently B MUST change to allow the result to become F + + Match eval: F, F, F => F + So match is currently F, what should MatchMayChangeInTheFuture() return? + Seems to depend on ANY of them being able to change to T. + */ + + if m.firstSuccessfulMatcher != nil { + // one of the matchers succeeded.. it must be able to change in order to affect the result + return oraclematcher.MatchMayChangeInTheFuture(m.firstSuccessfulMatcher, actual) + } else { + // so all matchers failed.. Any one of them changing would change the result. + for _, matcher := range m.Matchers { + if oraclematcher.MatchMayChangeInTheFuture(matcher, actual) { + return true + } + } + return false // none of were going to change + } +} diff --git a/vendor/github.com/onsi/gomega/matchers/panic_matcher.go b/vendor/github.com/onsi/gomega/matchers/panic_matcher.go new file mode 100644 index 000000000..640f4db1a --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/panic_matcher.go @@ -0,0 +1,46 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type PanicMatcher struct { + object interface{} +} + +func (matcher *PanicMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil { + return false, fmt.Errorf("PanicMatcher expects a non-nil actual.") + } + + actualType := reflect.TypeOf(actual) + if actualType.Kind() != reflect.Func { + return false, fmt.Errorf("PanicMatcher expects a function. Got:\n%s", format.Object(actual, 1)) + } + if !(actualType.NumIn() == 0 && actualType.NumOut() == 0) { + return false, fmt.Errorf("PanicMatcher expects a function with no arguments and no return value. Got:\n%s", format.Object(actual, 1)) + } + + success = false + defer func() { + if e := recover(); e != nil { + matcher.object = e + success = true + } + }() + + reflect.ValueOf(actual).Call([]reflect.Value{}) + + return +} + +func (matcher *PanicMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to panic") +} + +func (matcher *PanicMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not to panic, but panicked with\n%s", format.Object(matcher.object, 1))) +} diff --git a/vendor/github.com/onsi/gomega/matchers/receive_matcher.go b/vendor/github.com/onsi/gomega/matchers/receive_matcher.go new file mode 100644 index 000000000..2018a6128 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/receive_matcher.go @@ -0,0 +1,128 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type ReceiveMatcher struct { + Arg interface{} + receivedValue reflect.Value + channelClosed bool +} + +func (matcher *ReceiveMatcher) Match(actual interface{}) (success bool, err error) { + if !isChan(actual) { + return false, fmt.Errorf("ReceiveMatcher expects a channel. Got:\n%s", format.Object(actual, 1)) + } + + channelType := reflect.TypeOf(actual) + channelValue := reflect.ValueOf(actual) + + if channelType.ChanDir() == reflect.SendDir { + return false, fmt.Errorf("ReceiveMatcher matcher cannot be passed a send-only channel. Got:\n%s", format.Object(actual, 1)) + } + + var subMatcher omegaMatcher + var hasSubMatcher bool + + if matcher.Arg != nil { + subMatcher, hasSubMatcher = (matcher.Arg).(omegaMatcher) + if !hasSubMatcher { + argType := reflect.TypeOf(matcher.Arg) + if argType.Kind() != reflect.Ptr { + return false, fmt.Errorf("Cannot assign a value from the channel:\n%s\nTo:\n%s\nYou need to pass a pointer!", format.Object(actual, 1), format.Object(matcher.Arg, 1)) + } + } + } + + winnerIndex, value, open := reflect.Select([]reflect.SelectCase{ + {Dir: reflect.SelectRecv, Chan: channelValue}, + {Dir: reflect.SelectDefault}, + }) + + var closed bool + var didReceive bool + if winnerIndex == 0 { + closed = !open + didReceive = open + } + matcher.channelClosed = closed + + if closed { + return false, nil + } + + if hasSubMatcher { + if didReceive { + matcher.receivedValue = value + return subMatcher.Match(matcher.receivedValue.Interface()) + } + return false, nil + } + + if didReceive { + if matcher.Arg != nil { + outValue := reflect.ValueOf(matcher.Arg) + + if value.Type().AssignableTo(outValue.Elem().Type()) { + outValue.Elem().Set(value) + return true, nil + } + if value.Type().Kind() == reflect.Interface && value.Elem().Type().AssignableTo(outValue.Elem().Type()) { + outValue.Elem().Set(value.Elem()) + return true, nil + } else { + return false, fmt.Errorf("Cannot assign a value from the channel:\n%s\nType:\n%s\nTo:\n%s", format.Object(actual, 1), format.Object(value.Interface(), 1), format.Object(matcher.Arg, 1)) + } + + } + + return true, nil + } + return false, nil +} + +func (matcher *ReceiveMatcher) FailureMessage(actual interface{}) (message string) { + subMatcher, hasSubMatcher := (matcher.Arg).(omegaMatcher) + + closedAddendum := "" + if matcher.channelClosed { + closedAddendum = " The channel is closed." + } + + if hasSubMatcher { + if matcher.receivedValue.IsValid() { + return subMatcher.FailureMessage(matcher.receivedValue.Interface()) + } + return "When passed a matcher, ReceiveMatcher's channel *must* receive something." + } + return format.Message(actual, "to receive something."+closedAddendum) +} + +func (matcher *ReceiveMatcher) NegatedFailureMessage(actual interface{}) (message string) { + subMatcher, hasSubMatcher := (matcher.Arg).(omegaMatcher) + + closedAddendum := "" + if matcher.channelClosed { + closedAddendum = " The channel is closed." + } + + if hasSubMatcher { + if matcher.receivedValue.IsValid() { + return subMatcher.NegatedFailureMessage(matcher.receivedValue.Interface()) + } + return "When passed a matcher, ReceiveMatcher's channel *must* receive something." + } + return format.Message(actual, "not to receive anything."+closedAddendum) +} + +func (matcher *ReceiveMatcher) MatchMayChangeInTheFuture(actual interface{}) bool { + if !isChan(actual) { + return false + } + + return !matcher.channelClosed +} diff --git a/vendor/github.com/onsi/gomega/matchers/semi_structured_data_support.go b/vendor/github.com/onsi/gomega/matchers/semi_structured_data_support.go new file mode 100644 index 000000000..639295684 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/semi_structured_data_support.go @@ -0,0 +1,92 @@ +package matchers + +import ( + "fmt" + "reflect" + "strings" +) + +func formattedMessage(comparisonMessage string, failurePath []interface{}) string { + var diffMessage string + if len(failurePath) == 0 { + diffMessage = "" + } else { + diffMessage = fmt.Sprintf("\n\nfirst mismatched key: %s", formattedFailurePath(failurePath)) + } + return fmt.Sprintf("%s%s", comparisonMessage, diffMessage) +} + +func formattedFailurePath(failurePath []interface{}) string { + formattedPaths := []string{} + for i := len(failurePath) - 1; i >= 0; i-- { + switch p := failurePath[i].(type) { + case int: + formattedPaths = append(formattedPaths, fmt.Sprintf(`[%d]`, p)) + default: + if i != len(failurePath)-1 { + formattedPaths = append(formattedPaths, ".") + } + formattedPaths = append(formattedPaths, fmt.Sprintf(`"%s"`, p)) + } + } + return strings.Join(formattedPaths, "") +} + +func deepEqual(a interface{}, b interface{}) (bool, []interface{}) { + var errorPath []interface{} + if reflect.TypeOf(a) != reflect.TypeOf(b) { + return false, errorPath + } + + switch a.(type) { + case []interface{}: + if len(a.([]interface{})) != len(b.([]interface{})) { + return false, errorPath + } + + for i, v := range a.([]interface{}) { + elementEqual, keyPath := deepEqual(v, b.([]interface{})[i]) + if !elementEqual { + return false, append(keyPath, i) + } + } + return true, errorPath + + case map[interface{}]interface{}: + if len(a.(map[interface{}]interface{})) != len(b.(map[interface{}]interface{})) { + return false, errorPath + } + + for k, v1 := range a.(map[interface{}]interface{}) { + v2, ok := b.(map[interface{}]interface{})[k] + if !ok { + return false, errorPath + } + elementEqual, keyPath := deepEqual(v1, v2) + if !elementEqual { + return false, append(keyPath, k) + } + } + return true, errorPath + + case map[string]interface{}: + if len(a.(map[string]interface{})) != len(b.(map[string]interface{})) { + return false, errorPath + } + + for k, v1 := range a.(map[string]interface{}) { + v2, ok := b.(map[string]interface{})[k] + if !ok { + return false, errorPath + } + elementEqual, keyPath := deepEqual(v1, v2) + if !elementEqual { + return false, append(keyPath, k) + } + } + return true, errorPath + + default: + return a == b, errorPath + } +} diff --git a/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go b/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go new file mode 100644 index 000000000..721ed5529 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go @@ -0,0 +1,33 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type SucceedMatcher struct { +} + +func (matcher *SucceedMatcher) Match(actual interface{}) (success bool, err error) { + // is purely nil? + if actual == nil { + return true, nil + } + + // must be an 'error' type + if !isError(actual) { + return false, fmt.Errorf("Expected an error-type. Got:\n%s", format.Object(actual, 1)) + } + + // must be nil (or a pointer to a nil) + return isNil(actual), nil +} + +func (matcher *SucceedMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("Expected success, but got an error:\n%s\n%s", format.Object(actual, 1), format.IndentString(actual.(error).Error(), 1)) +} + +func (matcher *SucceedMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return "Expected failure, but got no error." +} diff --git a/vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraph.go b/vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraph.go new file mode 100644 index 000000000..8aaf8759d --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraph.go @@ -0,0 +1,41 @@ +package bipartitegraph + +import "errors" +import "fmt" + +import . "github.com/onsi/gomega/matchers/support/goraph/node" +import . "github.com/onsi/gomega/matchers/support/goraph/edge" + +type BipartiteGraph struct { + Left NodeOrderedSet + Right NodeOrderedSet + Edges EdgeSet +} + +func NewBipartiteGraph(leftValues, rightValues []interface{}, neighbours func(interface{}, interface{}) (bool, error)) (*BipartiteGraph, error) { + left := NodeOrderedSet{} + for i := range leftValues { + left = append(left, Node{Id: i}) + } + + right := NodeOrderedSet{} + for j := range rightValues { + right = append(right, Node{Id: j + len(left)}) + } + + edges := EdgeSet{} + for i, leftValue := range leftValues { + for j, rightValue := range rightValues { + neighbours, err := neighbours(leftValue, rightValue) + if err != nil { + return nil, errors.New(fmt.Sprintf("error determining adjacency for %v and %v: %s", leftValue, rightValue, err.Error())) + } + + if neighbours { + edges = append(edges, Edge{Node1: left[i], Node2: right[j]}) + } + } + } + + return &BipartiteGraph{left, right, edges}, nil +} diff --git a/vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraphmatching.go b/vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraphmatching.go new file mode 100644 index 000000000..8181f43a4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/support/goraph/bipartitegraph/bipartitegraphmatching.go @@ -0,0 +1,159 @@ +package bipartitegraph + +import . "github.com/onsi/gomega/matchers/support/goraph/node" +import . "github.com/onsi/gomega/matchers/support/goraph/edge" +import "github.com/onsi/gomega/matchers/support/goraph/util" + +func (bg *BipartiteGraph) LargestMatching() (matching EdgeSet) { + paths := bg.maximalDisjointSLAPCollection(matching) + + for len(paths) > 0 { + for _, path := range paths { + matching = matching.SymmetricDifference(path) + } + paths = bg.maximalDisjointSLAPCollection(matching) + } + + return +} + +func (bg *BipartiteGraph) maximalDisjointSLAPCollection(matching EdgeSet) (result []EdgeSet) { + guideLayers := bg.createSLAPGuideLayers(matching) + if len(guideLayers) == 0 { + return + } + + used := make(map[Node]bool) + + for _, u := range guideLayers[len(guideLayers)-1] { + slap, found := bg.findDisjointSLAP(u, matching, guideLayers, used) + if found { + for _, edge := range slap { + used[edge.Node1] = true + used[edge.Node2] = true + } + result = append(result, slap) + } + } + + return +} + +func (bg *BipartiteGraph) findDisjointSLAP( + start Node, + matching EdgeSet, + guideLayers []NodeOrderedSet, + used map[Node]bool, +) ([]Edge, bool) { + return bg.findDisjointSLAPHelper(start, EdgeSet{}, len(guideLayers)-1, matching, guideLayers, used) +} + +func (bg *BipartiteGraph) findDisjointSLAPHelper( + currentNode Node, + currentSLAP EdgeSet, + currentLevel int, + matching EdgeSet, + guideLayers []NodeOrderedSet, + used map[Node]bool, +) (EdgeSet, bool) { + used[currentNode] = true + + if currentLevel == 0 { + return currentSLAP, true + } + + for _, nextNode := range guideLayers[currentLevel-1] { + if used[nextNode] { + continue + } + + edge, found := bg.Edges.FindByNodes(currentNode, nextNode) + if !found { + continue + } + + if matching.Contains(edge) == util.Odd(currentLevel) { + continue + } + + currentSLAP = append(currentSLAP, edge) + slap, found := bg.findDisjointSLAPHelper(nextNode, currentSLAP, currentLevel-1, matching, guideLayers, used) + if found { + return slap, true + } + currentSLAP = currentSLAP[:len(currentSLAP)-1] + } + + used[currentNode] = false + return nil, false +} + +func (bg *BipartiteGraph) createSLAPGuideLayers(matching EdgeSet) (guideLayers []NodeOrderedSet) { + used := make(map[Node]bool) + currentLayer := NodeOrderedSet{} + + for _, node := range bg.Left { + if matching.Free(node) { + used[node] = true + currentLayer = append(currentLayer, node) + } + } + + if len(currentLayer) == 0 { + return []NodeOrderedSet{} + } + guideLayers = append(guideLayers, currentLayer) + + done := false + + for !done { + lastLayer := currentLayer + currentLayer = NodeOrderedSet{} + + if util.Odd(len(guideLayers)) { + for _, leftNode := range lastLayer { + for _, rightNode := range bg.Right { + if used[rightNode] { + continue + } + + edge, found := bg.Edges.FindByNodes(leftNode, rightNode) + if !found || matching.Contains(edge) { + continue + } + + currentLayer = append(currentLayer, rightNode) + used[rightNode] = true + + if matching.Free(rightNode) { + done = true + } + } + } + } else { + for _, rightNode := range lastLayer { + for _, leftNode := range bg.Left { + if used[leftNode] { + continue + } + + edge, found := bg.Edges.FindByNodes(leftNode, rightNode) + if !found || !matching.Contains(edge) { + continue + } + + currentLayer = append(currentLayer, leftNode) + used[leftNode] = true + } + } + + } + + if len(currentLayer) == 0 { + return []NodeOrderedSet{} + } + guideLayers = append(guideLayers, currentLayer) + } + + return +} diff --git a/vendor/github.com/onsi/gomega/matchers/support/goraph/edge/edge.go b/vendor/github.com/onsi/gomega/matchers/support/goraph/edge/edge.go new file mode 100644 index 000000000..4fd15cc06 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/support/goraph/edge/edge.go @@ -0,0 +1,61 @@ +package edge + +import . "github.com/onsi/gomega/matchers/support/goraph/node" + +type Edge struct { + Node1 Node + Node2 Node +} + +type EdgeSet []Edge + +func (ec EdgeSet) Free(node Node) bool { + for _, e := range ec { + if e.Node1 == node || e.Node2 == node { + return false + } + } + + return true +} + +func (ec EdgeSet) Contains(edge Edge) bool { + for _, e := range ec { + if e == edge { + return true + } + } + + return false +} + +func (ec EdgeSet) FindByNodes(node1, node2 Node) (Edge, bool) { + for _, e := range ec { + if (e.Node1 == node1 && e.Node2 == node2) || (e.Node1 == node2 && e.Node2 == node1) { + return e, true + } + } + + return Edge{}, false +} + +func (ec EdgeSet) SymmetricDifference(ec2 EdgeSet) EdgeSet { + edgesToInclude := make(map[Edge]bool) + + for _, e := range ec { + edgesToInclude[e] = true + } + + for _, e := range ec2 { + edgesToInclude[e] = !edgesToInclude[e] + } + + result := EdgeSet{} + for e, include := range edgesToInclude { + if include { + result = append(result, e) + } + } + + return result +} diff --git a/vendor/github.com/onsi/gomega/matchers/support/goraph/node/node.go b/vendor/github.com/onsi/gomega/matchers/support/goraph/node/node.go new file mode 100644 index 000000000..800c2ea8c --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/support/goraph/node/node.go @@ -0,0 +1,7 @@ +package node + +type Node struct { + Id int +} + +type NodeOrderedSet []Node diff --git a/vendor/github.com/onsi/gomega/matchers/support/goraph/util/util.go b/vendor/github.com/onsi/gomega/matchers/support/goraph/util/util.go new file mode 100644 index 000000000..d76a1ee00 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/support/goraph/util/util.go @@ -0,0 +1,7 @@ +package util + +import "math" + +func Odd(n int) bool { + return math.Mod(float64(n), 2.0) == 1.0 +} diff --git a/vendor/github.com/onsi/gomega/matchers/type_support.go b/vendor/github.com/onsi/gomega/matchers/type_support.go new file mode 100644 index 000000000..75afcd844 --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/type_support.go @@ -0,0 +1,179 @@ +/* +Gomega matchers + +This package implements the Gomega matchers and does not typically need to be imported. +See the docs for Gomega for documentation on the matchers + +http://onsi.github.io/gomega/ +*/ +package matchers + +import ( + "encoding/json" + "fmt" + "reflect" +) + +type omegaMatcher interface { + Match(actual interface{}) (success bool, err error) + FailureMessage(actual interface{}) (message string) + NegatedFailureMessage(actual interface{}) (message string) +} + +func isBool(a interface{}) bool { + return reflect.TypeOf(a).Kind() == reflect.Bool +} + +func isNumber(a interface{}) bool { + if a == nil { + return false + } + kind := reflect.TypeOf(a).Kind() + return reflect.Int <= kind && kind <= reflect.Float64 +} + +func isInteger(a interface{}) bool { + kind := reflect.TypeOf(a).Kind() + return reflect.Int <= kind && kind <= reflect.Int64 +} + +func isUnsignedInteger(a interface{}) bool { + kind := reflect.TypeOf(a).Kind() + return reflect.Uint <= kind && kind <= reflect.Uint64 +} + +func isFloat(a interface{}) bool { + kind := reflect.TypeOf(a).Kind() + return reflect.Float32 <= kind && kind <= reflect.Float64 +} + +func toInteger(a interface{}) int64 { + if isInteger(a) { + return reflect.ValueOf(a).Int() + } else if isUnsignedInteger(a) { + return int64(reflect.ValueOf(a).Uint()) + } else if isFloat(a) { + return int64(reflect.ValueOf(a).Float()) + } + panic(fmt.Sprintf("Expected a number! Got <%T> %#v", a, a)) +} + +func toUnsignedInteger(a interface{}) uint64 { + if isInteger(a) { + return uint64(reflect.ValueOf(a).Int()) + } else if isUnsignedInteger(a) { + return reflect.ValueOf(a).Uint() + } else if isFloat(a) { + return uint64(reflect.ValueOf(a).Float()) + } + panic(fmt.Sprintf("Expected a number! Got <%T> %#v", a, a)) +} + +func toFloat(a interface{}) float64 { + if isInteger(a) { + return float64(reflect.ValueOf(a).Int()) + } else if isUnsignedInteger(a) { + return float64(reflect.ValueOf(a).Uint()) + } else if isFloat(a) { + return reflect.ValueOf(a).Float() + } + panic(fmt.Sprintf("Expected a number! Got <%T> %#v", a, a)) +} + +func isError(a interface{}) bool { + _, ok := a.(error) + return ok +} + +func isChan(a interface{}) bool { + if isNil(a) { + return false + } + return reflect.TypeOf(a).Kind() == reflect.Chan +} + +func isMap(a interface{}) bool { + if a == nil { + return false + } + return reflect.TypeOf(a).Kind() == reflect.Map +} + +func isArrayOrSlice(a interface{}) bool { + if a == nil { + return false + } + switch reflect.TypeOf(a).Kind() { + case reflect.Array, reflect.Slice: + return true + default: + return false + } +} + +func isString(a interface{}) bool { + if a == nil { + return false + } + return reflect.TypeOf(a).Kind() == reflect.String +} + +func toString(a interface{}) (string, bool) { + aString, isString := a.(string) + if isString { + return aString, true + } + + aBytes, isBytes := a.([]byte) + if isBytes { + return string(aBytes), true + } + + aStringer, isStringer := a.(fmt.Stringer) + if isStringer { + return aStringer.String(), true + } + + aJSONRawMessage, isJSONRawMessage := a.(json.RawMessage) + if isJSONRawMessage { + return string(aJSONRawMessage), true + } + + return "", false +} + +func lengthOf(a interface{}) (int, bool) { + if a == nil { + return 0, false + } + switch reflect.TypeOf(a).Kind() { + case reflect.Map, reflect.Array, reflect.String, reflect.Chan, reflect.Slice: + return reflect.ValueOf(a).Len(), true + default: + return 0, false + } +} +func capOf(a interface{}) (int, bool) { + if a == nil { + return 0, false + } + switch reflect.TypeOf(a).Kind() { + case reflect.Array, reflect.Chan, reflect.Slice: + return reflect.ValueOf(a).Cap(), true + default: + return 0, false + } +} + +func isNil(a interface{}) bool { + if a == nil { + return true + } + + switch reflect.TypeOf(a).Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return reflect.ValueOf(a).IsNil() + } + + return false +} diff --git a/vendor/github.com/onsi/gomega/matchers/with_transform.go b/vendor/github.com/onsi/gomega/matchers/with_transform.go new file mode 100644 index 000000000..8e58d8a0f --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/with_transform.go @@ -0,0 +1,72 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/internal/oraclematcher" + "github.com/onsi/gomega/types" +) + +type WithTransformMatcher struct { + // input + Transform interface{} // must be a function of one parameter that returns one value + Matcher types.GomegaMatcher + + // cached value + transformArgType reflect.Type + + // state + transformedValue interface{} +} + +func NewWithTransformMatcher(transform interface{}, matcher types.GomegaMatcher) *WithTransformMatcher { + if transform == nil { + panic("transform function cannot be nil") + } + txType := reflect.TypeOf(transform) + if txType.NumIn() != 1 { + panic("transform function must have 1 argument") + } + if txType.NumOut() != 1 { + panic("transform function must have 1 return value") + } + + return &WithTransformMatcher{ + Transform: transform, + Matcher: matcher, + transformArgType: reflect.TypeOf(transform).In(0), + } +} + +func (m *WithTransformMatcher) Match(actual interface{}) (bool, error) { + // return error if actual's type is incompatible with Transform function's argument type + actualType := reflect.TypeOf(actual) + if !actualType.AssignableTo(m.transformArgType) { + return false, fmt.Errorf("Transform function expects '%s' but we have '%s'", m.transformArgType, actualType) + } + + // call the Transform function with `actual` + fn := reflect.ValueOf(m.Transform) + result := fn.Call([]reflect.Value{reflect.ValueOf(actual)}) + m.transformedValue = result[0].Interface() // expect exactly one value + + return m.Matcher.Match(m.transformedValue) +} + +func (m *WithTransformMatcher) FailureMessage(_ interface{}) (message string) { + return m.Matcher.FailureMessage(m.transformedValue) +} + +func (m *WithTransformMatcher) NegatedFailureMessage(_ interface{}) (message string) { + return m.Matcher.NegatedFailureMessage(m.transformedValue) +} + +func (m *WithTransformMatcher) MatchMayChangeInTheFuture(_ interface{}) bool { + // TODO: Maybe this should always just return true? (Only an issue for non-deterministic transformers.) + // + // Querying the next matcher is fine if the transformer always will return the same value. + // But if the transformer is non-deterministic and returns a different value each time, then there + // is no point in querying the next matcher, since it can only comment on the last transformed value. + return oraclematcher.MatchMayChangeInTheFuture(m.Matcher, m.transformedValue) +} diff --git a/vendor/github.com/onsi/gomega/types/types.go b/vendor/github.com/onsi/gomega/types/types.go new file mode 100644 index 000000000..ac59a3a5a --- /dev/null +++ b/vendor/github.com/onsi/gomega/types/types.go @@ -0,0 +1,26 @@ +package types + +type TWithHelper interface { + Helper() +} + +type GomegaFailHandler func(message string, callerSkip ...int) + +type GomegaFailWrapper struct { + Fail GomegaFailHandler + TWithHelper TWithHelper +} + +//A simple *testing.T interface wrapper +type GomegaTestingT interface { + Fatalf(format string, args ...interface{}) +} + +//All Gomega matchers must implement the GomegaMatcher interface +// +//For details on writing custom matchers, check out: http://onsi.github.io/gomega/#adding-your-own-matchers +type GomegaMatcher interface { + Match(actual interface{}) (success bool, err error) + FailureMessage(actual interface{}) (message string) + NegatedFailureMessage(actual interface{}) (message string) +} -- 2.45.2 From f486e20e7b93101305ae0d3e5ac4d0f28ba41970 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Wed, 30 Jan 2019 17:30:34 -0600 Subject: [PATCH 02/17] Apply go fmt changes to statediff --- cmd/utils/flags.go | 6 +++--- statediff/builder/builder.go | 4 ++-- statediff/builder/builder_test.go | 24 ++++++++++++------------ statediff/builder/struct.go | 2 +- statediff/publisher/csv.go | 2 +- statediff/testhelpers/test_data.go | 4 ++-- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 2beeb2900..0b5c0c955 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -759,18 +759,18 @@ var ( } StateDiffFlag = cli.BoolFlag{ - Name: "statediff", + Name: "statediff", Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", } StateDiffModeFlag = cli.StringFlag{ - Name: "statediff.mode", + Name: "statediff.mode", Usage: "Enables the user to determine which persistence mode they'd like to store the state diffs in.", Value: "csv", } StateDiffPathFlag = cli.StringFlag{ - Name: "statediff.path", + Name: "statediff.path", Usage: "Enables the user to determine where to persist the state diffs.", Value: ".", } diff --git a/statediff/builder/builder.go b/statediff/builder/builder.go index 825869150..1d66db380 100644 --- a/statediff/builder/builder.go +++ b/statediff/builder/builder.go @@ -113,7 +113,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address it, _ := trie.NewDifferenceIterator(a, b) for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", common.Hash(it.Hash())) + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) if it.Leaf() { // lookup address @@ -242,7 +242,7 @@ func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage { if it.Leaf() { log.Debug("Found leaf in storage", "path", pathToStr(it)) path := pathToStr(it) - storageKey:= hexutil.Encode(it.LeafKey()) + storageKey := hexutil.Encode(it.LeafKey()) storageValue := hexutil.Encode(it.LeafBlob()) storageDiffs[path] = DiffStorage{ Key: &storageKey, diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go index 72ebfff18..6354a13e2 100644 --- a/statediff/builder/builder_test.go +++ b/statediff/builder/builder_test.go @@ -23,14 +23,14 @@ var ( testBankFunds = big.NewInt(100000000) genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) - account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 - account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e - contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") - contractAddr common.Address - emptyAccountDiffEventualMap = make(map[common.Address]b.AccountDiff) - emptyAccountDiffIncrementalMap = make(map[common.Address]b.AccountDiff) + account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") + contractAddr common.Address + emptyAccountDiffEventualMap = make(map[common.Address]b.AccountDiff) + emptyAccountDiffIncrementalMap = make(map[common.Address]b.AccountDiff) block0Hash, block1Hash, block2Hash, block3Hash common.Hash block0, block1, block2, block3 *types.Block builder b.Builder @@ -73,10 +73,10 @@ func TestBuilder(t *testing.T) { newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" originalStorageLocation = common.HexToHash("0") originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).String() - updatedStorageLocation = common.HexToHash("2") - updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).String() - originalStorageValue = "0x01" - updatedStorageValue = "0x03" + updatedStorageLocation = common.HexToHash("2") + updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).String() + originalStorageValue = "0x01" + updatedStorageValue = "0x03" ) var tests = []struct { diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go index e3ad599f1..295b09f10 100644 --- a/statediff/builder/struct.go +++ b/statediff/builder/struct.go @@ -64,7 +64,7 @@ type AccountDiff struct { } type DiffStorage struct { - Key *string `json:"key" gencodec:"optional"` + Key *string `json:"key" gencodec:"optional"` Value *string `json:"value" gencodec:"optional"` } type DiffString struct { diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go index 82eb3e466..8be88ed7f 100644 --- a/statediff/publisher/csv.go +++ b/statediff/publisher/csv.go @@ -2,12 +2,12 @@ package publisher import ( "encoding/csv" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff/builder" "os" "path/filepath" "strconv" "time" - "github.com/ethereum/go-ethereum/common" ) var ( diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index e7d94b373..34194d2f4 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -21,12 +21,12 @@ var ( Key: &StorageKey, Value: &StorageValue, }} - emptyStorage = map[string]builder.DiffStorage{} + emptyStorage = map[string]builder.DiffStorage{} address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") anotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") ContractAddress = address.String() AnotherContractAddress = anotherAddress.String() - CreatedAccountDiffs = map[common.Address]builder.AccountDiff{ + CreatedAccountDiffs = map[common.Address]builder.AccountDiff{ address: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, -- 2.45.2 From ba8abf407f9cf5eaa6339feda8daeaa92c7f7c42 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 31 Jan 2019 08:18:11 -0600 Subject: [PATCH 03/17] Apply goimports to statediff --- statediff/builder/builder_test.go | 7 ++++--- statediff/config_test.go | 3 ++- statediff/extractor/extractor_test.go | 9 +++++---- statediff/publisher/csv.go | 5 +++-- statediff/publisher/publisher_test.go | 11 ++++++----- statediff/service/service_test.go | 9 +++++---- statediff/testhelpers/test_data.go | 5 +++-- 7 files changed, 28 insertions(+), 21 deletions(-) diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go index 6354a13e2..95163100b 100644 --- a/statediff/builder/builder_test.go +++ b/statediff/builder/builder_test.go @@ -2,6 +2,10 @@ package builder_test import ( "bytes" + "math/big" + "reflect" + "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" @@ -10,9 +14,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" b "github.com/ethereum/go-ethereum/statediff/builder" - "math/big" - "reflect" - "testing" ) var ( diff --git a/statediff/config_test.go b/statediff/config_test.go index 82f2d3a92..5246d1cd1 100644 --- a/statediff/config_test.go +++ b/statediff/config_test.go @@ -1,9 +1,10 @@ package statediff_test import ( + "testing" + "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff/testhelpers" - "testing" ) func TestNewMode(t *testing.T) { diff --git a/statediff/extractor/extractor_test.go b/statediff/extractor/extractor_test.go index b75ea60b0..0ed036c12 100644 --- a/statediff/extractor/extractor_test.go +++ b/statediff/extractor/extractor_test.go @@ -2,14 +2,15 @@ package extractor_test import ( "bytes" - "github.com/ethereum/go-ethereum/core/types" - b "github.com/ethereum/go-ethereum/statediff/builder" - e "github.com/ethereum/go-ethereum/statediff/extractor" - "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" "math/big" "math/rand" "reflect" "testing" + + "github.com/ethereum/go-ethereum/core/types" + b "github.com/ethereum/go-ethereum/statediff/builder" + e "github.com/ethereum/go-ethereum/statediff/extractor" + "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" ) var publisher mocks.Publisher diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go index 8be88ed7f..ce405ac14 100644 --- a/statediff/publisher/csv.go +++ b/statediff/publisher/csv.go @@ -2,12 +2,13 @@ package publisher import ( "encoding/csv" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/builder" "os" "path/filepath" "strconv" "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/builder" ) var ( diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go index 4319b7bd7..8a26fb4e2 100644 --- a/statediff/publisher/publisher_test.go +++ b/statediff/publisher/publisher_test.go @@ -3,11 +3,6 @@ package publisher_test import ( "bytes" "encoding/csv" - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/builder" - p "github.com/ethereum/go-ethereum/statediff/publisher" - "github.com/ethereum/go-ethereum/statediff/testhelpers" - "github.com/pkg/errors" "io/ioutil" "os" "path/filepath" @@ -15,6 +10,12 @@ import ( "strconv" "strings" "testing" + + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/builder" + p "github.com/ethereum/go-ethereum/statediff/publisher" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + "github.com/pkg/errors" ) var ( diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go index 0025239af..15b4bc1b2 100644 --- a/statediff/service/service_test.go +++ b/statediff/service/service_test.go @@ -1,15 +1,16 @@ package service_test import ( + "math/big" + "math/rand" + "reflect" + "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" service2 "github.com/ethereum/go-ethereum/statediff/service" "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" - "math/big" - "math/rand" - "reflect" - "testing" ) func TestServiceLoop(t *testing.T) { diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 34194d2f4..a24f32c15 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -1,10 +1,11 @@ package testhelpers import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/builder" "math/big" "math/rand" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/builder" ) var ( -- 2.45.2 From 119e5dfebee8041ce9ad445dc030e973e3723e1d Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Thu, 31 Jan 2019 10:05:33 -0600 Subject: [PATCH 04/17] Apply gosimple changes to statediff --- statediff/builder/helpers.go | 3 --- statediff/publisher/csv.go | 16 ++++------------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go index 26602ebc9..891525cc2 100644 --- a/statediff/builder/helpers.go +++ b/statediff/builder/helpers.go @@ -54,7 +54,6 @@ func findIntersection(a, b []string) []string { if iOfA >= lenA { return updates } - break // a[iOfA] == b[iOfB] case 0: updates = append(updates, a[iOfA]) @@ -63,14 +62,12 @@ func findIntersection(a, b []string) []string { if iOfA >= lenA || iOfB >= lenB { return updates } - break // a[iOfA] > b[iOfB] case 1: iOfB++ if iOfB >= lenB { return updates } - break } } diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go index ce405ac14..54526951a 100644 --- a/statediff/publisher/csv.go +++ b/statediff/publisher/csv.go @@ -47,9 +47,7 @@ func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) var data [][]string data = append(data, Headers) - for _, row := range accumulateAccountRows(sd) { - data = append(data, row) - } + data = append(data, accumulateAccountRows(sd)...) for _, value := range data { err := writer.Write(value) if err != nil { @@ -65,25 +63,19 @@ func accumulateAccountRows(sd builder.StateDiff) [][]string { for accountAddr, accountDiff := range sd.CreatedAccounts { formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction) - for _, accountData := range formattedAccountData { - accountRows = append(accountRows, accountData) - } + accountRows = append(accountRows, formattedAccountData...) } for accountAddr, accountDiff := range sd.UpdatedAccounts { formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction) - for _, accountData := range formattedAccountData { - accountRows = append(accountRows, accountData) - } + accountRows = append(accountRows, formattedAccountData...) } for accountAddr, accountDiff := range sd.DeletedAccounts { formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction) - for _, accountData := range formattedAccountData { - accountRows = append(accountRows, accountData) - } + accountRows = append(accountRows, formattedAccountData...) } return accountRows -- 2.45.2 From 47b39271d0d2a9158a3b9ebd2eabacd99843f60f Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 11 Feb 2019 16:08:18 -0600 Subject: [PATCH 05/17] Gracefully exit geth command(#4) --- statediff/service/service.go | 69 +++++++++++++++++------ statediff/service/service_test.go | 50 ++++++++++++---- statediff/testhelpers/mocks/blockchain.go | 37 ++++++++++-- 3 files changed, 123 insertions(+), 33 deletions(-) diff --git a/statediff/service/service.go b/statediff/service/service.go index 2b93a1dd1..dbb898915 100644 --- a/statediff/service/service.go +++ b/statediff/service/service.go @@ -49,34 +49,69 @@ func (StateDiffService) APIs() []rpc.API { return []rpc.API{} } -func (sds *StateDiffService) Loop(events chan core.ChainEvent) { - for elem := range events { - currentBlock := elem.Block - parentHash := currentBlock.ParentHash() - parentBlock := sds.BlockChain.GetBlockByHash(parentHash) +func (sds *StateDiffService) Loop(chainEventCh chan core.ChainEvent) { + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() - stateDiffLocation, err := sds.Extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err) - } else { - log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation) + blocksCh := make(chan *types.Block, 10) + errCh := chainEventSub.Err() + quitCh := make(chan struct{}) + + go func() { + HandleChainEventChLoop: + for { + select { + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + log.Debug("Event received from chainEventCh", "event", chainEvent) + blocksCh <- chainEvent.Block + //if node stopped + case err := <-errCh: + log.Warn("Error from chain event subscription, breaking loop.", "error", err) + break HandleChainEventChLoop + } + } + close(quitCh) + }() + + //loop through chain events until no more +HandleBlockChLoop: + for { + select { + case block := <-blocksCh: + currentBlock := block + parentHash := currentBlock.ParentHash() + parentBlock := sds.BlockChain.GetBlockByHash(parentHash) + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + break HandleBlockChLoop + } + + stateDiffLocation, err := sds.Extractor.ExtractStateDiff(*parentBlock, *currentBlock) + if err != nil { + log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err) + } else { + log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation) + } + case <-quitCh: + log.Debug("Quitting the statediff block channel") + return } } } -var eventsChannel chan core.ChainEvent - func (sds *StateDiffService) Start(server *p2p.Server) error { log.Info("Starting statediff service") - eventsChannel := make(chan core.ChainEvent, 10) - sds.BlockChain.SubscribeChainEvent(eventsChannel) - go sds.Loop(eventsChannel) + + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + return nil } func (StateDiffService) Stop() error { log.Info("Stopping statediff service") - close(eventsChannel) - return nil } diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go index 15b4bc1b2..daf3445c3 100644 --- a/statediff/service/service_test.go +++ b/statediff/service/service_test.go @@ -9,16 +9,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - service2 "github.com/ethereum/go-ethereum/statediff/service" + s "github.com/ethereum/go-ethereum/statediff/service" "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" ) func TestServiceLoop(t *testing.T) { - testServiceLoop(t) + testErrorInChainEventLoop(t) + testErrorInBlockLoop(t) } var ( - eventsChannel = make(chan core.ChainEvent, 10) + eventsChannel = make(chan core.ChainEvent, 1) parentHeader1 = types.Header{Number: big.NewInt(rand.Int63())} parentHeader2 = types.Header{Number: big.NewInt(rand.Int63())} @@ -31,29 +32,30 @@ var ( header1 = types.Header{ParentHash: parentHash1} header2 = types.Header{ParentHash: parentHash2} + header3 = types.Header{ParentHash: common.HexToHash("parent hash")} block1 = types.NewBlock(&header1, nil, nil, nil) block2 = types.NewBlock(&header2, nil, nil, nil) + block3 = types.NewBlock(&header3, nil, nil, nil) event1 = core.ChainEvent{Block: block1} event2 = core.ChainEvent{Block: block2} + event3 = core.ChainEvent{Block: block3} ) -func testServiceLoop(t *testing.T) { - eventsChannel <- event1 - eventsChannel <- event2 - +func testErrorInChainEventLoop(t *testing.T) { + //the first chain event causes and error (in blockchain mock) extractor := mocks.Extractor{} - close(eventsChannel) blockChain := mocks.BlockChain{} - service := service2.StateDiffService{ + service := s.StateDiffService{ Builder: nil, Extractor: &extractor, BlockChain: &blockChain, } - blockChain.SetParentBlockToReturn([]*types.Block{parentBlock1, parentBlock2}) + blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) service.Loop(eventsChannel) //parent and current blocks are passed to the extractor @@ -75,3 +77,31 @@ func testServiceLoop(t *testing.T) { t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) } } + +func testErrorInBlockLoop(t *testing.T) { + //second block's parent block can't be found + extractor := mocks.Extractor{} + + blockChain := mocks.BlockChain{} + service := s.StateDiffService{ + Builder: nil, + Extractor: &extractor, + BlockChain: &blockChain, + } + + blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + service.Loop(eventsChannel) + + //only the first current block (and it's parent) are passed to the extractor + expectedCurrentBlocks := []types.Block{*block1} + if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks) + } + expectedParentBlocks := []types.Block{*parentBlock1} + if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks) + } +} diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index baa7b3cec..f2d77ea34 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -1,6 +1,8 @@ package mocks import ( + "errors" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -11,24 +13,47 @@ type BlockChain struct { ParentHashesLookedUp []common.Hash parentBlocksToReturn []*types.Block callCount int + ChainEvents []core.ChainEvent } -func (mc *BlockChain) SetParentBlockToReturn(blocks []*types.Block) { +func (mc *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { mc.parentBlocksToReturn = blocks } func (mc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { mc.ParentHashesLookedUp = append(mc.ParentHashesLookedUp, hash) - var parentBlock types.Block + var parentBlock *types.Block if len(mc.parentBlocksToReturn) > 0 { - parentBlock = *mc.parentBlocksToReturn[mc.callCount] + parentBlock = mc.parentBlocksToReturn[mc.callCount] } mc.callCount++ - return &parentBlock + return parentBlock } -func (BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { - panic("implement me") +func (bc *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { + bc.ChainEvents = chainEvents +} + +func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { + subErr := errors.New("Subscription Error") + + var eventCounter int + subscription := event.NewSubscription(func(quit <-chan struct{}) error { + for _, chainEvent := range bc.ChainEvents { + if eventCounter > 1 { + return subErr + } + select { + case ch <- chainEvent: + case <-quit: + return nil + } + eventCounter++ + } + return nil + }) + + return subscription } -- 2.45.2 From efbb9042ace4452918ebf8c2096b9b5dc62c362b Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Thu, 21 Feb 2019 14:36:04 -0600 Subject: [PATCH 06/17] Statediff for full node (#6) * Open a trie from the in-memory database * Use a node's LeafKey as an identifier instead of the address It was proving difficult to find look the address up from a given path with a full node (sometimes the value wouldn't exist in the disk db). So, instead, for now we are using the node's LeafKey with is a Keccak256 hash of the address, so if we know the address we can figure out which LeafKey it matches up to. * Make sure that statediff has been processed before pruning * Use blockchain stateCache.OpenTrie for storage diffs * Clean up log lines and remove unnecessary fields from builder * Apply go fmt changes * Add a sleep to the blockchain test * Address PR comments * Address PR comments --- cmd/geth/config.go | 8 ++- core/blockchain.go | 62 ++++++++++++----- core/blockchain_test.go | 85 ++++++++++++++++++++++- eth/backend.go | 1 + eth/config.go | 4 ++ statediff/builder/builder.go | 66 +++++++++--------- statediff/builder/builder_test.go | 62 ++++++++++------- statediff/builder/helpers.go | 3 +- statediff/builder/struct.go | 11 +-- statediff/publisher/csv.go | 4 +- statediff/publisher/publisher_test.go | 8 +-- statediff/service/service.go | 5 +- statediff/testhelpers/mocks/blockchain.go | 5 ++ statediff/testhelpers/test_data.go | 19 +++-- 14 files changed, 243 insertions(+), 100 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 29210e549..48c0c574c 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -156,6 +156,11 @@ func makeFullNode(ctx *cli.Context) *node.Node { } utils.RegisterEthService(stack, &cfg.Eth) + if ctx.GlobalBool(utils.StateDiffFlag.Name) { + cfg.Eth.StateDiff = true + utils.RegisterStateDiffService(stack, ctx) + } + if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) { utils.RegisterDashboardService(stack, &cfg.Dashboard, gitCommit) } @@ -183,9 +188,6 @@ func makeFullNode(ctx *cli.Context) *node.Node { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) } - if ctx.GlobalBool(utils.StateDiffFlag.Name) { - utils.RegisterStateDiffService(stack, ctx) - } return stack } diff --git a/core/blockchain.go b/core/blockchain.go index 833de3bc7..73140a7bd 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -114,6 +114,7 @@ type CacheConfig struct { TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + ProcessingStateDiffs bool // Whether statediffs processing should be taken into a account before a trie is pruned } // BlockChain represents the canonical chain given a database with a genesis @@ -176,6 +177,8 @@ type BlockChain struct { badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. + + stateDiffsProcessed map[common.Hash]int } // NewBlockChain returns a fully initialised block chain using information @@ -196,24 +199,25 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par txLookupCache, _ := lru.New(txLookupCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) badBlocks, _ := lru.New(badBlockLimit) - + stateDiffsProcessed := make(map[common.Hash]int) bc := &BlockChain{ - chainConfig: chainConfig, - cacheConfig: cacheConfig, - db: db, - triegc: prque.New(nil), - stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit), - quit: make(chan struct{}), - shouldPreserve: shouldPreserve, - bodyCache: bodyCache, - bodyRLPCache: bodyRLPCache, - receiptsCache: receiptsCache, - blockCache: blockCache, - txLookupCache: txLookupCache, - futureBlocks: futureBlocks, - engine: engine, - vmConfig: vmConfig, - badBlocks: badBlocks, + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triegc: prque.New(nil), + stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit), + quit: make(chan struct{}), + shouldPreserve: shouldPreserve, + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + receiptsCache: receiptsCache, + blockCache: blockCache, + txLookupCache: txLookupCache, + futureBlocks: futureBlocks, + engine: engine, + vmConfig: vmConfig, + badBlocks: badBlocks, + stateDiffsProcessed: stateDiffsProcessed, } bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) @@ -1251,6 +1255,11 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { return nil } +func (bc *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) { + count := bc.stateDiffsProcessed[hash] + bc.stateDiffsProcessed[hash] = count + 1 +} + // WriteBlockWithState writes the block and all associated state to the database. func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { bc.chainmu.Lock() @@ -1335,6 +1344,16 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, number) break } + + if bc.cacheConfig.ProcessingStateDiffs { + if !bc.allowedRootToBeDereferenced(root.(common.Hash)) { + bc.triegc.Push(root, number) + break + } else { + delete(bc.stateDiffsProcessed, root.(common.Hash)) + } + } + triedb.Dereference(root.(common.Hash)) } } @@ -1389,6 +1408,15 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. return status, nil } +// since we need the state tries of the current block and its parent in-memory +// in order to process statediffs, we should avoid dereferencing roots until +// its statediff and its child have been processed +func (bc *BlockChain) allowedRootToBeDereferenced(root common.Hash) bool { + diffProcessedForSelfAndChildCount := 2 + count := bc.stateDiffsProcessed[root] + return count >= diffProcessedForSelfAndChildCount +} + // addFutureBlock checks if the block is within the max allowed window to get // accepted for future processing, and returns an error if the block is too far // ahead and was not added. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index db624c4dc..2480ab55d 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2227,8 +2227,8 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) { func BenchmarkBlockChain_1x1000Executions(b *testing.B) { var ( - numTxs = 1000 - numBlocks = 1 + numTxs= 1000 + numBlocks= 1 ) b.StopTimer() b.ResetTimer() @@ -2287,3 +2287,84 @@ func TestSideImportPrunedBlocks(t *testing.T) { t.Errorf("Got error, %v", err) } } + +func TestProcessingStateDiffs(t *testing.T) { + defaultTrieCleanCache := 256 + defaultTrieDirtyCache := 256 + defaultTrieTimeout := 60 * time.Minute + cacheConfig := &CacheConfig{ + TrieDirtyDisabled: false, + TrieCleanLimit: defaultTrieCleanCache, + TrieDirtyLimit: defaultTrieDirtyCache, + TrieTimeLimit: defaultTrieTimeout, + ProcessingStateDiffs: true, + } + db := rawdb.NewMemoryDatabase() + genesis := new(Genesis).MustCommit(db) + numberOfBlocks := TriesInMemory + engine := ethash.NewFaker() + blockchain, _ := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil) + blocks := makeBlockChain(genesis, numberOfBlocks+1, engine, db, canonicalSeed) + _, err := blockchain.InsertChain(blocks) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() + + //when adding a root hash to the collection, it will increment the count + firstStateRoot := blocks[0].Root() + blockchain.AddToStateDiffProcessedCollection(firstStateRoot) + value, ok := blockchain.stateDiffsProcessed[firstStateRoot] + if !ok { + t.Error("state root not found in collection") + } + if value != 1 { + t.Error("state root count not correct", "want", 1, "got", value) + } + + blockchain.AddToStateDiffProcessedCollection(firstStateRoot) + value, ok = blockchain.stateDiffsProcessed[firstStateRoot] + if !ok { + t.Error("state root not found in collection") + } + if value != 2 { + t.Error("state root count not correct", "want", 2, "got", value) + } + + moreBlocks := makeBlockChain(blocks[len(blocks)-1], 1, engine, db, canonicalSeed) + _, err = blockchain.InsertChain(moreBlocks) + + //a root hash can be dereferenced when it's state diff and it's child's state diff have been processed + //(i.e. it has a count of 2 in stateDiffsProcessed) + nodes := blockchain.stateCache.TrieDB().Nodes() + if containsRootHash(nodes, firstStateRoot) { + t.Errorf("stateRoot %s in nodes, want: %t, got: %t", firstStateRoot.Hex(), false, true) + } + + //a root hash should still be in the in-mem db if it's child's state diff hasn't yet been processed + //(i.e. it has a count of 1 stateDiffsProcessed) + secondStateRoot := blocks[1].Root() + blockchain.AddToStateDiffProcessedCollection(secondStateRoot) + if !containsRootHash(nodes, secondStateRoot) { + t.Errorf("stateRoot %s in nodes, want: %t, got: %t", secondStateRoot.Hex(), true, false) + } + + //the stateDiffsProcessed collection is cleaned up once a hash has been dereferenced + _, ok = blockchain.stateDiffsProcessed[firstStateRoot] + if ok { + t.Errorf("stateRoot %s in stateDiffsProcessed collection, want: %t, got: %t", + firstStateRoot.Hex(), + false, + ok, + ) + } +} + +func containsRootHash(collection []common.Hash, hash common.Hash) bool { + for _, n := range collection { + if n == hash { + return true + } + } + return false +} diff --git a/eth/backend.go b/eth/backend.go index ce37541f4..25e6b5128 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -184,6 +184,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, + ProcessingStateDiffs: config.StateDiff, } ) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve) diff --git a/eth/config.go b/eth/config.go index 5094a533b..16af87b12 100644 --- a/eth/config.go +++ b/eth/config.go @@ -61,6 +61,8 @@ var DefaultConfig = Config{ Blocks: 20, Percentile: 60, }, + + StateDiff: false, } func init() { @@ -157,4 +159,6 @@ type Config struct { // Istanbul block override (TODO: remove after the fork) OverrideIstanbul *big.Int + + StateDiff bool } diff --git a/statediff/builder/builder.go b/statediff/builder/builder.go index 1d66db380..a106c3ad8 100644 --- a/statediff/builder/builder.go +++ b/statediff/builder/builder.go @@ -22,6 +22,7 @@ package builder import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -35,25 +36,27 @@ type Builder interface { type builder struct { chainDB ethdb.Database - trieDB *trie.Database - cachedTrie *trie.Trie + blockChain *core.BlockChain } -func NewBuilder(db ethdb.Database) *builder { +type AccountsMap map[common.Hash]*state.Account + +func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) *builder { return &builder{ - chainDB: db, - trieDB: trie.NewDatabase(db), + chainDB: db, + blockChain: blockChain, } } func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { // Generate tries for old and new states - oldTrie, err := trie.New(oldStateRoot, sdb.trieDB) + stateCache := sdb.blockChain.StateCache() + oldTrie, err := stateCache.OpenTrie(oldStateRoot) if err != nil { log.Error("Error creating trie for oldStateRoot", "error", err) return nil, err } - newTrie, err := trie.New(newStateRoot, sdb.trieDB) + newTrie, err := stateCache.OpenTrie(newStateRoot) if err != nil { log.Error("Error creating trie for newStateRoot", "error", err) return nil, err @@ -108,33 +111,27 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block }, nil } -func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address]*state.Account, error) { - var diffAccounts = make(map[common.Address]*state.Account) +func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error) { + var diffAccounts = make(AccountsMap) it, _ := trie.NewDifferenceIterator(a, b) for { log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) if it.Leaf() { - - // lookup address - path := make([]byte, len(it.Path())-1) - copy(path, it.Path()) - addr, err := sdb.addressByPath(path) - if err != nil { - log.Error("Error looking up address via path", "path", path, "error", err) - return nil, err - } + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) // lookup account state var account state.Account if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { - log.Error("Error looking up account via address", "address", addr, "error", err) + log.Error("Error looking up account via address", "address", leafKeyHash, "error", err) return nil, err } // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", addr, "account", account) - diffAccounts[*addr] = &account + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = &account } cont := it.Next(true) if !cont { @@ -145,8 +142,8 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (map[common.Address return diffAccounts, nil } -func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account) (map[common.Address]AccountDiff, error) { - accountDiffs := make(map[common.Address]AccountDiff) +func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { + accountDiffs := make(AccountDiffsMap) for addr, val := range accounts { sr := val.Root storageDiffs, err := sdb.buildStorageDiffsEventual(sr) @@ -172,11 +169,11 @@ func (sdb *builder) buildDiffEventual(accounts map[common.Address]*state.Account return accountDiffs, nil } -func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Account, deletions map[common.Address]*state.Account, updatedKeys []string) (map[common.Address]AccountDiff, error) { - updatedAccounts := make(map[common.Address]AccountDiff) +func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) { + updatedAccounts := make(AccountDiffsMap) for _, val := range updatedKeys { - createdAcc := creations[common.HexToAddress(val)] - deletedAcc := deletions[common.HexToAddress(val)] + createdAcc := creations[common.HexToHash(val)] + deletedAcc := deletions[common.HexToHash(val)] oldSR := deletedAcc.Root newSR := createdAcc.Root if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { @@ -190,15 +187,15 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc nHexRoot := createdAcc.Root.Hex() contractRoot := DiffString{Value: &nHexRoot} - updatedAccounts[common.HexToAddress(val)] = AccountDiff{ + updatedAccounts[common.HexToHash(val)] = AccountDiff{ Nonce: nonce, Balance: balance, CodeHash: codeHash, ContractRoot: contractRoot, Storage: storageDiffs, } - delete(creations, common.HexToAddress(val)) - delete(deletions, common.HexToAddress(val)) + delete(creations, common.HexToHash(val)) + delete(deletions, common.HexToHash(val)) } } return updatedAccounts, nil @@ -206,7 +203,8 @@ func (sdb *builder) buildDiffIncremental(creations map[common.Address]*state.Acc func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffStorage, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) - sTrie, err := trie.New(sr, sdb.trieDB) + stateCache := sdb.blockChain.StateCache() + sTrie, err := stateCache.OpenTrie(sr) if err != nil { log.Info("error in build storage diff eventual", "error", err) return nil, err @@ -218,11 +216,13 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffSt func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffStorage, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) - oldTrie, err := trie.New(oldSR, sdb.trieDB) + stateCache := sdb.blockChain.StateCache() + + oldTrie, err := stateCache.OpenTrie(oldSR) if err != nil { return nil, err } - newTrie, err := trie.New(newSR, sdb.trieDB) + newTrie, err := stateCache.OpenTrie(newSR) if err != nil { return nil, err } diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go index 95163100b..4f8b7c468 100644 --- a/statediff/builder/builder_test.go +++ b/statediff/builder/builder_test.go @@ -10,10 +10,12 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" b "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff/testhelpers" ) var ( @@ -21,26 +23,33 @@ var ( testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + bankLeafKey = testhelpers.AddressToLeafKey(testBankAddress) testBankFunds = big.NewInt(100000000) genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + account1LeafKey = testhelpers.AddressToLeafKey(account1Addr) account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + account2LeafKey = testhelpers.AddressToLeafKey(account2Addr) contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") contractAddr common.Address - emptyAccountDiffEventualMap = make(map[common.Address]b.AccountDiff) - emptyAccountDiffIncrementalMap = make(map[common.Address]b.AccountDiff) + contractLeafKey common.Hash + emptyAccountDiffEventualMap = make(b.AccountDiffsMap) + emptyAccountDiffIncrementalMap = make(b.AccountDiffsMap) block0Hash, block1Hash, block2Hash, block3Hash common.Hash block0, block1, block2, block3 *types.Block builder b.Builder miningReward = int64(2000000000000000000) burnAddress = common.HexToAddress("0x0") + burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) ) func TestBuilder(t *testing.T) { - _, blockMap := makeChain(3, genesis) + _, blockMap, chain := makeChain(3, genesis) + contractLeafKey = testhelpers.AddressToLeafKey(contractAddr) + defer chain.Stop() block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") block2Hash = common.HexToHash("0xde75663f36a8497b4bdda2a4b52bd9540b705a2728c7391c59b8cb2cde5a2feb") @@ -50,7 +59,7 @@ func TestBuilder(t *testing.T) { block1 = blockMap[block1Hash] block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] - builder = b.NewBuilder(testdb) + builder = b.NewBuilder(testdb, chain) type arguments struct { oldStateRoot common.Hash @@ -113,15 +122,15 @@ func TestBuilder(t *testing.T) { &b.StateDiff{ BlockNumber: block1.Number().Int64(), BlockHash: block1.Hash(), - CreatedAccounts: map[common.Address]b.AccountDiff{ - account1Addr: { + CreatedAccounts: b.AccountDiffsMap{ + account1LeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(balanceChange10000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - burnAddress: { + burnLeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(miningReward)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -130,8 +139,8 @@ func TestBuilder(t *testing.T) { }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: map[common.Address]b.AccountDiff{ - testBankAddress: { + UpdatedAccounts: b.AccountDiffsMap{ + bankLeafKey: { Nonce: b.DiffUint64{Value: &nonce1}, Balance: b.DiffBigInt{Value: big.NewInt(testBankFunds.Int64() - balanceChange10000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -154,15 +163,15 @@ func TestBuilder(t *testing.T) { &b.StateDiff{ BlockNumber: block2.Number().Int64(), BlockHash: block2.Hash(), - CreatedAccounts: map[common.Address]b.AccountDiff{ - account2Addr: { + CreatedAccounts: b.AccountDiffsMap{ + account2LeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(balanceChange1000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - contractAddr: { + contractLeafKey: { Nonce: b.DiffUint64{Value: &nonce1}, Balance: b.DiffBigInt{Value: big.NewInt(0)}, CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", @@ -175,22 +184,22 @@ func TestBuilder(t *testing.T) { }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: map[common.Address]b.AccountDiff{ - testBankAddress: { + UpdatedAccounts: b.AccountDiffsMap{ + bankLeafKey: { Nonce: b.DiffUint64{Value: &nonce2}, Balance: b.DiffBigInt{Value: big.NewInt(block1BankBalance - balanceChange1000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - account1Addr: { + account1LeafKey: { Nonce: b.DiffUint64{Value: &nonce2}, Balance: b.DiffBigInt{Value: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - burnAddress: { + burnLeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(miningReward + miningReward)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -213,17 +222,17 @@ func TestBuilder(t *testing.T) { &b.StateDiff{ BlockNumber: block3.Number().Int64(), BlockHash: block3.Hash(), - CreatedAccounts: map[common.Address]b.AccountDiff{}, + CreatedAccounts: b.AccountDiffsMap{}, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: map[common.Address]b.AccountDiff{ - account2Addr: { + UpdatedAccounts: b.AccountDiffsMap{ + account2LeafKey: { Nonce: b.DiffUint64{Value: &nonce0}, Balance: b.DiffBigInt{Value: big.NewInt(block2Account2Balance + miningReward)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ContractRoot: b.DiffString{Value: &originalContractRoot}, Storage: map[string]b.DiffStorage{}, }, - contractAddr: { + contractLeafKey: { Nonce: b.DiffUint64{Value: &nonce1}, Balance: b.DiffBigInt{Value: big.NewInt(0)}, CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", @@ -234,7 +243,7 @@ func TestBuilder(t *testing.T) { Value: &updatedStorageValue}, }, }, - testBankAddress: { + bankLeafKey: { Nonce: b.DiffUint64{Value: &nonce3}, Balance: b.DiffBigInt{Value: big.NewInt(99989000)}, CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", @@ -252,7 +261,6 @@ func TestBuilder(t *testing.T) { if err != nil { t.Error(err) } - fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} for _, field := range fields { @@ -287,8 +295,14 @@ func equals(actual, expected interface{}) (success bool) { // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. -func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { +func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) { blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) + headers := make([]*types.Header, len(blocks)) + for i, block := range blocks { + headers[i] = block.Header() + } + chain, _ := core.NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil) + hashes := make([]common.Hash, n+1) hashes[len(hashes)-1] = parent.Hash() blockm := make(map[common.Hash]*types.Block, n+1) @@ -297,7 +311,7 @@ func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*type hashes[len(hashes)-i-2] = b.Hash() blockm[b.Hash()] = b } - return hashes, blockm + return hashes, blockm, chain } func testChainGen(i int, block *core.BlockGen) { diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go index 891525cc2..339454776 100644 --- a/statediff/builder/helpers.go +++ b/statediff/builder/helpers.go @@ -24,11 +24,10 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/trie" ) -func sortKeys(data map[common.Address]*state.Account) []string { +func sortKeys(data AccountsMap) []string { var keys []string for key := range data { keys = append(keys, key.Hex()) diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go index 295b09f10..416d8e816 100644 --- a/statediff/builder/struct.go +++ b/statediff/builder/struct.go @@ -26,12 +26,13 @@ import ( "github.com/ethereum/go-ethereum/common" ) +type AccountDiffsMap map[common.Hash]AccountDiff type StateDiff struct { - BlockNumber int64 `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - CreatedAccounts map[common.Address]AccountDiff `json:"createdAccounts" gencodec:"required"` - DeletedAccounts map[common.Address]AccountDiff `json:"deletedAccounts" gencodec:"required"` - UpdatedAccounts map[common.Address]AccountDiff `json:"updatedAccounts" gencodec:"required"` + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` + DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` encoded []byte err error diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go index 54526951a..13971a5c8 100644 --- a/statediff/publisher/csv.go +++ b/statediff/publisher/csv.go @@ -15,7 +15,7 @@ var ( Headers = []string{ "blockNumber", "blockHash", "accountAction", "codeHash", "nonceValue", "balanceValue", "contractRoot", "storageDiffPaths", - "accountAddress", "storageKey", "storageValue", + "accountLeafKey", "storageKey", "storageValue", } timeStampFormat = "20060102150405.00000" @@ -81,7 +81,7 @@ func accumulateAccountRows(sd builder.StateDiff) [][]string { return accountRows } -func formatAccountData(accountAddr common.Address, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { +func formatAccountData(accountAddr common.Hash, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { blockNumberString := strconv.FormatInt(sd.BlockNumber, 10) blockHash := sd.BlockHash.String() codeHash := accountDiff.CodeHash diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go index 8a26fb4e2..76aaf961e 100644 --- a/statediff/publisher/publisher_test.go +++ b/statediff/publisher/publisher_test.go @@ -35,7 +35,7 @@ var expectedCreatedAccountRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, testhelpers.StoragePath, - testhelpers.ContractAddress, + testhelpers.ContractLeafKey.Hex(), "0000000000000000000000000000000000000000000000000000000000000001", testhelpers.StorageValue, } @@ -49,7 +49,7 @@ var expectedCreatedAccountWithoutStorageUpdateRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, "", - testhelpers.AnotherContractAddress, + testhelpers.AnotherContractLeafKey.Hex(), "", "", } @@ -63,7 +63,7 @@ var expectedUpdatedAccountRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, testhelpers.StoragePath, - testhelpers.ContractAddress, + testhelpers.ContractLeafKey.Hex(), "0000000000000000000000000000000000000000000000000000000000000001", testhelpers.StorageValue, } @@ -77,7 +77,7 @@ var expectedDeletedAccountRow = []string{ strconv.FormatInt(testhelpers.NewBalanceValue, 10), testhelpers.ContractRoot, testhelpers.StoragePath, - testhelpers.ContractAddress, + testhelpers.ContractLeafKey.Hex(), "0000000000000000000000000000000000000000000000000000000000000001", testhelpers.StorageValue, } diff --git a/statediff/service/service.go b/statediff/service/service.go index dbb898915..19ea0c644 100644 --- a/statediff/service/service.go +++ b/statediff/service/service.go @@ -19,6 +19,7 @@ import ( type BlockChain interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription GetBlockByHash(hash common.Hash) *types.Block + AddToStateDiffProcessedCollection(hash common.Hash) } type StateDiffService struct { @@ -28,7 +29,7 @@ type StateDiffService struct { } func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config statediff.Config) (*StateDiffService, error) { - builder := b.NewBuilder(db) + builder := b.NewBuilder(db, blockChain) publisher, err := p.NewPublisher(config) if err != nil { return nil, err @@ -94,6 +95,8 @@ HandleBlockChLoop: log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err) } else { log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation) + sds.BlockChain.AddToStateDiffProcessedCollection(parentBlock.Root()) + sds.BlockChain.AddToStateDiffProcessedCollection(currentBlock.Root()) } case <-quitCh: log.Debug("Quitting the statediff block channel") diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index f2d77ea34..7b0d74c59 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -3,6 +3,8 @@ package mocks import ( "errors" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -16,6 +18,8 @@ type BlockChain struct { ChainEvents []core.ChainEvent } +func (mc *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} + func (mc *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { mc.parentBlocksToReturn = blocks } @@ -43,6 +47,7 @@ func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc subscription := event.NewSubscription(func(quit <-chan struct{}) error { for _, chainEvent := range bc.ChainEvents { if eventCounter > 1 { + time.Sleep(250 * time.Millisecond) return subErr } select { diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index a24f32c15..831bed218 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -5,9 +5,14 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/statediff/builder" ) +func AddressToLeafKey(address common.Address) common.Hash { + return common.BytesToHash(crypto.Keccak256(address[:])) +} + var ( BlockNumber = rand.Int63() BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" @@ -24,18 +29,18 @@ var ( }} emptyStorage = map[string]builder.DiffStorage{} address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + ContractLeafKey = AddressToLeafKey(address) anotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") - ContractAddress = address.String() - AnotherContractAddress = anotherAddress.String() - CreatedAccountDiffs = map[common.Address]builder.AccountDiff{ - address: { + AnotherContractLeafKey = AddressToLeafKey(anotherAddress) + CreatedAccountDiffs = builder.AccountDiffsMap{ + ContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, ContractRoot: builder.DiffString{Value: &ContractRoot}, CodeHash: CodeHash, Storage: storage, }, - anotherAddress: { + AnotherContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, CodeHash: CodeHash, @@ -44,7 +49,7 @@ var ( }, } - UpdatedAccountDiffs = map[common.Address]builder.AccountDiff{address: { + UpdatedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, CodeHash: CodeHash, @@ -52,7 +57,7 @@ var ( Storage: storage, }} - DeletedAccountDiffs = map[common.Address]builder.AccountDiff{address: { + DeletedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { Nonce: builder.DiffUint64{Value: &NewNonceValue}, Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, ContractRoot: builder.DiffString{Value: &ContractRoot}, -- 2.45.2 From 0acab0c3dd6f73ad928500fe6e79594b7ec8a6d5 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 26 Apr 2019 11:13:59 -0500 Subject: [PATCH 07/17] refactoring/reorganizing packages --- cmd/geth/main.go | 2 - cmd/geth/usage.go | 2 - cmd/utils/flags.go | 43 +-- core/blockchain.go | 17 +- eth/backend.go | 10 +- statediff/config.go | 93 ------- statediff/config_test.go | 24 -- statediff/extractor/extractor.go | 51 ---- statediff/extractor/extractor_test.go | 123 --------- statediff/helpers.go | 118 ++++++++ statediff/publisher/csv.go | 130 --------- statediff/publisher/publisher.go | 48 ---- statediff/publisher/publisher_test.go | 316 ---------------------- statediff/testhelpers/helpers.go | 83 +++++- statediff/testhelpers/mocks/blockchain.go | 46 +++- statediff/testhelpers/mocks/builder.go | 28 +- statediff/testhelpers/mocks/error.go | 5 - statediff/testhelpers/mocks/extractor.go | 20 -- statediff/testhelpers/mocks/publisher.go | 25 +- statediff/testhelpers/test_data.go | 107 +++++--- 20 files changed, 377 insertions(+), 914 deletions(-) delete mode 100644 statediff/config.go delete mode 100644 statediff/config_test.go delete mode 100644 statediff/extractor/extractor.go delete mode 100644 statediff/extractor/extractor_test.go create mode 100644 statediff/helpers.go delete mode 100644 statediff/publisher/csv.go delete mode 100644 statediff/publisher/publisher.go delete mode 100644 statediff/publisher/publisher_test.go delete mode 100644 statediff/testhelpers/mocks/error.go delete mode 100644 statediff/testhelpers/mocks/extractor.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index eaeb3adfb..582630163 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -149,8 +149,6 @@ var ( utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, utils.StateDiffFlag, - utils.StateDiffModeFlag, - utils.StateDiffPathFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index ce6eab279..77d816476 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -266,8 +266,6 @@ var AppHelpFlagGroups = []flagGroup{ Name: "STATE DIFF", Flags: []cli.Flag{ utils.StateDiffFlag, - utils.StateDiffModeFlag, - utils.StateDiffPathFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0b5c0c955..9d9ecf732 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -32,6 +32,8 @@ import ( "text/template" "time" + cli "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" @@ -62,11 +64,10 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" - pcsclite "github.com/gballet/go-libpcsclite" - cli "gopkg.in/urfave/cli.v1" - "github.com/ethereum/go-ethereum/statediff/service" "github.com/ethereum/go-ethereum/statediff" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" + + pcsclite "github.com/gballet/go-libpcsclite" ) var ( @@ -762,18 +763,6 @@ var ( Name: "statediff", Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", } - - StateDiffModeFlag = cli.StringFlag{ - Name: "statediff.mode", - Usage: "Enables the user to determine which persistence mode they'd like to store the state diffs in.", - Value: "csv", - } - - StateDiffPathFlag = cli.StringFlag{ - Name: "statediff.path", - Usage: "Enables the user to determine where to persist the state diffs.", - Value: ".", - } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -983,6 +972,9 @@ func setWS(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(WSApiFlag.Name) { cfg.WSModules = splitAndTrim(ctx.GlobalString(WSApiFlag.Name)) } + if ctx.GlobalBool(StateDiffFlag.Name) { + cfg.WSModules = append(cfg.WSModules, "statediff") + } } // setIPC creates an IPC path configuration from the set command line flags, @@ -1629,29 +1621,14 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st } } +// RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { - //based on the context, if path and mode are set, update the config here - //otherwise pass in an empty config - - modeFlag := ctx.GlobalString(StateDiffModeFlag.Name) - mode, err := statediff.NewMode(modeFlag) - if err != nil { - Fatalf("Failed to register State Diff Service", err) - } - - path := ctx.GlobalString(StateDiffPathFlag.Name) - - config := statediff.Config{ - Mode: mode, - Path: path, - } - if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum ctx.Service(ðServ) chainDb := ethServ.ChainDb() blockChain := ethServ.BlockChain() - return service.NewStateDiffService(chainDb, blockChain, config) + return statediff.NewStateDiffService(chainDb, blockChain) }); err != nil { Fatalf("Failed to register State Diff Service", err) } diff --git a/core/blockchain.go b/core/blockchain.go index 73140a7bd..2a930cccf 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -109,11 +109,11 @@ const ( // CacheConfig contains the configuration values for the trie caching/pruning // that's resident in a blockchain. type CacheConfig struct { - TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory - TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks - TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk - TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) - TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory + TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks + TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk + TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) + TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk ProcessingStateDiffs bool // Whether statediffs processing should be taken into a account before a trie is pruned } @@ -1344,16 +1344,19 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, number) break } - if bc.cacheConfig.ProcessingStateDiffs { if !bc.allowedRootToBeDereferenced(root.(common.Hash)) { bc.triegc.Push(root, number) break } else { + log.Debug("Current root found in stateDiffsProcessed collection with a count of 2, okay to dereference", + "root", root.(common.Hash).Hex(), + "blockNumber", uint64(-number), + "size of stateDiffsProcessed", len(bc.stateDiffsProcessed)) delete(bc.stateDiffsProcessed, root.(common.Hash)) } } - + log.Debug("Dereferencing", "root", root.(common.Hash).Hex()) triedb.Dereference(root.(common.Hash)) } } diff --git a/eth/backend.go b/eth/backend.go index 25e6b5128..2711e8642 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -179,11 +179,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { EVMInterpreter: config.EVMInterpreter, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, ProcessingStateDiffs: config.StateDiff, } ) diff --git a/statediff/config.go b/statediff/config.go deleted file mode 100644 index 7f5ec3c35..000000000 --- a/statediff/config.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package statediff - -import "fmt" - -type Config struct { - Mode StateDiffMode // Mode for storing diffs - Path string // Path for storing diffs -} - -type StateDiffMode int - -const ( - CSV StateDiffMode = iota - IPLD - LDB - SQL -) - -func (mode StateDiffMode) IsValid() bool { - return mode >= IPLD && mode <= SQL -} - -// String implements the stringer interface. -func (mode StateDiffMode) String() string { - switch mode { - case CSV: - return "csv" - case IPLD: - return "ipfs" - case LDB: - return "ldb" - case SQL: - return "sql" - default: - return "unknown" - } -} - -func NewMode(mode string) (StateDiffMode, error) { - stateDiffMode := StateDiffMode(0) - err := stateDiffMode.UnmarshalText([]byte(mode)) - return stateDiffMode, err -} - -func (mode StateDiffMode) MarshalText() ([]byte, error) { - switch mode { - case CSV: - return []byte("ipfs"), nil - case IPLD: - return []byte("ipfs"), nil - case LDB: - return []byte("ldb"), nil - case SQL: - return []byte("sql"), nil - default: - return nil, fmt.Errorf("unknown state diff storage mode %d", mode) - } -} - -func (mode *StateDiffMode) UnmarshalText(text []byte) error { - switch string(text) { - case "csv": - *mode = CSV - case "ipfs": - *mode = IPLD - case "ldb": - *mode = LDB - case "sql": - *mode = SQL - default: - return fmt.Errorf(`unknown state diff storage mode %q, want "ipfs", "ldb" or "sql"`, text) - } - return nil -} diff --git a/statediff/config_test.go b/statediff/config_test.go deleted file mode 100644 index 5246d1cd1..000000000 --- a/statediff/config_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package statediff_test - -import ( - "testing" - - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/testhelpers" -) - -func TestNewMode(t *testing.T) { - mode, err := statediff.NewMode("csv") - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - if mode != statediff.CSV { - t.Error() - } - - _, err = statediff.NewMode("not a real mode") - if err == nil { - t.Error("Expected an error, and got nil.") - } -} diff --git a/statediff/extractor/extractor.go b/statediff/extractor/extractor.go deleted file mode 100644 index 770973c8d..000000000 --- a/statediff/extractor/extractor.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package extractor - -import ( - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/statediff/builder" - "github.com/ethereum/go-ethereum/statediff/publisher" -) - -type Extractor interface { - ExtractStateDiff(parent, current types.Block) (string, error) -} - -type extractor struct { - Builder builder.Builder // Interface for building state diff objects from two blocks - Publisher publisher.Publisher // Interface for publishing state diff objects to a datastore (e.g. IPFS) -} - -func NewExtractor(builder builder.Builder, publisher publisher.Publisher) *extractor { - return &extractor{ - Builder: builder, - Publisher: publisher, - } -} - -func (e *extractor) ExtractStateDiff(parent, current types.Block) (string, error) { - stateDiff, err := e.Builder.BuildStateDiff(parent.Root(), current.Root(), current.Number().Int64(), current.Hash()) - if err != nil { - return "", err - } - - return e.Publisher.PublishStateDiff(stateDiff) -} diff --git a/statediff/extractor/extractor_test.go b/statediff/extractor/extractor_test.go deleted file mode 100644 index 0ed036c12..000000000 --- a/statediff/extractor/extractor_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package extractor_test - -import ( - "bytes" - "math/big" - "math/rand" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/core/types" - b "github.com/ethereum/go-ethereum/statediff/builder" - e "github.com/ethereum/go-ethereum/statediff/extractor" - "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" -) - -var publisher mocks.Publisher -var builder mocks.Builder -var currentBlockNumber *big.Int -var parentBlock, currentBlock *types.Block -var expectedStateDiff b.StateDiff -var extractor e.Extractor -var err error - -func TestExtractor(t *testing.T) { - publisher = mocks.Publisher{} - builder = mocks.Builder{} - extractor = e.NewExtractor(&builder, &publisher) - if err != nil { - t.Error(err) - } - - blockNumber := rand.Int63() - parentBlockNumber := big.NewInt(blockNumber - int64(1)) - currentBlockNumber = big.NewInt(blockNumber) - parentBlock = types.NewBlock(&types.Header{Number: parentBlockNumber}, nil, nil, nil) - currentBlock = types.NewBlock(&types.Header{Number: currentBlockNumber}, nil, nil, nil) - - expectedStateDiff = b.StateDiff{ - BlockNumber: blockNumber, - BlockHash: currentBlock.Hash(), - CreatedAccounts: nil, - DeletedAccounts: nil, - UpdatedAccounts: nil, - } - - testBuildStateDiffStruct(t) - testBuildStateDiffErrorHandling(t) - testPublishingStateDiff(t) - testPublisherErrorHandling(t) -} - -func testBuildStateDiffStruct(t *testing.T) { - builder.SetStateDiffToBuild(&expectedStateDiff) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - t.Error(err) - } - - if !equals(builder.OldStateRoot, parentBlock.Root()) { - t.Error() - } - if !equals(builder.NewStateRoot, currentBlock.Root()) { - t.Error() - } - if !equals(builder.BlockNumber, currentBlockNumber.Int64()) { - t.Error() - } - if !equals(builder.BlockHash, currentBlock.Hash()) { - t.Error() - } -} - -func testBuildStateDiffErrorHandling(t *testing.T) { - builder.SetBuilderError(mocks.Error) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err == nil { - t.Error(err) - } - - if !equals(err, mocks.Error) { - t.Error() - } - builder.SetBuilderError(nil) -} - -func testPublishingStateDiff(t *testing.T) { - builder.SetStateDiffToBuild(&expectedStateDiff) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - t.Error(err) - } - - if !equals(publisher.StateDiff, &expectedStateDiff) { - t.Error() - } -} - -func testPublisherErrorHandling(t *testing.T) { - publisher.SetPublisherError(mocks.Error) - - _, err = extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err == nil { - t.Error("Expected an error, but it didn't occur.") - } - if !equals(err, mocks.Error) { - t.Error() - } - - publisher.SetPublisherError(nil) -} - -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} diff --git a/statediff/helpers.go b/statediff/helpers.go new file mode 100644 index 000000000..96f9ddf30 --- /dev/null +++ b/statediff/helpers.go @@ -0,0 +1,118 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "sort" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/trie" +) + +func sortKeys(data AccountsMap) []string { + var keys []string + for key := range data { + keys = append(keys, key.Hex()) + } + sort.Strings(keys) + + return keys +} + +// BytesToNiblePath +func bytesToNiblePath(path []byte) string { + if hasTerm(path) { + path = path[:len(path)-1] + } + nibblePath := "" + for i, v := range common.ToHex(path) { + if i%2 == 0 && i > 1 { + continue + } + nibblePath = nibblePath + string(v) + } + + return nibblePath +} + +func findIntersection(a, b []string) []string { + lenA := len(a) + lenB := len(b) + iOfA, iOfB := 0, 0 + updates := make([]string, 0) + if iOfA >= lenA || iOfB >= lenB { + return updates + } + for { + switch strings.Compare(a[iOfA], b[iOfB]) { + // a[iOfA] < b[iOfB] + case -1: + iOfA++ + if iOfA >= lenA { + return updates + } + // a[iOfA] == b[iOfB] + case 0: + updates = append(updates, a[iOfA]) + iOfA++ + iOfB++ + if iOfA >= lenA || iOfB >= lenB { + return updates + } + // a[iOfA] > b[iOfB] + case 1: + iOfB++ + if iOfB >= lenB { + return updates + } + } + } + +} + +func pathToStr(it trie.NodeIterator) string { + return bytesToNiblePath(it.Path()) +} + +// Duplicated from trie/encoding.go +func hexToKeyBytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} diff --git a/statediff/publisher/csv.go b/statediff/publisher/csv.go deleted file mode 100644 index 13971a5c8..000000000 --- a/statediff/publisher/csv.go +++ /dev/null @@ -1,130 +0,0 @@ -package publisher - -import ( - "encoding/csv" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/builder" -) - -var ( - Headers = []string{ - "blockNumber", "blockHash", "accountAction", "codeHash", - "nonceValue", "balanceValue", "contractRoot", "storageDiffPaths", - "accountLeafKey", "storageKey", "storageValue", - } - - timeStampFormat = "20060102150405.00000" - deletedAccountAction = "deleted" - createdAccountAction = "created" - updatedAccountAction = "updated" -) - -func createCSVFilePath(path, blockNumber string) string { - now := time.Now() - timeStamp := now.Format(timeStampFormat) - suffix := timeStamp + "-" + blockNumber - filePath := filepath.Join(path, suffix) - filePath = filePath + ".csv" - return filePath -} - -func (p *publisher) publishStateDiffToCSV(sd builder.StateDiff) (string, error) { - filePath := createCSVFilePath(p.Config.Path, strconv.FormatInt(sd.BlockNumber, 10)) - - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return "", err - } - defer file.Close() - - writer := csv.NewWriter(file) - defer writer.Flush() - - var data [][]string - data = append(data, Headers) - data = append(data, accumulateAccountRows(sd)...) - for _, value := range data { - err := writer.Write(value) - if err != nil { - return "", err - } - } - - return filePath, nil -} - -func accumulateAccountRows(sd builder.StateDiff) [][]string { - var accountRows [][]string - for accountAddr, accountDiff := range sd.CreatedAccounts { - formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, createdAccountAction) - - accountRows = append(accountRows, formattedAccountData...) - } - - for accountAddr, accountDiff := range sd.UpdatedAccounts { - formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, updatedAccountAction) - - accountRows = append(accountRows, formattedAccountData...) - } - - for accountAddr, accountDiff := range sd.DeletedAccounts { - formattedAccountData := formatAccountData(accountAddr, accountDiff, sd, deletedAccountAction) - - accountRows = append(accountRows, formattedAccountData...) - } - - return accountRows -} - -func formatAccountData(accountAddr common.Hash, accountDiff builder.AccountDiff, sd builder.StateDiff, accountAction string) [][]string { - blockNumberString := strconv.FormatInt(sd.BlockNumber, 10) - blockHash := sd.BlockHash.String() - codeHash := accountDiff.CodeHash - nonce := strconv.FormatUint(*accountDiff.Nonce.Value, 10) - balance := accountDiff.Balance.Value.String() - newContractRoot := accountDiff.ContractRoot.Value - address := accountAddr.String() - var result [][]string - - if len(accountDiff.Storage) > 0 { - for storagePath, storage := range accountDiff.Storage { - formattedAccountData := []string{ - blockNumberString, - blockHash, - accountAction, - codeHash, - nonce, - balance, - *newContractRoot, - storagePath, - address, - *storage.Key, - *storage.Value, - } - - result = append(result, formattedAccountData) - } - } else { - formattedAccountData := []string{ - blockNumberString, - blockHash, - accountAction, - codeHash, - nonce, - balance, - *newContractRoot, - "", - address, - "", - "", - } - result = append(result, formattedAccountData) - } - - return result -} diff --git a/statediff/publisher/publisher.go b/statediff/publisher/publisher.go deleted file mode 100644 index ff1925513..000000000 --- a/statediff/publisher/publisher.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package publisher - -import ( - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/builder" -) - -type Publisher interface { - PublishStateDiff(sd *builder.StateDiff) (string, error) -} - -type publisher struct { - Config statediff.Config -} - -func NewPublisher(config statediff.Config) (*publisher, error) { - return &publisher{ - Config: config, - }, nil -} - -func (p *publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { - switch p.Config.Mode { - case statediff.CSV: - return p.publishStateDiffToCSV(*sd) - default: - return p.publishStateDiffToCSV(*sd) - } -} diff --git a/statediff/publisher/publisher_test.go b/statediff/publisher/publisher_test.go deleted file mode 100644 index 76aaf961e..000000000 --- a/statediff/publisher/publisher_test.go +++ /dev/null @@ -1,316 +0,0 @@ -package publisher_test - -import ( - "bytes" - "encoding/csv" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - "testing" - - "github.com/ethereum/go-ethereum/statediff" - "github.com/ethereum/go-ethereum/statediff/builder" - p "github.com/ethereum/go-ethereum/statediff/publisher" - "github.com/ethereum/go-ethereum/statediff/testhelpers" - "github.com/pkg/errors" -) - -var ( - tempDir = os.TempDir() - testFilePrefix = "test-statediff" - publisher p.Publisher - dir string - err error -) - -var expectedCreatedAccountRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "created", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - testhelpers.StoragePath, - testhelpers.ContractLeafKey.Hex(), - "0000000000000000000000000000000000000000000000000000000000000001", - testhelpers.StorageValue, -} - -var expectedCreatedAccountWithoutStorageUpdateRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "created", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - "", - testhelpers.AnotherContractLeafKey.Hex(), - "", - "", -} - -var expectedUpdatedAccountRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "updated", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - testhelpers.StoragePath, - testhelpers.ContractLeafKey.Hex(), - "0000000000000000000000000000000000000000000000000000000000000001", - testhelpers.StorageValue, -} - -var expectedDeletedAccountRow = []string{ - strconv.FormatInt(testhelpers.BlockNumber, 10), - testhelpers.BlockHash, - "deleted", - testhelpers.CodeHash, - strconv.FormatUint(testhelpers.NewNonceValue, 10), - strconv.FormatInt(testhelpers.NewBalanceValue, 10), - testhelpers.ContractRoot, - testhelpers.StoragePath, - testhelpers.ContractLeafKey.Hex(), - "0000000000000000000000000000000000000000000000000000000000000001", - testhelpers.StorageValue, -} - -func TestPublisher(t *testing.T) { - dir, err = ioutil.TempDir(tempDir, testFilePrefix) - if err != nil { - t.Error(err) - } - config := statediff.Config{ - Path: dir, - Mode: statediff.CSV, - } - publisher, err = p.NewPublisher(config) - if err != nil { - t.Error(err) - } - - type Test func(t *testing.T) - - var tests = []Test{ - testFileName, - testColumnHeaders, - testAccountDiffs, - testWhenNoDiff, - testDefaultPublisher, - testDefaultDirectory, - } - - for _, test := range tests { - test(t) - err := removeFilesFromDir(dir) - if err != nil { - t.Errorf("Error removing files from temp dir: %s", dir) - } - } -} - -func removeFilesFromDir(dir string) error { - files, err := filepath.Glob(filepath.Join(dir, "*")) - if err != nil { - return err - } - - for _, file := range files { - err = os.RemoveAll(file) - if err != nil { - return err - } - } - return nil -} - -func testFileName(t *testing.T) { - fileName, err := publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - if !strings.HasPrefix(fileName, dir) { - t.Errorf(testhelpers.TestFailureFormatString, t.Name(), dir, fileName) - } - blockNumberWithFileExt := strconv.FormatInt(testhelpers.BlockNumber, 10) + ".csv" - if !strings.HasSuffix(fileName, blockNumberWithFileExt) { - t.Errorf(testhelpers.TestFailureFormatString, t.Name(), blockNumberWithFileExt, fileName) - } -} - -func testColumnHeaders(t *testing.T) { - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if len(lines) < 1 { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[0], p.Headers) { - t.Error() - } -} - -func testAccountDiffs(t *testing.T) { - // it persists the created, updated and deleted account diffs to a CSV file - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if len(lines) <= 3 { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[1], expectedCreatedAccountRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[2], expectedCreatedAccountWithoutStorageUpdateRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[3], expectedUpdatedAccountRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[4], expectedDeletedAccountRow) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func testWhenNoDiff(t *testing.T) { - //it creates an empty CSV when there is no diff - emptyDiff := builder.StateDiff{} - _, err = publisher.PublishStateDiff(&emptyDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - if !equals(len(lines), 1) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func testDefaultPublisher(t *testing.T) { - //it defaults to publishing state diffs to a CSV file when no mode is configured - config := statediff.Config{Path: dir} - publisher, err = p.NewPublisher(config) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(len(lines), 5) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[0], p.Headers) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func testDefaultDirectory(t *testing.T) { - //it defaults to publishing CSV files in the current directory when no path is configured - config := statediff.Config{} - publisher, err = p.NewPublisher(config) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - err := os.Chdir(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - _, err = publisher.PublishStateDiff(&testhelpers.TestStateDiff) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - file, err := getTestDiffFile(dir) - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - - lines, err := csv.NewReader(file).ReadAll() - if err != nil { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(len(lines), 5) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } - if !equals(lines[0], p.Headers) { - t.Errorf(testhelpers.ErrorFormatString, t.Name(), err) - } -} - -func getTestDiffFile(dir string) (*os.File, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, err - } - if len(files) == 0 { - return nil, errors.New("There are 0 files.") - } - - fileName := files[0].Name() - filePath := filepath.Join(dir, fileName) - - return os.Open(filePath) -} - -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go index 0bfc53e5e..5126c6556 100644 --- a/statediff/testhelpers/helpers.go +++ b/statediff/testhelpers/helpers.go @@ -1,4 +1,83 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package testhelpers -var ErrorFormatString = "Error: %s, %+v" -var TestFailureFormatString = "Test failed: %s\nexpected %+v, got %+v\n" +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +// MakeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func MakeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), Testdb, n, testChainGen) + headers := make([]*types.Header, len(blocks)) + for i, block := range blocks { + headers[i] = block.Header() + } + chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil) + + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm, chain +} + +func testChainGen(i int, block *core.BlockGen) { + signer := types.HomesteadSigner{} + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // account1Addr passes it on to account #2. + // account1Addr creates a test contract. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey) + nonce := block.TxNonce(Account1Addr) + tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key) + nonce++ + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key) + ContractAddr = crypto.CreateAddress(Account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 + block.AddTx(tx1) + block.AddTx(tx2) + block.AddTx(tx3) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(Account2Addr) + //get function: 60cd2685 + //put function: c16431b9 + data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + block.AddTx(tx) + } +} diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index 7b0d74c59..cececde6f 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -1,3 +1,19 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package mocks import ( @@ -11,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/event" ) +// BlockChain is a mock blockchain for testing type BlockChain struct { ParentHashesLookedUp []common.Hash parentBlocksToReturn []*types.Block @@ -18,34 +35,39 @@ type BlockChain struct { ChainEvents []core.ChainEvent } -func (mc *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} +// AddToStateDiffProcessedCollection mock method +func (blockChain *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} -func (mc *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { - mc.parentBlocksToReturn = blocks +// SetParentBlocksToReturn mock method +func (blockChain *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { + blockChain.parentBlocksToReturn = blocks } -func (mc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { - mc.ParentHashesLookedUp = append(mc.ParentHashesLookedUp, hash) +// GetBlockByHash mock method +func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + blockChain.ParentHashesLookedUp = append(blockChain.ParentHashesLookedUp, hash) var parentBlock *types.Block - if len(mc.parentBlocksToReturn) > 0 { - parentBlock = mc.parentBlocksToReturn[mc.callCount] + if len(blockChain.parentBlocksToReturn) > 0 { + parentBlock = blockChain.parentBlocksToReturn[blockChain.callCount] } - mc.callCount++ + blockChain.callCount++ return parentBlock } -func (bc *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { - bc.ChainEvents = chainEvents +// SetChainEvents mock method +func (blockChain *BlockChain) SetChainEvents(chainEvents []core.ChainEvent) { + blockChain.ChainEvents = chainEvents } -func (bc *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { +// SubscribeChainEvent mock method +func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { subErr := errors.New("Subscription Error") var eventCounter int subscription := event.NewSubscription(func(quit <-chan struct{}) error { - for _, chainEvent := range bc.ChainEvents { + for _, chainEvent := range blockChain.ChainEvents { if eventCounter > 1 { time.Sleep(250 * time.Millisecond) return subErr diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index ae9ff5ced..e9668629e 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -1,20 +1,38 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package mocks import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/statediff" ) +// Builder is a mock state diff builder type Builder struct { OldStateRoot common.Hash NewStateRoot common.Hash BlockNumber int64 BlockHash common.Hash - stateDiff *builder.StateDiff + stateDiff statediff.StateDiff builderError error } -func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*builder.StateDiff, error) { +// BuildStateDiff mock method +func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (statediff.StateDiff, error) { builder.OldStateRoot = oldStateRoot builder.NewStateRoot = newStateRoot builder.BlockNumber = blockNumber @@ -23,10 +41,12 @@ func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, b return builder.stateDiff, builder.builderError } -func (builder *Builder) SetStateDiffToBuild(stateDiff *builder.StateDiff) { +// SetStateDiffToBuild mock method +func (builder *Builder) SetStateDiffToBuild(stateDiff statediff.StateDiff) { builder.stateDiff = stateDiff } +// SetBuilderError mock method func (builder *Builder) SetBuilderError(err error) { builder.builderError = err } diff --git a/statediff/testhelpers/mocks/error.go b/statediff/testhelpers/mocks/error.go deleted file mode 100644 index 7c40452ae..000000000 --- a/statediff/testhelpers/mocks/error.go +++ /dev/null @@ -1,5 +0,0 @@ -package mocks - -import "errors" - -var Error = errors.New("mock error") diff --git a/statediff/testhelpers/mocks/extractor.go b/statediff/testhelpers/mocks/extractor.go deleted file mode 100644 index 067497646..000000000 --- a/statediff/testhelpers/mocks/extractor.go +++ /dev/null @@ -1,20 +0,0 @@ -package mocks - -import "github.com/ethereum/go-ethereum/core/types" - -type Extractor struct { - ParentBlocks []types.Block - CurrentBlocks []types.Block - extractError error -} - -func (me *Extractor) ExtractStateDiff(parent, current types.Block) (string, error) { - me.ParentBlocks = append(me.ParentBlocks, parent) - me.CurrentBlocks = append(me.CurrentBlocks, current) - - return "", me.extractError -} - -func (me *Extractor) SetExtractError(err error) { - me.extractError = err -} diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go index efbe4e2ab..3ea18abf0 100644 --- a/statediff/testhelpers/mocks/publisher.go +++ b/statediff/testhelpers/mocks/publisher.go @@ -1,17 +1,36 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package mocks -import "github.com/ethereum/go-ethereum/statediff/builder" +import "github.com/ethereum/go-ethereum/statediff" +// Publisher mock type Publisher struct { - StateDiff *builder.StateDiff + StateDiff *statediff.StateDiff publisherError error } -func (publisher *Publisher) PublishStateDiff(sd *builder.StateDiff) (string, error) { +// PublishStateDiff mock method +func (publisher *Publisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { publisher.StateDiff = sd return "", publisher.publisherError } +// SetPublisherError mock method func (publisher *Publisher) SetPublisherError(err error) { publisher.publisherError = err } diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 831bed218..17dac8473 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -1,3 +1,19 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package testhelpers import ( @@ -5,71 +21,94 @@ import ( "math/rand" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/statediff/builder" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" ) +// AddressToLeafKey hashes an returns an address func AddressToLeafKey(address common.Address) common.Hash { return common.BytesToHash(crypto.Keccak256(address[:])) } +// Test variables var ( BlockNumber = rand.Int63() BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" - CodeHash = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") NewNonceValue = rand.Uint64() NewBalanceValue = rand.Int63() - ContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - StoragePath = "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" - StorageKey = "0000000000000000000000000000000000000000000000000000000000000001" - StorageValue = "0x03" - storage = map[string]builder.DiffStorage{StoragePath: { - Key: &StorageKey, - Value: &StorageValue, + ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() + StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() + StorageValue = common.Hex2Bytes("0x03") + storage = []statediff.StorageDiff{{ + Key: StorageKey, + Value: StorageValue, + Path: StoragePath, + Proof: [][]byte{}, }} - emptyStorage = map[string]builder.DiffStorage{} + emptyStorage = make([]statediff.StorageDiff, 0) address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") ContractLeafKey = AddressToLeafKey(address) anotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") AnotherContractLeafKey = AddressToLeafKey(anotherAddress) - CreatedAccountDiffs = builder.AccountDiffsMap{ + testAccount = state.Account{ + Nonce: NewNonceValue, + Balance: big.NewInt(NewBalanceValue), + Root: ContractRoot, + CodeHash: CodeHash, + } + valueBytes, _ = rlp.EncodeToBytes(testAccount) + CreatedAccountDiffs = statediff.AccountDiffsMap{ ContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - CodeHash: CodeHash, - Storage: storage, + Key: ContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storage, }, AnotherContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - CodeHash: CodeHash, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - Storage: emptyStorage, + Key: AnotherContractLeafKey.Bytes(), + Value: valueBytes, + Storage: emptyStorage, }, } - UpdatedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - CodeHash: CodeHash, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - Storage: storage, + UpdatedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + Key: ContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storage, }} - DeletedAccountDiffs = builder.AccountDiffsMap{ContractLeafKey: { - Nonce: builder.DiffUint64{Value: &NewNonceValue}, - Balance: builder.DiffBigInt{Value: big.NewInt(NewBalanceValue)}, - ContractRoot: builder.DiffString{Value: &ContractRoot}, - CodeHash: CodeHash, - Storage: storage, + DeletedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + Key: ContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storage, }} - TestStateDiff = builder.StateDiff{ + TestStateDiff = statediff.StateDiff{ BlockNumber: BlockNumber, BlockHash: common.HexToHash(BlockHash), CreatedAccounts: CreatedAccountDiffs, DeletedAccounts: DeletedAccountDiffs, UpdatedAccounts: UpdatedAccountDiffs, } + Testdb = rawdb.NewMemoryDatabase() + + TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 + BankLeafKey = AddressToLeafKey(TestBankAddress) + TestBankFunds = big.NewInt(100000000) + Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) + + Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + Account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + Account1Addr = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 + Account2Addr = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e + Account1LeafKey = AddressToLeafKey(Account1Addr) + Account2LeafKey = AddressToLeafKey(Account2Addr) + ContractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") + ContractAddr common.Address ) -- 2.45.2 From eca50a6078862e1a60d5453d2cd1372c3c85e578 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 26 Apr 2019 11:16:50 -0500 Subject: [PATCH 08/17] refactoring statediff builder and types and adjusted to relay proofs and paths (still need to make this optional) --- statediff/{builder => }/builder.go | 147 +++++------ statediff/builder/builder_test.go | 365 --------------------------- statediff/builder/helpers.go | 114 --------- statediff/builder/struct.go | 79 ------ statediff/builder_test.go | 392 +++++++++++++++++++++++++++++ statediff/types.go | 113 +++++++++ 6 files changed, 581 insertions(+), 629 deletions(-) rename statediff/{builder => }/builder.go (70%) delete mode 100644 statediff/builder/builder_test.go delete mode 100644 statediff/builder/helpers.go delete mode 100644 statediff/builder/struct.go create mode 100644 statediff/builder_test.go create mode 100644 statediff/types.go diff --git a/statediff/builder/builder.go b/statediff/builder.go similarity index 70% rename from statediff/builder/builder.go rename to statediff/builder.go index a106c3ad8..4e8ab0e21 100644 --- a/statediff/builder/builder.go +++ b/statediff/builder.go @@ -17,7 +17,7 @@ // Contains a batch of utility type declarations used by the tests. As the node // operates on unique types, a lot of them are needed to check various features. -package builder +package statediff import ( "github.com/ethereum/go-ethereum/common" @@ -30,8 +30,9 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// Builder interface exposes the method for building a state diff between two blocks type Builder interface { - BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) } type builder struct { @@ -39,27 +40,27 @@ type builder struct { blockChain *core.BlockChain } -type AccountsMap map[common.Hash]*state.Account - -func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) *builder { +// NewBuilder is used to create a builder +func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) Builder { return &builder{ chainDB: db, blockChain: blockChain, } } -func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (*StateDiff, error) { +// BuildStateDiff builds a StateDiff object from two blocks +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states stateCache := sdb.blockChain.StateCache() oldTrie, err := stateCache.OpenTrie(oldStateRoot) if err != nil { log.Error("Error creating trie for oldStateRoot", "error", err) - return nil, err + return StateDiff{}, err } newTrie, err := stateCache.OpenTrie(newStateRoot) if err != nil { log.Error("Error creating trie for newStateRoot", "error", err) - return nil, err + return StateDiff{}, err } // Find created accounts @@ -68,7 +69,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { log.Error("Error collecting creation diff nodes", "error", err) - return nil, err + return StateDiff{}, err } // Find deleted accounts @@ -77,7 +78,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { log.Error("Error collecting deletion diff nodes", "error", err) - return nil, err + return StateDiff{}, err } // Find all the diffed keys @@ -89,20 +90,20 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { log.Error("Error building diff for updated accounts", "error", err) - return nil, err + return StateDiff{}, err } createdAccounts, err := sdb.buildDiffEventual(creations) if err != nil { log.Error("Error building diff for created accounts", "error", err) - return nil, err + return StateDiff{}, err } deletedAccounts, err := sdb.buildDiffEventual(deletions) if err != nil { log.Error("Error building diff for deleted accounts", "error", err) - return nil, err + return StateDiff{}, err } - return &StateDiff{ + return StateDiff{ BlockNumber: blockNumber, BlockHash: blockHash, CreatedAccounts: createdAccounts, @@ -118,20 +119,31 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error for { log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) if it.Leaf() { + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) leafKey := make([]byte, len(it.LeafKey())) copy(leafKey, it.LeafKey()) leafKeyHash := common.BytesToHash(leafKey) - + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) // lookup account state var account state.Account - if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil { + if err := rlp.DecodeBytes(leafValue, &account); err != nil { log.Error("Error looking up account via address", "address", leafKeyHash, "error", err) return nil, err } - + aw := accountWrapper{ + Account: account, + RawKey: leafKey, + RawValue: leafValue, + Proof: leafProof, + Path: leafPath, + } // record account to diffs (creation if we are looking at new - old; deletion if old - new) log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = &account + diffAccounts[leafKeyHash] = aw } cont := it.Next(true) if !cont { @@ -144,25 +156,18 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { accountDiffs := make(AccountDiffsMap) - for addr, val := range accounts { - sr := val.Root - storageDiffs, err := sdb.buildStorageDiffsEventual(sr) + for _, val := range accounts { + storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root) if err != nil { - log.Error("Failed building eventual storage diffs", "Address", addr, "error", err) + log.Error("Failed building eventual storage diffs", "Address", common.BytesToHash(val.RawKey), "error", err) return nil, err } - - codeHash := hexutil.Encode(val.CodeHash) - hexRoot := val.Root.Hex() - nonce := DiffUint64{Value: &val.Nonce} - balance := DiffBigInt{Value: val.Balance} - contractRoot := DiffString{Value: &hexRoot} - accountDiffs[addr] = AccountDiff{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - ContractRoot: contractRoot, - Storage: storageDiffs, + accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ + Key: val.RawKey, + Value: val.RawValue, + Proof: val.Proof, + Path: val.Path, + Storage: storageDiffs, } } @@ -174,34 +179,28 @@ func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions Accoun for _, val := range updatedKeys { createdAcc := creations[common.HexToHash(val)] deletedAcc := deletions[common.HexToHash(val)] - oldSR := deletedAcc.Root - newSR := createdAcc.Root - if storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR); err != nil { + oldSR := deletedAcc.Account.Root + newSR := createdAcc.Account.Root + storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR) + if err != nil { log.Error("Failed building storage diffs", "Address", val, "error", err) return nil, err - } else { - nonce := DiffUint64{Value: &createdAcc.Nonce} - balance := DiffBigInt{Value: createdAcc.Balance} - codeHash := hexutil.Encode(createdAcc.CodeHash) - - nHexRoot := createdAcc.Root.Hex() - contractRoot := DiffString{Value: &nHexRoot} - - updatedAccounts[common.HexToHash(val)] = AccountDiff{ - Nonce: nonce, - Balance: balance, - CodeHash: codeHash, - ContractRoot: contractRoot, - Storage: storageDiffs, - } - delete(creations, common.HexToHash(val)) - delete(deletions, common.HexToHash(val)) } + updatedAccounts[common.HexToHash(val)] = AccountDiff{ + Key: createdAcc.RawKey, + Value: createdAcc.RawValue, + Proof: createdAcc.Proof, + Path: createdAcc.Path, + Storage: storageDiffs, + } + delete(creations, common.HexToHash(val)) + delete(deletions, common.HexToHash(val)) } + return updatedAccounts, nil } -func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffStorage, error) { +func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, error) { log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) stateCache := sdb.blockChain.StateCache() sTrie, err := stateCache.OpenTrie(sr) @@ -214,7 +213,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) (map[string]DiffSt return storageDiffs, nil } -func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) (map[string]DiffStorage, error) { +func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) ([]StorageDiff, error) { log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) stateCache := sdb.blockChain.StateCache() @@ -235,21 +234,27 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common return storageDiffs, nil } -func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage { - storageDiffs := make(map[string]DiffStorage) +func buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { + storageDiffs := make([]StorageDiff, 0) for { log.Debug("Iterating over state at path ", "path", pathToStr(it)) if it.Leaf() { log.Debug("Found leaf in storage", "path", pathToStr(it)) - path := pathToStr(it) - storageKey := hexutil.Encode(it.LeafKey()) - storageValue := hexutil.Encode(it.LeafBlob()) - storageDiffs[path] = DiffStorage{ - Key: &storageKey, - Value: &storageValue, - } + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: leafPath, + Proof: leafProof, + }) } - cont := it.Next(true) if !cont { break @@ -261,12 +266,12 @@ func buildStorageDiffsFromTrie(it trie.NodeIterator) map[string]DiffStorage { func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...))) - if addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)); err != nil { + addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)) + if err != nil { log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err) return nil, err - } else { - addr := common.BytesToAddress(addrBytes) - log.Debug("Address found", "Address", addr) - return &addr, nil } + addr := common.BytesToAddress(addrBytes) + log.Debug("Address found", "Address", addr) + return &addr, nil } diff --git a/statediff/builder/builder_test.go b/statediff/builder/builder_test.go deleted file mode 100644 index 4f8b7c468..000000000 --- a/statediff/builder/builder_test.go +++ /dev/null @@ -1,365 +0,0 @@ -package builder_test - -import ( - "bytes" - "math/big" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" - b "github.com/ethereum/go-ethereum/statediff/builder" - "github.com/ethereum/go-ethereum/statediff/testhelpers" -) - -var ( - testdb = ethdb.NewMemDatabase() - - testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 - bankLeafKey = testhelpers.AddressToLeafKey(testBankAddress) - testBankFunds = big.NewInt(100000000) - genesis = core.GenesisBlockForTesting(testdb, testBankAddress, testBankFunds) - - account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - account2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - account1Addr = crypto.PubkeyToAddress(account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 - account1LeafKey = testhelpers.AddressToLeafKey(account1Addr) - account2Addr = crypto.PubkeyToAddress(account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e - account2LeafKey = testhelpers.AddressToLeafKey(account2Addr) - contractCode = common.Hex2Bytes("608060405234801561001057600080fd5b50602060405190810160405280600160ff16815250600090600161003592919061003b565b506100a5565b826064810192821561006f579160200282015b8281111561006e578251829060ff1690559160200191906001019061004e565b5b50905061007c9190610080565b5090565b6100a291905b8082111561009e576000816000905550600101610086565b5090565b90565b610124806100b46000396000f3fe6080604052348015600f57600080fd5b5060043610604f576000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146054578063c16431b9146093575b600080fd5b607d60048036036020811015606857600080fd5b810190808035906020019092919050505060c8565b6040518082815260200191505060405180910390f35b60c66004803603604081101560a757600080fd5b81019080803590602001909291908035906020019092919050505060e0565b005b6000808260648110151560d757fe5b01549050919050565b8060008360648110151560ef57fe5b0181905550505056fea165627a7a7230582064e918c3140a117bf3aa65865a9b9e83fae21ad1720506e7933b2a9f54bb40260029") - contractAddr common.Address - contractLeafKey common.Hash - emptyAccountDiffEventualMap = make(b.AccountDiffsMap) - emptyAccountDiffIncrementalMap = make(b.AccountDiffsMap) - block0Hash, block1Hash, block2Hash, block3Hash common.Hash - block0, block1, block2, block3 *types.Block - builder b.Builder - miningReward = int64(2000000000000000000) - burnAddress = common.HexToAddress("0x0") - burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) -) - -func TestBuilder(t *testing.T) { - _, blockMap, chain := makeChain(3, genesis) - contractLeafKey = testhelpers.AddressToLeafKey(contractAddr) - defer chain.Stop() - block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") - block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") - block2Hash = common.HexToHash("0xde75663f36a8497b4bdda2a4b52bd9540b705a2728c7391c59b8cb2cde5a2feb") - block3Hash = common.HexToHash("0x76c6d0e39285cee40d5e5fadc6141ca88c8ab8bd1a15d46717205af2efbb4a3c") - - block0 = blockMap[block0Hash] - block1 = blockMap[block1Hash] - block2 = blockMap[block2Hash] - block3 = blockMap[block3Hash] - builder = b.NewBuilder(testdb, chain) - - type arguments struct { - oldStateRoot common.Hash - newStateRoot common.Hash - blockNumber int64 - blockHash common.Hash - } - - var ( - balanceChange10000 = int64(10000) - balanceChange1000 = int64(1000) - block1BankBalance = int64(99990000) - block1Account1Balance = int64(10000) - block2Account2Balance = int64(1000) - nonce0 = uint64(0) - nonce1 = uint64(1) - nonce2 = uint64(2) - nonce3 = uint64(3) - originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" - newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" - originalStorageLocation = common.HexToHash("0") - originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).String() - updatedStorageLocation = common.HexToHash("2") - updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).String() - originalStorageValue = "0x01" - updatedStorageValue = "0x03" - ) - - var tests = []struct { - name string - startingArguments arguments - expected *b.StateDiff - }{ - { - "testEmptyDiff", - arguments{ - oldStateRoot: block0.Root(), - newStateRoot: block0.Root(), - blockNumber: block0.Number().Int64(), - blockHash: block0Hash, - }, - &b.StateDiff{ - BlockNumber: block0.Number().Int64(), - BlockHash: block0Hash, - CreatedAccounts: emptyAccountDiffEventualMap, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: emptyAccountDiffIncrementalMap, - }, - }, - { - "testBlock1", - //10000 transferred from testBankAddress to account1Addr - arguments{ - oldStateRoot: block0.Root(), - newStateRoot: block1.Root(), - blockNumber: block1.Number().Int64(), - blockHash: block1Hash, - }, - &b.StateDiff{ - BlockNumber: block1.Number().Int64(), - BlockHash: block1.Hash(), - CreatedAccounts: b.AccountDiffsMap{ - account1LeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(balanceChange10000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - burnLeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(miningReward)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: b.AccountDiffsMap{ - bankLeafKey: { - Nonce: b.DiffUint64{Value: &nonce1}, - Balance: b.DiffBigInt{Value: big.NewInt(testBankFunds.Int64() - balanceChange10000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - }, - }, - { - "testBlock2", - //1000 transferred from testBankAddress to account1Addr - //1000 transferred from account1Addr to account2Addr - arguments{ - oldStateRoot: block1.Root(), - newStateRoot: block2.Root(), - blockNumber: block2.Number().Int64(), - blockHash: block2Hash, - }, - &b.StateDiff{ - BlockNumber: block2.Number().Int64(), - BlockHash: block2.Hash(), - CreatedAccounts: b.AccountDiffsMap{ - account2LeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(balanceChange1000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - contractLeafKey: { - Nonce: b.DiffUint64{Value: &nonce1}, - Balance: b.DiffBigInt{Value: big.NewInt(0)}, - CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", - ContractRoot: b.DiffString{Value: &contractContractRoot}, - Storage: map[string]b.DiffStorage{ - originalStorageKey: { - Key: &originalStorageKey, - Value: &originalStorageValue}, - }, - }, - }, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: b.AccountDiffsMap{ - bankLeafKey: { - Nonce: b.DiffUint64{Value: &nonce2}, - Balance: b.DiffBigInt{Value: big.NewInt(block1BankBalance - balanceChange1000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - account1LeafKey: { - Nonce: b.DiffUint64{Value: &nonce2}, - Balance: b.DiffBigInt{Value: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - burnLeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(miningReward + miningReward)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - }, - }, - { - "testBlock3", - //the contract's storage is changed - //and the block is mined by account 2 - arguments{ - oldStateRoot: block2.Root(), - newStateRoot: block3.Root(), - blockNumber: block3.Number().Int64(), - blockHash: block3.Hash(), - }, - &b.StateDiff{ - BlockNumber: block3.Number().Int64(), - BlockHash: block3.Hash(), - CreatedAccounts: b.AccountDiffsMap{}, - DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: b.AccountDiffsMap{ - account2LeafKey: { - Nonce: b.DiffUint64{Value: &nonce0}, - Balance: b.DiffBigInt{Value: big.NewInt(block2Account2Balance + miningReward)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - contractLeafKey: { - Nonce: b.DiffUint64{Value: &nonce1}, - Balance: b.DiffBigInt{Value: big.NewInt(0)}, - CodeHash: "0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea", - ContractRoot: b.DiffString{Value: &newContractRoot}, - Storage: map[string]b.DiffStorage{ - "0x405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace": { - Key: &updatedStorageKey, - Value: &updatedStorageValue}, - }, - }, - bankLeafKey: { - Nonce: b.DiffUint64{Value: &nonce3}, - Balance: b.DiffBigInt{Value: big.NewInt(99989000)}, - CodeHash: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - ContractRoot: b.DiffString{Value: &originalContractRoot}, - Storage: map[string]b.DiffStorage{}, - }, - }, - }, - }, - } - - for _, test := range tests { - arguments := test.startingArguments - diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) - if err != nil { - t.Error(err) - } - fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} - - for _, field := range fields { - reflectionOfDiff := reflect.ValueOf(diff) - diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) - - reflectionOfExpected := reflect.ValueOf(test.expected) - expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) - - diffValueInterface := diffValue.Interface() - expectedValueInterface := expectedValue.Interface() - - if !equals(diffValueInterface, expectedValueInterface) { - t.Logf("Test failed: %s", test.name) - t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) - } - } - } -} - -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block, *core.BlockChain) { - blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, testChainGen) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - chain, _ := core.NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil) - - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - blockm[b.Hash()] = b - } - return hashes, blockm, chain -} - -func testChainGen(i int, block *core.BlockGen) { - signer := types.HomesteadSigner{} - switch i { - case 0: - // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) - block.AddTx(tx) - case 1: - // In block 2, the test bank sends some more ether to account #1. - // account1Addr passes it on to account #2. - // account1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - nonce := block.TxNonce(account1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, account1Key) - nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), contractCode), signer, account1Key) - contractAddr = crypto.CreateAddress(account1Addr, nonce) //0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592 - block.AddTx(tx1) - block.AddTx(tx2) - block.AddTx(tx3) - case 2: - // Block 3 is empty but was mined by account #2. - block.SetCoinbase(account2Addr) - //get function: 60cd2685 - //put function: c16431b9 - data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), contractAddr, big.NewInt(0), 100000, nil, data), signer, testBankKey) - block.AddTx(tx) - } -} - -/* -contract test { - - uint256[100] data; - - constructor() public { - data = [1]; - } - - function Put(uint256 addr, uint256 value) { - data[addr] = value; - } - - function Get(uint256 addr) constant returns (uint256 value) { - return data[addr]; - } -} -*/ diff --git a/statediff/builder/helpers.go b/statediff/builder/helpers.go deleted file mode 100644 index 339454776..000000000 --- a/statediff/builder/helpers.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package builder - -import ( - "sort" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/trie" -) - -func sortKeys(data AccountsMap) []string { - var keys []string - for key := range data { - keys = append(keys, key.Hex()) - } - sort.Strings(keys) - - return keys -} - -func findIntersection(a, b []string) []string { - lenA := len(a) - lenB := len(b) - iOfA, iOfB := 0, 0 - updates := make([]string, 0) - if iOfA >= lenA || iOfB >= lenB { - return updates - } - for { - switch strings.Compare(a[iOfA], b[iOfB]) { - // a[iOfA] < b[iOfB] - case -1: - iOfA++ - if iOfA >= lenA { - return updates - } - // a[iOfA] == b[iOfB] - case 0: - updates = append(updates, a[iOfA]) - iOfA++ - iOfB++ - if iOfA >= lenA || iOfB >= lenB { - return updates - } - // a[iOfA] > b[iOfB] - case 1: - iOfB++ - if iOfB >= lenB { - return updates - } - } - } - -} - -func pathToStr(it trie.NodeIterator) string { - path := it.Path() - if hasTerm(path) { - path = path[:len(path)-1] - } - nibblePath := "" - for i, v := range common.ToHex(path) { - if i%2 == 0 && i > 1 { - continue - } - nibblePath = nibblePath + string(v) - } - - return nibblePath -} - -// Duplicated from trie/encoding.go -func hexToKeyBytes(hex []byte) []byte { - if hasTerm(hex) { - hex = hex[:len(hex)-1] - } - if len(hex)&1 != 0 { - panic("can't convert hex key of odd length") - } - key := make([]byte, (len(hex)+1)/2) - decodeNibbles(hex, key) - - return key -} - -func decodeNibbles(nibbles []byte, bytes []byte) { - for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { - bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] - } -} - -// hasTerm returns whether a hex key has the terminator flag. -func hasTerm(s []byte) bool { - return len(s) > 0 && s[len(s)-1] == 16 -} diff --git a/statediff/builder/struct.go b/statediff/builder/struct.go deleted file mode 100644 index 416d8e816..000000000 --- a/statediff/builder/struct.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains a batch of utility type declarations used by the tests. As the node -// operates on unique types, a lot of them are needed to check various features. - -package builder - -import ( - "encoding/json" - "math/big" - - "github.com/ethereum/go-ethereum/common" -) - -type AccountDiffsMap map[common.Hash]AccountDiff -type StateDiff struct { - BlockNumber int64 `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` - DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` - UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` - - encoded []byte - err error -} - -func (self *StateDiff) ensureEncoded() { - if self.encoded == nil && self.err == nil { - self.encoded, self.err = json.Marshal(self) - } -} - -// Implement Encoder interface for StateDiff -func (sd *StateDiff) Length() int { - sd.ensureEncoded() - return len(sd.encoded) -} - -// Implement Encoder interface for StateDiff -func (sd *StateDiff) Encode() ([]byte, error) { - sd.ensureEncoded() - return sd.encoded, sd.err -} - -type AccountDiff struct { - Nonce DiffUint64 `json:"nonce" gencodec:"required"` - Balance DiffBigInt `json:"balance" gencodec:"required"` - CodeHash string `json:"codeHash" gencodec:"required"` - ContractRoot DiffString `json:"contractRoot" gencodec:"required"` - Storage map[string]DiffStorage `json:"storage" gencodec:"required"` -} - -type DiffStorage struct { - Key *string `json:"key" gencodec:"optional"` - Value *string `json:"value" gencodec:"optional"` -} -type DiffString struct { - Value *string `json:"value" gencodec:"optional"` -} -type DiffUint64 struct { - Value *uint64 `json:"value" gencodec:"optional"` -} -type DiffBigInt struct { - Value *big.Int `json:"value" gencodec:"optional"` -} diff --git a/statediff/builder_test.go b/statediff/builder_test.go new file mode 100644 index 000000000..b4df80f81 --- /dev/null +++ b/statediff/builder_test.go @@ -0,0 +1,392 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" +) + +var ( + contractLeafKey common.Hash + emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) + emptyAccountDiffIncrementalMap = make(statediff.AccountDiffsMap) + block0Hash, block1Hash, block2Hash, block3Hash common.Hash + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(2000000000000000000) + burnAddress = common.HexToAddress("0x0") + burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) +) + +func TestBuilder(t *testing.T) { + _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block2Hash = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5") + block3Hash = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a") + + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + block2 = blockMap[block2Hash] + block3 = blockMap[block3Hash] + builder = statediff.NewBuilder(testhelpers.Testdb, chain) + + type arguments struct { + oldStateRoot common.Hash + newStateRoot common.Hash + blockNumber int64 + blockHash common.Hash + } + + var ( + balanceChange10000 = int64(10000) + balanceChange1000 = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + block2Account2Balance = int64(1000) + nonce0 = uint64(0) + nonce1 = uint64(1) + nonce2 = uint64(2) + nonce3 = uint64(3) + originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" + originalStorageLocation = common.HexToHash("0") + originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes() + updatedStorageLocation = common.HexToHash("2") + updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes() + originalStorageValue = common.Hex2Bytes("01") + updatedStorageValue = common.Hex2Bytes("03") + account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + bankAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(contractContractRoot), + }) + bankAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1BankBalance - balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(block2Account2Balance + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(newContractRoot), + }) + bankAccount3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce3, + Balance: big.NewInt(99989000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + ) + + var tests = []struct { + name string + startingArguments arguments + expected *statediff.StateDiff + }{ + { + "testEmptyDiff", + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block0.Root(), + blockNumber: block0.Number().Int64(), + blockHash: block0Hash, + }, + &statediff.StateDiff{ + BlockNumber: block0.Number().Int64(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block1.Root(), + blockNumber: block1.Number().Int64(), + blockHash: block1Hash, + }, + &statediff.StateDiff{ + BlockNumber: block1.Number().Int64(), + BlockHash: block1.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account1LeafKey: { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + burnLeafKey: { + Key: burnLeafKey.Bytes(), + Value: burnAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 1, 132, 5, 245, 185, 240, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + arguments{ + oldStateRoot: block1.Root(), + newStateRoot: block2.Root(), + blockNumber: block2.Number().Int64(), + blockHash: block2Hash, + }, + &statediff.StateDiff{ + BlockNumber: block2.Number().Int64(), + BlockHash: block2.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account2LeafKey: { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + Storage: []statediff.StorageDiff{}, + }, + contractLeafKey: { + Key: contractLeafKey.Bytes(), + Value: contractAccount, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 130, 30, 37, 86, 162, 144, 200, 100, 5, 248, 22, 10, 45, 102, 32, 66, 164, 49, 186, 69, 107, 157, 178, 101, 199, 155, 184, 55, 192, 75, 229, 240, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Key: originalStorageKey, + Value: originalStorageValue, + Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}}, + Path: []byte{2, 9, 0, 13, 14, 12, 13, 9, 5, 4, 8, 11, 6, 2, 10, 8, 13, 6, 0, 3, 4, 5, 10, 9, 8, 8, 3, 8, 6, 15, 12, 8, 4, 11, 10, 6, 11, 12, 9, 5, 4, 8, 4, 0, 0, 8, 15, 6, 3, 6, 2, 15, 9, 3, 1, 6, 0, 14, 15, 3, 14, 5, 6, 3, 16}, + }, + }, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 2, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + testhelpers.Account1LeafKey: { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account3, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + burnLeafKey: { + Key: burnLeafKey.Bytes(), + Value: burnAccount2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 55, 130, 218, 206, 157, 144, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + arguments{ + oldStateRoot: block2.Root(), + newStateRoot: block3.Root(), + blockNumber: block3.Number().Int64(), + blockHash: block3.Hash(), + }, + &statediff.StateDiff{ + BlockNumber: block3.Number().Int64(), + BlockHash: block3.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{}, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account2LeafKey: { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account4, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + Storage: []statediff.StorageDiff{}, + }, + contractLeafKey: { + Key: contractLeafKey.Bytes(), + Value: contractAccount2, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 113, 224, 209, 75, 43, 147, 229, 199, 249, 116, 142, 105, 225, 254, 95, 23, 73, 138, 28, 58, 195, 206, 194, 159, 150, 175, 19, 215, 248, 164, 224, 112, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Key: updatedStorageKey, + Value: updatedStorageValue, + Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, + {226, 160, 48, 87, 135, 250, 18, 168, 35, 224, 242, 183, 99, 28, 196, 27, 59, 168, 130, 139, 51, 33, 202, 129, 17, 17, 250, 117, 205, 58, 163, 187, 90, 206, 3}}, + Path: []byte{4, 0, 5, 7, 8, 7, 15, 10, 1, 2, 10, 8, 2, 3, 14, 0, 15, 2, 11, 7, 6, 3, 1, 12, 12, 4, 1, 11, 3, 11, 10, 8, 8, 2, 8, 11, 3, 3, 2, 1, 12, 10, 8, 1, 1, 1, 1, 1, 15, 10, 7, 5, 12, 13, 3, 10, 10, 3, 11, 11, 5, 10, 12, 14, 16}, + }, + }, + }, + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount3, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + } + + for _, test := range tests { + arguments := test.startingArguments + diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) + if err != nil { + t.Error(err) + } + fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} + + for _, field := range fields { + reflectionOfDiff := reflect.ValueOf(diff) + diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) + + reflectionOfExpected := reflect.ValueOf(test.expected) + expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) + + diffValueInterface := diffValue.Interface() + expectedValueInterface := expectedValue.Interface() + + if !equals(diffValueInterface, expectedValueInterface) { + t.Logf("Test failed: %s", test.name) + t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) + } + } + } +} + +func equals(actual, expected interface{}) (success bool) { + if actualByteSlice, ok := actual.([]byte); ok { + if expectedByteSlice, ok := expected.([]byte); ok { + return bytes.Equal(actualByteSlice, expectedByteSlice) + } + } + + return reflect.DeepEqual(actual, expected) +} + +/* +contract test { + + uint256[100] data; + + constructor() public { + data = [1]; + } + + function Put(uint256 addr, uint256 value) { + data[addr] = value; + } + + function Get(uint256 addr) constant returns (uint256 value) { + return data[addr]; + } +} +*/ diff --git a/statediff/types.go b/statediff/types.go new file mode 100644 index 000000000..41e83dddb --- /dev/null +++ b/statediff/types.go @@ -0,0 +1,113 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package statediff + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" +) + +// AccountsMap is a mapping of keccak256(address) => accountWrapper +type AccountsMap map[common.Hash]accountWrapper + +// AccountWrapper is used to temporary associate the unpacked account with its raw values +type accountWrapper struct { + Account state.Account + RawKey []byte + RawValue []byte + Proof [][]byte + Path []byte +} + +// StateDiff is the final output structure from the builder +type StateDiff struct { + BlockNumber int64 `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` + DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` + + encoded []byte + err error +} + +func (sd *StateDiff) ensureEncoded() { + if sd.encoded == nil && sd.err == nil { + sd.encoded, sd.err = json.Marshal(sd) + } +} + +// Length to implement Encoder interface for StateDiff +func (sd *StateDiff) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Encode to implement Encoder interface for StateDiff +func (sd *StateDiff) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err +} + +// AccountDiffsMap is a mapping of keccak256(address) => AccountDiff +type AccountDiffsMap map[common.Hash]AccountDiff + +// AccountDiff holds the data for a single state diff leaf node +type AccountDiff struct { + Key []byte `json:"key" gencodec:"required"` + Value []byte `json:"value" gencodec:"required"` + Proof [][]byte `json:"proof" gencodec:"required"` + Storage []StorageDiff `json:"storage" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` +} + +// StorageDiff holds the data for a single storage diff leaf node +type StorageDiff struct { + Key []byte `json:"key" gencodec:"required"` + Value []byte `json:"value" gencodec:"required"` + Proof [][]byte `json:"proof" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` +} + +/* +// State trie leaf is just a short node, below +// that has an rlp encoded account as the value + + +// SO each account diffs map is reall a map of shortnode keys to values +// Flatten to a slice of short nodes? + +// Need to coerce into: + +type TrieNode struct { + // leaf, extension or branch + nodeKind string + + // If leaf or extension: [0] is key, [1] is val. + // If branch: [0] - [16] are children. + elements []interface{} + + // IPLD block information + cid *cid.Cid + rawdata []byte +} +*/ -- 2.45.2 From e1ed4951aa8828dd070d225b9e864392b71e1720 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 26 Apr 2019 11:18:23 -0500 Subject: [PATCH 09/17] refactoring state diff service and adding api which allows for streaming state diff payloads over an rpc websocket subscription --- statediff/api.go | 91 ++++++++ statediff/builder.go | 2 +- statediff/builder_test.go | 2 +- statediff/helpers.go | 2 +- statediff/service.go | 244 ++++++++++++++++++++ statediff/service/service.go | 120 ---------- statediff/service/service_test.go | 107 --------- statediff/service_test.go | 130 +++++++++++ statediff/testhelpers/helpers.go | 2 +- statediff/testhelpers/mocks/blockchain.go | 2 +- statediff/testhelpers/mocks/builder.go | 2 +- statediff/testhelpers/mocks/publisher.go | 2 +- statediff/testhelpers/mocks/service.go | 171 ++++++++++++++ statediff/testhelpers/mocks/service_test.go | 127 ++++++++++ statediff/testhelpers/test_data.go | 2 +- statediff/types.go | 2 +- 16 files changed, 772 insertions(+), 236 deletions(-) create mode 100644 statediff/api.go create mode 100644 statediff/service.go delete mode 100644 statediff/service/service.go delete mode 100644 statediff/service/service_test.go create mode 100644 statediff/service_test.go create mode 100644 statediff/testhelpers/mocks/service.go create mode 100644 statediff/testhelpers/mocks/service_test.go diff --git a/statediff/api.go b/statediff/api.go new file mode 100644 index 000000000..26f03c9c3 --- /dev/null +++ b/statediff/api.go @@ -0,0 +1,91 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "context" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +// APIName is the namespace used for the state diffing service API +const APIName = "statediff" + +// APIVersion is the version of the state diffing service API +const APIVersion = "0.0.1" + +// PublicStateDiffAPI provides the a websocket service +// that can be used to stream out state diffs as they +// are produced by a full node +type PublicStateDiffAPI struct { + sds IService + + mu sync.Mutex + lastUsed map[string]time.Time // keeps track when a filter was polled for the last time. +} + +// NewPublicStateDiffAPI create a new state diff websocket streaming service. +func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { + return &PublicStateDiffAPI{ + sds: sds, + lastUsed: make(map[string]time.Time), + mu: sync.Mutex{}, + } +} + +// Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created +func (api *PublicStateDiffAPI) Subscribe(ctx context.Context) (*rpc.Subscription, error) { + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // create subscription and start waiting for statediff events + rpcSub := notifier.CreateSubscription() + + go func() { + // subscribe to events from the state diff service + payloadChannel := make(chan Payload) + quitChan := make(chan bool) + api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) + + // loop and await state diff payloads and relay them to the subscriber with then notifier + for { + select { + case packet := <-payloadChannel: + if err := notifier.Notify(rpcSub.ID, packet); err != nil { + log.Error("Failed to send state diff packet", "err", err) + } + case <-rpcSub.Err(): + err := api.sds.Unsubscribe(rpcSub.ID) + if err != nil { + log.Error("Failed to unsubscribe from the state diff service", err) + } + return + case <-quitChan: + // don't need to unsubscribe, statediff service does so before sending the quit signal + return + } + } + }() + + return rpcSub, nil +} diff --git a/statediff/builder.go b/statediff/builder.go index 4e8ab0e21..05e7d8572 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/builder_test.go b/statediff/builder_test.go index b4df80f81..5ecf647e3 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/helpers.go b/statediff/helpers.go index 96f9ddf30..89852b55c 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/service.go b/statediff/service.go new file mode 100644 index 000000000..d03a7b513 --- /dev/null +++ b/statediff/service.go @@ -0,0 +1,244 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +import ( + "bytes" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rlp" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" +) + +type blockChain interface { + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + GetBlockByHash(hash common.Hash) *types.Block + AddToStateDiffProcessedCollection(hash common.Hash) +} + +// IService is the state-diffing service interface +type IService interface { + // APIs(), Protocols(), Start() and Stop() + node.Service + // Main event loop for processing state diffs + Loop(chainEventCh chan core.ChainEvent) + // Method to subscribe to receive state diff processing output + Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) + // Method to unsubscribe from state diff processing + Unsubscribe(id rpc.ID) error +} + +// Service is the underlying struct for the state diffing service +type Service struct { + // Used to sync access to the Subscriptions + sync.Mutex + // Used to build the state diff objects + Builder Builder + // Used to subscribe to chain events (blocks) + BlockChain blockChain + // Used to signal shutdown of the service + QuitChan chan bool + // A mapping of rpc.IDs to their subscription channels + Subscriptions map[rpc.ID]Subscription +} + +// Subscription struct holds our subscription channels +type Subscription struct { + PayloadChan chan<- Payload + QuitChan chan<- bool +} + +// Payload packages the data to send to StateDiffingService subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp" gencodec:"required"` + StateDiffRlp []byte `json:"stateDiffRlp" gencodec:"required"` + Err error `json:"error"` +} + +// NewStateDiffService creates a new StateDiffingService +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*Service, error) { + return &Service{ + Mutex: sync.Mutex{}, + BlockChain: blockChain, + Builder: NewBuilder(db, blockChain), + QuitChan: make(chan bool), + Subscriptions: make(map[rpc.ID]Subscription), + }, nil +} + +// Protocols exports the services p2p protocols, this service has none +func (sds *Service) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs returns the RPC descriptors the StateDiffingService offers +func (sds *Service) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: APIName, + Version: APIVersion, + Service: NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Loop is the main processing method +func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { + + chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) + defer chainEventSub.Unsubscribe() + + blocksCh := make(chan *types.Block, 10) + errCh := chainEventSub.Err() + + go func() { + HandleChainEventChLoop: + for { + select { + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + log.Debug("Event received from chainEventCh", "event", chainEvent) + blocksCh <- chainEvent.Block + //if node stopped + case err := <-errCh: + log.Warn("Error from chain event subscription, breaking loop.", "error", err) + close(sds.QuitChan) + break HandleChainEventChLoop + case <-sds.QuitChan: + break HandleChainEventChLoop + } + } + }() + + //loop through chain events until no more +HandleBlockChLoop: + for { + select { + case block := <-blocksCh: + currentBlock := block + parentHash := currentBlock.ParentHash() + parentBlock := sds.BlockChain.GetBlockByHash(parentHash) + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + break HandleBlockChLoop + } + + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + if err != nil { + log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) + } + rlpBuff := new(bytes.Buffer) + currentBlock.EncodeRLP(rlpBuff) + blockRlp := rlpBuff.Bytes() + stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) + payload := Payload{ + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, + } + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// Subscribe is used by the API to subscribe to the StateDiffingService loop +func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { + log.Info("Subscribing to the statediff service") + sds.Lock() + sds.Subscriptions[id] = Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.Unlock() +} + +// Unsubscribe is used to unsubscribe to the StateDiffingService loop +func (sds *Service) Unsubscribe(id rpc.ID) error { + log.Info("Unsubscribing from the statediff service") + sds.Lock() + _, ok := sds.Subscriptions[id] + if !ok { + return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id) + } + delete(sds.Subscriptions, id) + sds.Unlock() + return nil +} + +// Start is used to begin the StateDiffingService +func (sds *Service) Start(*p2p.Server) error { + log.Info("Starting statediff service") + + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + + return nil +} + +// Stop is used to close down the StateDiffingService +func (sds *Service) Stop() error { + log.Info("Stopping statediff service") + close(sds.QuitChan) + return nil +} + +// send is used to fan out and serve a payload to any subscriptions +func (sds *Service) send(payload Payload) { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.PayloadChan <- payload: + log.Info("sending state diff payload to subscription %s", id) + default: + log.Info("unable to send payload to subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} + +// close is used to close all listening subscriptions +func (sds *Service) close() { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.QuitChan <- true: + delete(sds.Subscriptions, id) + log.Info("closing subscription %s", id) + default: + log.Info("unable to close subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} diff --git a/statediff/service/service.go b/statediff/service/service.go deleted file mode 100644 index 19ea0c644..000000000 --- a/statediff/service/service.go +++ /dev/null @@ -1,120 +0,0 @@ -package service - -import ( - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff" - b "github.com/ethereum/go-ethereum/statediff/builder" - e "github.com/ethereum/go-ethereum/statediff/extractor" - p "github.com/ethereum/go-ethereum/statediff/publisher" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" -) - -type BlockChain interface { - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - GetBlockByHash(hash common.Hash) *types.Block - AddToStateDiffProcessedCollection(hash common.Hash) -} - -type StateDiffService struct { - Builder *b.Builder - Extractor e.Extractor - BlockChain BlockChain -} - -func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config statediff.Config) (*StateDiffService, error) { - builder := b.NewBuilder(db, blockChain) - publisher, err := p.NewPublisher(config) - if err != nil { - return nil, err - } - - extractor := e.NewExtractor(builder, publisher) - return &StateDiffService{ - BlockChain: blockChain, - Extractor: extractor, - }, nil -} - -func (StateDiffService) Protocols() []p2p.Protocol { - return []p2p.Protocol{} -} - -func (StateDiffService) APIs() []rpc.API { - return []rpc.API{} -} - -func (sds *StateDiffService) Loop(chainEventCh chan core.ChainEvent) { - chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) - defer chainEventSub.Unsubscribe() - - blocksCh := make(chan *types.Block, 10) - errCh := chainEventSub.Err() - quitCh := make(chan struct{}) - - go func() { - HandleChainEventChLoop: - for { - select { - //Notify chain event channel of events - case chainEvent := <-chainEventCh: - log.Debug("Event received from chainEventCh", "event", chainEvent) - blocksCh <- chainEvent.Block - //if node stopped - case err := <-errCh: - log.Warn("Error from chain event subscription, breaking loop.", "error", err) - break HandleChainEventChLoop - } - } - close(quitCh) - }() - - //loop through chain events until no more -HandleBlockChLoop: - for { - select { - case block := <-blocksCh: - currentBlock := block - parentHash := currentBlock.ParentHash() - parentBlock := sds.BlockChain.GetBlockByHash(parentHash) - if parentBlock == nil { - log.Error("Parent block is nil, skipping this block", - "parent block hash", parentHash.String(), - "current block number", currentBlock.Number()) - break HandleBlockChLoop - } - - stateDiffLocation, err := sds.Extractor.ExtractStateDiff(*parentBlock, *currentBlock) - if err != nil { - log.Error("Error extracting statediff", "block number", currentBlock.Number(), "error", err) - } else { - log.Info("Statediff extracted", "block number", currentBlock.Number(), "location", stateDiffLocation) - sds.BlockChain.AddToStateDiffProcessedCollection(parentBlock.Root()) - sds.BlockChain.AddToStateDiffProcessedCollection(currentBlock.Root()) - } - case <-quitCh: - log.Debug("Quitting the statediff block channel") - return - } - } -} - -func (sds *StateDiffService) Start(server *p2p.Server) error { - log.Info("Starting statediff service") - - chainEventCh := make(chan core.ChainEvent, 10) - go sds.Loop(chainEventCh) - - return nil -} - -func (StateDiffService) Stop() error { - log.Info("Stopping statediff service") - return nil -} diff --git a/statediff/service/service_test.go b/statediff/service/service_test.go deleted file mode 100644 index daf3445c3..000000000 --- a/statediff/service/service_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package service_test - -import ( - "math/big" - "math/rand" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - s "github.com/ethereum/go-ethereum/statediff/service" - "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" -) - -func TestServiceLoop(t *testing.T) { - testErrorInChainEventLoop(t) - testErrorInBlockLoop(t) -} - -var ( - eventsChannel = make(chan core.ChainEvent, 1) - - parentHeader1 = types.Header{Number: big.NewInt(rand.Int63())} - parentHeader2 = types.Header{Number: big.NewInt(rand.Int63())} - - parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil) - parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil) - - parentHash1 = parentBlock1.Hash() - parentHash2 = parentBlock2.Hash() - - header1 = types.Header{ParentHash: parentHash1} - header2 = types.Header{ParentHash: parentHash2} - header3 = types.Header{ParentHash: common.HexToHash("parent hash")} - - block1 = types.NewBlock(&header1, nil, nil, nil) - block2 = types.NewBlock(&header2, nil, nil, nil) - block3 = types.NewBlock(&header3, nil, nil, nil) - - event1 = core.ChainEvent{Block: block1} - event2 = core.ChainEvent{Block: block2} - event3 = core.ChainEvent{Block: block3} -) - -func testErrorInChainEventLoop(t *testing.T) { - //the first chain event causes and error (in blockchain mock) - extractor := mocks.Extractor{} - - blockChain := mocks.BlockChain{} - service := s.StateDiffService{ - Builder: nil, - Extractor: &extractor, - BlockChain: &blockChain, - } - - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) - blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) - service.Loop(eventsChannel) - - //parent and current blocks are passed to the extractor - expectedCurrentBlocks := []types.Block{*block1, *block2} - if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks) - } - expectedParentBlocks := []types.Block{*parentBlock1, *parentBlock2} - if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks) - } - - //look up the parent block from its hash - expectedHashes := []common.Hash{block1.ParentHash(), block2.ParentHash()} - if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) - } -} - -func testErrorInBlockLoop(t *testing.T) { - //second block's parent block can't be found - extractor := mocks.Extractor{} - - blockChain := mocks.BlockChain{} - service := s.StateDiffService{ - Builder: nil, - Extractor: &extractor, - BlockChain: &blockChain, - } - - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) - blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) - service.Loop(eventsChannel) - - //only the first current block (and it's parent) are passed to the extractor - expectedCurrentBlocks := []types.Block{*block1} - if !reflect.DeepEqual(extractor.CurrentBlocks, expectedCurrentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedCurrentBlocks) - } - expectedParentBlocks := []types.Block{*parentBlock1} - if !reflect.DeepEqual(extractor.ParentBlocks, expectedParentBlocks) { - t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", extractor.CurrentBlocks, expectedParentBlocks) - } -} diff --git a/statediff/service_test.go b/statediff/service_test.go new file mode 100644 index 000000000..d5edee04e --- /dev/null +++ b/statediff/service_test.go @@ -0,0 +1,130 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff_test + +import ( + "bytes" + "math/big" + "math/rand" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" +) + +func TestServiceLoop(t *testing.T) { + testErrorInChainEventLoop(t) + testErrorInBlockLoop(t) +} + +var ( + eventsChannel = make(chan core.ChainEvent, 1) + + parentRoot1 = common.HexToHash("0x01") + parentRoot2 = common.HexToHash("0x02") + parentHeader1 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot1} + parentHeader2 = types.Header{Number: big.NewInt(rand.Int63()), Root: parentRoot2} + + parentBlock1 = types.NewBlock(&parentHeader1, nil, nil, nil) + parentBlock2 = types.NewBlock(&parentHeader2, nil, nil, nil) + + parentHash1 = parentBlock1.Hash() + parentHash2 = parentBlock2.Hash() + + testRoot1 = common.HexToHash("0x03") + testRoot2 = common.HexToHash("0x04") + testRoot3 = common.HexToHash("0x04") + header1 = types.Header{ParentHash: parentHash1, Root: testRoot1} + header2 = types.Header{ParentHash: parentHash2, Root: testRoot2} + header3 = types.Header{ParentHash: common.HexToHash("parent hash"), Root: testRoot3} + + testBlock1 = types.NewBlock(&header1, nil, nil, nil) + testBlock2 = types.NewBlock(&header2, nil, nil, nil) + testBlock3 = types.NewBlock(&header3, nil, nil, nil) + + event1 = core.ChainEvent{Block: testBlock1} + event2 = core.ChainEvent{Block: testBlock2} + event3 = core.ChainEvent{Block: testBlock3} +) + +func testErrorInChainEventLoop(t *testing.T) { + //the first chain event causes and error (in blockchain mock) + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + service := statediff.Service{ + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[rpc.ID]statediff.Subscription), + } + testRoot2 = common.HexToHash("0xTestRoot2") + blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) + service.Loop(eventsChannel) + if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) + } + if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) + } + if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) + } + //look up the parent block from its hash + expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} + if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) + } +} + +func testErrorInBlockLoop(t *testing.T) { + //second block's parent block can't be found + builder := mocks.Builder{} + blockChain := mocks.BlockChain{} + service := statediff.Service{ + Builder: &builder, + BlockChain: &blockChain, + QuitChan: make(chan bool), + Subscriptions: make(map[rpc.ID]statediff.Subscription), + } + + blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) + blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + service.Loop(eventsChannel) + + if !bytes.Equal(builder.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock1.Hash()) + } + if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock1.Root()) + } + if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock1.Root().Bytes()) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock1.Root()) + } +} diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go index 5126c6556..8f52bc8cc 100644 --- a/statediff/testhelpers/helpers.go +++ b/statediff/testhelpers/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index cececde6f..f2c097d38 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index e9668629e..bb2e38f76 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go index 3ea18abf0..6a6018746 100644 --- a/statediff/testhelpers/mocks/publisher.go +++ b/statediff/testhelpers/mocks/publisher.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go new file mode 100644 index 000000000..edb2d453d --- /dev/null +++ b/statediff/testhelpers/mocks/service.go @@ -0,0 +1,171 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" +) + +// MockStateDiffService is a mock state diff service +type MockStateDiffService struct { + sync.Mutex + Builder statediff.Builder + ReturnProtocol []p2p.Protocol + ReturnAPIs []rpc.API + BlockChan chan *types.Block + ParentBlockChan chan *types.Block + QuitChan chan bool + Subscriptions map[rpc.ID]statediff.Subscription +} + +// Protocols mock method +func (sds *MockStateDiffService) Protocols() []p2p.Protocol { + return []p2p.Protocol{} +} + +// APIs mock method +func (sds *MockStateDiffService) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: statediff.APIName, + Version: statediff.APIVersion, + Service: statediff.NewPublicStateDiffAPI(sds), + Public: true, + }, + } +} + +// Loop mock method +func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { + //loop through chain events until no more +HandleBlockChLoop: + for { + select { + case block := <-sds.BlockChan: + currentBlock := block + parentBlock := <-sds.ParentBlockChan + parentHash := parentBlock.Hash() + if parentBlock == nil { + log.Error("Parent block is nil, skipping this block", + "parent block hash", parentHash.String(), + "current block number", currentBlock.Number()) + break HandleBlockChLoop + } + + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + if err != nil { + log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) + } + rlpBuff := new(bytes.Buffer) + currentBlock.EncodeRLP(rlpBuff) + blockRlp := rlpBuff.Bytes() + stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) + payload := statediff.Payload{ + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, + } + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + case <-sds.QuitChan: + log.Debug("Quitting the statediff block channel") + sds.close() + return + } + } +} + +// Subscribe mock method +func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool) { + log.Info("Subscribing to the statediff service") + sds.Lock() + sds.Subscriptions[id] = statediff.Subscription{ + PayloadChan: sub, + QuitChan: quitChan, + } + sds.Unlock() +} + +// Unsubscribe mock method +func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error { + log.Info("Unsubscribing from the statediff service") + sds.Lock() + _, ok := sds.Subscriptions[id] + if !ok { + return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id) + } + delete(sds.Subscriptions, id) + sds.Unlock() + return nil +} + +func (sds *MockStateDiffService) send(payload statediff.Payload) { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.PayloadChan <- payload: + log.Info("sending state diff payload to subscription %s", id) + default: + log.Info("unable to send payload to subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} + +func (sds *MockStateDiffService) close() { + sds.Lock() + for id, sub := range sds.Subscriptions { + select { + case sub.QuitChan <- true: + delete(sds.Subscriptions, id) + log.Info("closing subscription %s", id) + default: + log.Info("unable to close subscription %s; channel has no receiver", id) + } + } + sds.Unlock() +} + +// Start mock method +func (sds *MockStateDiffService) Start(server *p2p.Server) error { + log.Info("Starting statediff service") + if sds.ParentBlockChan == nil || sds.BlockChan == nil { + return errors.New("mock StateDiffingService requires preconfiguration with a MockParentBlockChan and MockBlockChan") + } + chainEventCh := make(chan core.ChainEvent, 10) + go sds.Loop(chainEventCh) + + return nil +} + +// Stop mock method +func (sds *MockStateDiffService) Stop() error { + log.Info("Stopping statediff service") + close(sds.QuitChan) + return nil +} diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go new file mode 100644 index 000000000..1a4f83cf5 --- /dev/null +++ b/statediff/testhelpers/mocks/service_test.go @@ -0,0 +1,127 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mocks + +import ( + "bytes" + "math/big" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/statediff" + "github.com/ethereum/go-ethereum/statediff/testhelpers" +) + +var block0, block1 *types.Block +var burnLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0")) +var emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) +var account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), +}) +var burnAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(0), + Balance: big.NewInt(2000000000000000000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), +}) +var bankAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: uint64(1), + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - 10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), +}) + +func TestAPI(t *testing.T) { + _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) + defer chain.Stop() + block0Hash := common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash := common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + blockChan := make(chan *types.Block) + parentBlockChain := make(chan *types.Block) + serviceQuitChan := make(chan bool) + mockService := MockStateDiffService{ + Mutex: sync.Mutex{}, + Builder: statediff.NewBuilder(testhelpers.Testdb, chain), + BlockChan: blockChan, + ParentBlockChan: parentBlockChain, + QuitChan: serviceQuitChan, + Subscriptions: make(map[rpc.ID]statediff.Subscription), + } + mockService.Start(nil) + id := rpc.NewID() + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + mockService.Subscribe(id, payloadChan, quitChan) + blockChan <- block1 + parentBlockChain <- block0 + expectedBlockRlp, _ := rlp.EncodeToBytes(block1) + expectedStateDiff := &statediff.StateDiff{ + BlockNumber: block1.Number().Int64(), + BlockHash: block1.Hash(), + CreatedAccounts: statediff.AccountDiffsMap{ + testhelpers.Account1LeafKey: { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + burnLeafKey: { + Key: burnLeafKey.Bytes(), + Value: burnAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 113, 160, 51, 128, 199, 183, 174, 129, 165, 142, 185, 141, 156, 120, 222, 74, 31, 215, 253, 149, 53, 252, 149, 62, 210, 190, 96, 45, 170, 164, 23, 103, 49, 42, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 0, 0, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: statediff.AccountDiffsMap{ + testhelpers.BankLeafKey: { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 1, 132, 5, 245, 185, 240, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + } + expectedStateDiffRlp, _ := rlp.EncodeToBytes(expectedStateDiff) + select { + case payload := <-payloadChan: + if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { + t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp) + } + if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffRlp) { + t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiffRlp, expectedStateDiffRlp) + } + case <-quitChan: + t.Errorf("channel quit before delivering payload") + } +} diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 17dac8473..1203be611 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/statediff/types.go b/statediff/types.go index 41e83dddb..816aae6d6 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify -- 2.45.2 From ba5057158d3141ff2e7df0c844097dfc25d0a867 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 13 May 2019 10:31:42 -0500 Subject: [PATCH 10/17] make proofs and paths optional + compress service loop into single for loop (may be missing something here) --- cmd/geth/main.go | 2 + cmd/geth/usage.go | 2 + cmd/utils/flags.go | 14 +- statediff/builder.go | 191 ++++++++++++-------- statediff/builder_test.go | 5 +- statediff/config.go | 23 +++ statediff/service.go | 97 ++++------ statediff/testhelpers/mocks/service.go | 13 +- statediff/testhelpers/mocks/service_test.go | 20 +- statediff/types.go | 13 ++ 10 files changed, 235 insertions(+), 145 deletions(-) create mode 100644 statediff/config.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 582630163..b88db8791 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -149,6 +149,8 @@ var ( utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, utils.StateDiffFlag, + utils.StateDiffPathsAndProofs, + utils.StateDiffLeafNodesOnly, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 77d816476..c30746b76 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -266,6 +266,8 @@ var AppHelpFlagGroups = []flagGroup{ Name: "STATE DIFF", Flags: []cli.Flag{ utils.StateDiffFlag, + utils.StateDiffPathsAndProofs, + utils.StateDiffLeafNodesOnly, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 9d9ecf732..7d72a87cf 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -763,6 +763,14 @@ var ( Name: "statediff", Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", } + StateDiffPathsAndProofs = cli.BoolFlag{ + Name: "statediff.pathsandproofs", + Usage: "Path and proof sets for the state and storage nodes are generated", + } + StateDiffLeafNodesOnly = cli.BoolFlag{ + Name: "statediff.leafs", + Usage: "Consider only leaf nodes of the storage and state tries", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1623,12 +1631,16 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st // RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { + config := statediff.Config{ + PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), + LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name), + } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum ctx.Service(ðServ) chainDb := ethServ.ChainDb() blockChain := ethServ.BlockChain() - return statediff.NewStateDiffService(chainDb, blockChain) + return statediff.NewStateDiffService(chainDb, blockChain, config) }); err != nil { Fatalf("Failed to register State Diff Service", err) } diff --git a/statediff/builder.go b/statediff/builder.go index 05e7d8572..04e00ae95 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -20,6 +20,7 @@ package statediff import ( + "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" @@ -37,13 +38,15 @@ type Builder interface { type builder struct { chainDB ethdb.Database + config Config blockChain *core.BlockChain } -// NewBuilder is used to create a builder -func NewBuilder(db ethdb.Database, blockChain *core.BlockChain) Builder { +// NewBuilder is used to create a state diff builder +func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) Builder { return &builder{ chainDB: db, + config: config, blockChain: blockChain, } } @@ -54,13 +57,11 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block stateCache := sdb.blockChain.StateCache() oldTrie, err := stateCache.OpenTrie(oldStateRoot) if err != nil { - log.Error("Error creating trie for oldStateRoot", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err) } newTrie, err := stateCache.OpenTrie(newStateRoot) if err != nil { - log.Error("Error creating trie for newStateRoot", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error creating trie for newStateRoot: %v", err) } // Find created accounts @@ -68,8 +69,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt := newTrie.NodeIterator([]byte{}) creations, err := sdb.collectDiffNodes(oldIt, newIt) if err != nil { - log.Error("Error collecting creation diff nodes", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error collecting creation diff nodes: %v", err) } // Find deleted accounts @@ -77,8 +77,7 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block newIt = newTrie.NodeIterator([]byte{}) deletions, err := sdb.collectDiffNodes(newIt, oldIt) if err != nil { - log.Error("Error collecting deletion diff nodes", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error collecting deletion diff nodes: %v", err) } // Find all the diffed keys @@ -89,18 +88,15 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block // Build and return the statediff updatedAccounts, err := sdb.buildDiffIncremental(creations, deletions, updatedKeys) if err != nil { - log.Error("Error building diff for updated accounts", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error building diff for updated accounts: %v", err) } createdAccounts, err := sdb.buildDiffEventual(creations) if err != nil { - log.Error("Error building diff for created accounts", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error building diff for created accounts: %v", err) } deletedAccounts, err := sdb.buildDiffEventual(deletions) if err != nil { - log.Error("Error building diff for deleted accounts", "error", err) - return StateDiff{}, err + return StateDiff{}, fmt.Errorf("error building diff for deleted accounts: %v", err) } return StateDiff{ @@ -116,38 +112,69 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error var diffAccounts = make(AccountsMap) it, _ := trie.NewDifferenceIterator(a, b) - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) - if it.Leaf() { - leafProof := make([][]byte, len(it.LeafProof())) - copy(leafProof, it.LeafProof()) - leafPath := make([]byte, len(it.Path())) - copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - log.Error("Error looking up account via address", "address", leafKeyHash, "error", err) - return nil, err + if sdb.config.PathsAndProofs { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) + if it.Leaf() { + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: account, + RawKey: leafKey, + RawValue: leafValue, + Proof: leafProof, + Path: leafPath, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw } - aw := accountWrapper{ - Account: account, - RawKey: leafKey, - RawValue: leafValue, - Proof: leafProof, - Path: leafPath, + cont := it.Next(true) + if !cont { + break } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: account, + RawKey: leafKey, + RawValue: leafValue, + Proof: nil, + Path: nil, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw + } + cont := it.Next(true) + if !cont { + break + } } } @@ -159,8 +186,7 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er for _, val := range accounts { storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root) if err != nil { - log.Error("Failed building eventual storage diffs", "Address", common.BytesToHash(val.RawKey), "error", err) - return nil, err + return nil, fmt.Errorf("failed building eventual storage diffs for address: %s, error: %v", common.BytesToHash(val.RawKey), err) } accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ Key: val.RawKey, @@ -209,7 +235,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, er return nil, err } it := sTrie.NodeIterator(make([]byte, 0)) - storageDiffs := buildStorageDiffsFromTrie(it) + storageDiffs := sdb.buildStorageDiffsFromTrie(it) return storageDiffs, nil } @@ -229,35 +255,58 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common oldIt := oldTrie.NodeIterator(make([]byte, 0)) newIt := newTrie.NodeIterator(make([]byte, 0)) it, _ := trie.NewDifferenceIterator(oldIt, newIt) - storageDiffs := buildStorageDiffsFromTrie(it) + storageDiffs := sdb.buildStorageDiffsFromTrie(it) return storageDiffs, nil } -func buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { +func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { storageDiffs := make([]StorageDiff, 0) - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafProof := make([][]byte, len(it.LeafProof())) - copy(leafProof, it.LeafProof()) - leafPath := make([]byte, len(it.Path())) - copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: leafPath, - Proof: leafProof, - }) + if sdb.config.PathsAndProofs { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafProof := make([][]byte, len(it.LeafProof())) + copy(leafProof, it.LeafProof()) + leafPath := make([]byte, len(it.Path())) + copy(leafPath, it.Path()) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: leafPath, + Proof: leafProof, + }) + } + cont := it.Next(true) + if !cont { + break + } } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: nil, + Proof: nil, + }) + } + cont := it.Next(true) + if !cont { + break + } } } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 5ecf647e3..49bd39043 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -56,7 +56,10 @@ func TestBuilder(t *testing.T) { block1 = blockMap[block1Hash] block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] - builder = statediff.NewBuilder(testhelpers.Testdb, chain) + config := statediff.Config{ + PathsAndProofs: true, + } + builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) type arguments struct { oldStateRoot common.Hash diff --git a/statediff/config.go b/statediff/config.go new file mode 100644 index 000000000..a18f2c75a --- /dev/null +++ b/statediff/config.go @@ -0,0 +1,23 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package statediff + +// Config is used to carry in parameters from CLI configuration +type Config struct { + PathsAndProofs bool + LeafsOnly bool +} diff --git a/statediff/service.go b/statediff/service.go index d03a7b513..2b6b167f9 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -18,18 +18,17 @@ package statediff import ( "bytes" + "encoding/json" "fmt" "sync" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" ) @@ -66,25 +65,12 @@ type Service struct { Subscriptions map[rpc.ID]Subscription } -// Subscription struct holds our subscription channels -type Subscription struct { - PayloadChan chan<- Payload - QuitChan chan<- bool -} - -// Payload packages the data to send to StateDiffingService subscriptions -type Payload struct { - BlockRlp []byte `json:"blockRlp" gencodec:"required"` - StateDiffRlp []byte `json:"stateDiffRlp" gencodec:"required"` - Err error `json:"error"` -} - // NewStateDiffService creates a new StateDiffingService -func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain) (*Service, error) { +func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Config) (*Service, error) { return &Service{ Mutex: sync.Mutex{}, BlockChain: blockChain, - Builder: NewBuilder(db, blockChain), + Builder: NewBuilder(db, blockChain, config), QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]Subscription), }, nil @@ -109,70 +95,61 @@ func (sds *Service) APIs() []rpc.API { // Loop is the main processing method func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { - chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) defer chainEventSub.Unsubscribe() - - blocksCh := make(chan *types.Block, 10) errCh := chainEventSub.Err() - go func() { - HandleChainEventChLoop: - for { - select { - //Notify chain event channel of events - case chainEvent := <-chainEventCh: - log.Debug("Event received from chainEventCh", "event", chainEvent) - blocksCh <- chainEvent.Block - //if node stopped - case err := <-errCh: - log.Warn("Error from chain event subscription, breaking loop.", "error", err) - close(sds.QuitChan) - break HandleChainEventChLoop - case <-sds.QuitChan: - break HandleChainEventChLoop - } - } - }() - - //loop through chain events until no more -HandleBlockChLoop: for { select { - case block := <-blocksCh: - currentBlock := block + //Notify chain event channel of events + case chainEvent := <-chainEventCh: + log.Debug("Event received from chainEventCh", "event", chainEvent) + currentBlock := chainEvent.Block parentHash := currentBlock.ParentHash() parentBlock := sds.BlockChain.GetBlockByHash(parentHash) if parentBlock == nil { log.Error("Parent block is nil, skipping this block", "parent block hash", parentHash.String(), "current block number", currentBlock.Number()) - break HandleBlockChLoop + continue } - - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) - if err != nil { + if err := sds.process(currentBlock, parentBlock); err != nil { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } - rlpBuff := new(bytes.Buffer) - currentBlock.EncodeRLP(rlpBuff) - blockRlp := rlpBuff.Bytes() - stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) - payload := Payload{ - BlockRlp: blockRlp, - StateDiffRlp: stateDiffRlp, - Err: err, - } - // If we have any websocket subscription listening in, send the data to them - sds.send(payload) + case err := <-errCh: + log.Warn("Error from chain event subscription, breaking loop.", "error", err) + sds.close() + return case <-sds.QuitChan: - log.Debug("Quitting the statediff block channel") + log.Info("Quitting the statediff block channel") sds.close() return } } } +// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions +func (sds *Service) process(currentBlock, parentBlock *types.Block) error { + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + if err != nil { + return err + } + + rlpBuff := new(bytes.Buffer) + currentBlock.EncodeRLP(rlpBuff) + blockRlp := rlpBuff.Bytes() + stateDiffBytes, _ := json.Marshal(stateDiff) + payload := Payload{ + BlockRlp: blockRlp, + StateDiff: stateDiffBytes, + Err: err, + } + + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + return nil +} + // Subscribe is used by the API to subscribe to the StateDiffingService loop func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index edb2d453d..65ab0f8c8 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -18,6 +18,7 @@ package mocks import ( "bytes" + "encoding/json" "errors" "fmt" "sync" @@ -26,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" ) @@ -63,7 +63,6 @@ func (sds *MockStateDiffService) APIs() []rpc.API { // Loop mock method func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { //loop through chain events until no more -HandleBlockChLoop: for { select { case block := <-sds.BlockChan: @@ -74,7 +73,7 @@ HandleBlockChLoop: log.Error("Parent block is nil, skipping this block", "parent block hash", parentHash.String(), "current block number", currentBlock.Number()) - break HandleBlockChLoop + continue } stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) @@ -84,11 +83,11 @@ HandleBlockChLoop: rlpBuff := new(bytes.Buffer) currentBlock.EncodeRLP(rlpBuff) blockRlp := rlpBuff.Bytes() - stateDiffRlp, _ := rlp.EncodeToBytes(stateDiff) + stateDiffBytes, _ := json.Marshal(stateDiff) payload := statediff.Payload{ - BlockRlp: blockRlp, - StateDiffRlp: stateDiffRlp, - Err: err, + BlockRlp: blockRlp, + StateDiff: stateDiffBytes, + Err: err, } // If we have any websocket subscription listening in, send the data to them sds.send(payload) diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index 1a4f83cf5..99420460a 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -18,6 +18,7 @@ package mocks import ( "bytes" + "encoding/json" "math/big" "sync" "testing" @@ -63,9 +64,12 @@ func TestAPI(t *testing.T) { blockChan := make(chan *types.Block) parentBlockChain := make(chan *types.Block) serviceQuitChan := make(chan bool) + config := statediff.Config{ + PathsAndProofs: true, + } mockService := MockStateDiffService{ Mutex: sync.Mutex{}, - Builder: statediff.NewBuilder(testhelpers.Testdb, chain), + Builder: statediff.NewBuilder(testhelpers.Testdb, chain, config), BlockChan: blockChan, ParentBlockChan: parentBlockChain, QuitChan: serviceQuitChan, @@ -79,7 +83,7 @@ func TestAPI(t *testing.T) { blockChan <- block1 parentBlockChain <- block0 expectedBlockRlp, _ := rlp.EncodeToBytes(block1) - expectedStateDiff := &statediff.StateDiff{ + expectedStateDiff := statediff.StateDiff{ BlockNumber: block1.Number().Int64(), BlockHash: block1.Hash(), CreatedAccounts: statediff.AccountDiffsMap{ @@ -112,14 +116,20 @@ func TestAPI(t *testing.T) { }, }, } - expectedStateDiffRlp, _ := rlp.EncodeToBytes(expectedStateDiff) + expectedStateDiffBytes, err := json.Marshal(expectedStateDiff) + if err != nil { + t.Error(err) + } select { case payload := <-payloadChan: if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp) } - if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffRlp) { - t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiffRlp, expectedStateDiffRlp) + if !bytes.Equal(payload.StateDiff, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiff, expectedStateDiffBytes) + } + if payload.Err != nil { + t.Errorf("payload should not contain an error, but does: %v", payload.Err) } case <-quitChan: t.Errorf("channel quit before delivering payload") diff --git a/statediff/types.go b/statediff/types.go index 816aae6d6..70f9d2e44 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -26,6 +26,19 @@ import ( "github.com/ethereum/go-ethereum/core/state" ) +// Subscription struct holds our subscription channels +type Subscription struct { + PayloadChan chan<- Payload + QuitChan chan<- bool +} + +// Payload packages the data to send to StateDiffingService subscriptions +type Payload struct { + BlockRlp []byte `json:"blockRlp" gencodec:"required"` + StateDiff []byte `json:"stateDiff" gencodec:"required"` + Err error `json:"error"` +} + // AccountsMap is a mapping of keccak256(address) => accountWrapper type AccountsMap map[common.Hash]accountWrapper -- 2.45.2 From 6e6b421c48d2aeaa25433bb1fa8e9d55ddf42548 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 15 May 2019 13:06:57 -0500 Subject: [PATCH 11/17] option to process intermediate nodes --- cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 +- statediff/builder.go | 221 +++++++++++++++++++++++++++----------- statediff/builder_test.go | 1 + statediff/config.go | 1 + statediff/types.go | 2 +- 6 files changed, 166 insertions(+), 67 deletions(-) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index c30746b76..e92964aec 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -268,6 +268,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, utils.StateDiffLeafNodesOnly, + utils.StateDiffWatchedAddresses, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7d72a87cf..13d7ff7dd 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -765,12 +765,16 @@ var ( } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", - Usage: "Path and proof sets for the state and storage nodes are generated", + Usage: "Path and proof sets for the state and storage nodes are generated; only works with leaf nodes", } StateDiffLeafNodesOnly = cli.BoolFlag{ Name: "statediff.leafs", Usage: "Consider only leaf nodes of the storage and state tries", } + StateDiffWatchedAddresses = cli.StringSliceFlag{ + Name: "statediff.watchedaddresses", + Usage: "If provided, state diffing process is restricted to these addresses", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1634,6 +1638,7 @@ func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name), + WatchedAddress: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum diff --git a/statediff/builder.go b/statediff/builder.go index 04e00ae95..8c71322cf 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -21,6 +21,7 @@ package statediff import ( "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" @@ -40,6 +41,7 @@ type builder struct { chainDB ethdb.Database config Config blockChain *core.BlockChain + stateCache state.Database } // NewBuilder is used to create a state diff builder @@ -54,12 +56,12 @@ func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) B // BuildStateDiff builds a StateDiff object from two blocks func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states - stateCache := sdb.blockChain.StateCache() - oldTrie, err := stateCache.OpenTrie(oldStateRoot) + sdb.stateCache = sdb.blockChain.StateCache() + oldTrie, err := sdb.stateCache.OpenTrie(oldStateRoot) if err != nil { return StateDiff{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err) } - newTrie, err := stateCache.OpenTrie(newStateRoot) + newTrie, err := sdb.stateCache.OpenTrie(newStateRoot) if err != nil { return StateDiff{}, fmt.Errorf("error creating trie for newStateRoot: %v", err) } @@ -114,7 +116,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error if sdb.config.PathsAndProofs { for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "hashold", it.Hash()) + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) if it.Leaf() { leafProof := make([][]byte, len(it.LeafProof())) copy(leafProof, it.LeafProof()) @@ -128,10 +130,10 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error // lookup account state var account state.Account if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) } aw := accountWrapper{ - Account: account, + Account: &account, RawKey: leafKey, RawValue: leafValue, Proof: leafProof, @@ -147,33 +149,80 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error } } } else { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address: %s, error: %v", leafKeyHash.Hex(), err) + if sdb.config.LeafsOnly { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: &account, + RawKey: leafKey, + RawValue: leafValue, + Proof: nil, + Path: nil, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw } - aw := accountWrapper{ - Account: account, - RawKey: leafKey, - RawValue: leafValue, - Proof: nil, - Path: nil, + cont := it.Next(true) + if !cont { + break } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Account: &account, + RawKey: leafKey, + RawValue: leafValue, + Proof: nil, + Path: nil, + } + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw + } else { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err) + } + aw := accountWrapper{ + Account: nil, + RawKey: nodeKey.Bytes(), + RawValue: node, + Proof: nil, + Path: nil, + } + log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node) + diffAccounts[nodeKey] = aw + } + cont := it.Next(true) + if !cont { + break + } } } } @@ -183,10 +232,15 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { accountDiffs := make(AccountDiffsMap) + var err error for _, val := range accounts { - storageDiffs, err := sdb.buildStorageDiffsEventual(val.Account.Root) - if err != nil { - return nil, fmt.Errorf("failed building eventual storage diffs for address: %s, error: %v", common.BytesToHash(val.RawKey), err) + // If account is not nil, we need to process storage diffs + var storageDiffs []StorageDiff + if val.Account != nil { + storageDiffs, err = sdb.buildStorageDiffsEventual(val.Account.Root) + if err != nil { + return nil, fmt.Errorf("failed building eventual storage diffs for %s\r\nerror: %v", common.BytesToHash(val.RawKey), err) + } } accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ Key: val.RawKey, @@ -202,15 +256,19 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) { updatedAccounts := make(AccountDiffsMap) + var err error for _, val := range updatedKeys { - createdAcc := creations[common.HexToHash(val)] - deletedAcc := deletions[common.HexToHash(val)] - oldSR := deletedAcc.Account.Root - newSR := createdAcc.Account.Root - storageDiffs, err := sdb.buildStorageDiffsIncremental(oldSR, newSR) - if err != nil { - log.Error("Failed building storage diffs", "Address", val, "error", err) - return nil, err + hashKey := common.HexToHash(val) + createdAcc := creations[hashKey] + deletedAcc := deletions[hashKey] + var storageDiffs []StorageDiff + if deletedAcc.Account != nil && createdAcc.Account != nil { + oldSR := deletedAcc.Account.Root + newSR := createdAcc.Account.Root + storageDiffs, err = sdb.buildStorageDiffsIncremental(oldSR, newSR) + if err != nil { + return nil, fmt.Errorf("failed building incremental storage diffs for %s\r\nerror: %v", hashKey.Hex(), err) + } } updatedAccounts[common.HexToHash(val)] = AccountDiff{ Key: createdAcc.RawKey, @@ -235,8 +293,7 @@ func (sdb *builder) buildStorageDiffsEventual(sr common.Hash) ([]StorageDiff, er return nil, err } it := sTrie.NodeIterator(make([]byte, 0)) - storageDiffs := sdb.buildStorageDiffsFromTrie(it) - return storageDiffs, nil + return sdb.buildStorageDiffsFromTrie(it) } func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common.Hash) ([]StorageDiff, error) { @@ -255,12 +312,10 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common oldIt := oldTrie.NodeIterator(make([]byte, 0)) newIt := newTrie.NodeIterator(make([]byte, 0)) it, _ := trie.NewDifferenceIterator(oldIt, newIt) - storageDiffs := sdb.buildStorageDiffsFromTrie(it) - - return storageDiffs, nil + return sdb.buildStorageDiffsFromTrie(it) } -func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDiff { +func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDiff, error) { storageDiffs := make([]StorageDiff, 0) if sdb.config.PathsAndProofs { for { @@ -288,29 +343,65 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) []StorageDif } } } else { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: nil, - Proof: nil, - }) + if sdb.config.LeafsOnly { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: nil, + Proof: nil, + }) + } + cont := it.Next(true) + if !cont { + break + } } - cont := it.Next(true) - if !cont { - break + } else { + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + storageDiffs = append(storageDiffs, StorageDiff{ + Key: leafKey, + Value: leafValue, + Path: nil, + Proof: nil, + }) + } else { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err) + } + storageDiffs = append(storageDiffs, StorageDiff{ + Key: nodeKey.Bytes(), + Value: node, + Path: nil, + Proof: nil, + }) + log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node) + } + cont := it.Next(true) + if !cont { + break + } } } } - return storageDiffs + return storageDiffs, nil } func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 49bd39043..b7e3138d1 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -58,6 +58,7 @@ func TestBuilder(t *testing.T) { block3 = blockMap[block3Hash] config := statediff.Config{ PathsAndProofs: true, + LeafsOnly: true, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) diff --git a/statediff/config.go b/statediff/config.go index a18f2c75a..4a568982f 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -20,4 +20,5 @@ package statediff type Config struct { PathsAndProofs bool LeafsOnly bool + WatchedAddress []string } diff --git a/statediff/types.go b/statediff/types.go index 70f9d2e44..aa9460fde 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -44,7 +44,7 @@ type AccountsMap map[common.Hash]accountWrapper // AccountWrapper is used to temporary associate the unpacked account with its raw values type accountWrapper struct { - Account state.Account + Account *state.Account RawKey []byte RawValue []byte Proof [][]byte -- 2.45.2 From f67a060c8c18e4bf824062db36a9af9d9c355425 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 16 May 2019 12:03:46 -0500 Subject: [PATCH 12/17] make state diff rlp serializable --- statediff/builder.go | 21 +-- statediff/builder_test.go | 155 +++++++++----------- statediff/service.go | 26 +++- statediff/testhelpers/mocks/builder.go | 6 +- statediff/testhelpers/mocks/service.go | 16 +- statediff/testhelpers/mocks/service_test.go | 41 +++--- statediff/testhelpers/test_data.go | 12 +- statediff/types.go | 50 ++----- 8 files changed, 155 insertions(+), 172 deletions(-) diff --git a/statediff/builder.go b/statediff/builder.go index 8c71322cf..26c4faaac 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -21,6 +21,7 @@ package statediff import ( "fmt" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -34,7 +35,7 @@ import ( // Builder interface exposes the method for building a state diff between two blocks type Builder interface { - BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) + BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) } type builder struct { @@ -54,7 +55,7 @@ func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) B } // BuildStateDiff builds a StateDiff object from two blocks -func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (StateDiff, error) { +func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states sdb.stateCache = sdb.blockChain.StateCache() oldTrie, err := sdb.stateCache.OpenTrie(oldStateRoot) @@ -230,8 +231,8 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error return diffAccounts, nil } -func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, error) { - accountDiffs := make(AccountDiffsMap) +func (sdb *builder) buildDiffEventual(accounts AccountsMap) ([]AccountDiff, error) { + accountDiffs := make([]AccountDiff, 0) var err error for _, val := range accounts { // If account is not nil, we need to process storage diffs @@ -242,20 +243,20 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) (AccountDiffsMap, er return nil, fmt.Errorf("failed building eventual storage diffs for %s\r\nerror: %v", common.BytesToHash(val.RawKey), err) } } - accountDiffs[common.BytesToHash(val.RawKey)] = AccountDiff{ + accountDiffs = append(accountDiffs, AccountDiff{ Key: val.RawKey, Value: val.RawValue, Proof: val.Proof, Path: val.Path, Storage: storageDiffs, - } + }) } return accountDiffs, nil } -func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) (AccountDiffsMap, error) { - updatedAccounts := make(AccountDiffsMap) +func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions AccountsMap, updatedKeys []string) ([]AccountDiff, error) { + updatedAccounts := make([]AccountDiff, 0) var err error for _, val := range updatedKeys { hashKey := common.HexToHash(val) @@ -270,13 +271,13 @@ func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions Accoun return nil, fmt.Errorf("failed building incremental storage diffs for %s\r\nerror: %v", hashKey.Hex(), err) } } - updatedAccounts[common.HexToHash(val)] = AccountDiff{ + updatedAccounts = append(updatedAccounts, AccountDiff{ Key: createdAcc.RawKey, Value: createdAcc.RawValue, Proof: createdAcc.Proof, Path: createdAcc.Path, Storage: storageDiffs, - } + }) delete(creations, common.HexToHash(val)) delete(deletions, common.HexToHash(val)) } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index b7e3138d1..372241d10 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -19,7 +19,7 @@ package statediff_test import ( "bytes" "math/big" - "reflect" + "sort" "testing" "github.com/ethereum/go-ethereum/common" @@ -33,8 +33,8 @@ import ( var ( contractLeafKey common.Hash - emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) - emptyAccountDiffIncrementalMap = make(statediff.AccountDiffsMap) + emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) + emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0) block0Hash, block1Hash, block2Hash, block3Hash common.Hash block0, block1, block2, block3 *types.Block builder statediff.Builder @@ -65,7 +65,7 @@ func TestBuilder(t *testing.T) { type arguments struct { oldStateRoot common.Hash newStateRoot common.Hash - blockNumber int64 + blockNumber *big.Int blockHash common.Hash } @@ -166,11 +166,11 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block0.Root(), newStateRoot: block0.Root(), - blockNumber: block0.Number().Int64(), + blockNumber: block0.Number(), blockHash: block0Hash, }, &statediff.StateDiff{ - BlockNumber: block0.Number().Int64(), + BlockNumber: block0.Number(), BlockHash: block0Hash, CreatedAccounts: emptyAccountDiffEventualMap, DeletedAccounts: emptyAccountDiffEventualMap, @@ -183,22 +183,14 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block0.Root(), newStateRoot: block1.Root(), - blockNumber: block1.Number().Int64(), + blockNumber: block1.Number(), blockHash: block1Hash, }, &statediff.StateDiff{ - BlockNumber: block1.Number().Int64(), + BlockNumber: block1.Number(), BlockHash: block1.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account1LeafKey: { - Key: testhelpers.Account1LeafKey.Bytes(), - Value: account1, - Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, - {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, - Storage: []statediff.StorageDiff{}, - }, - burnLeafKey: { + CreatedAccounts: []statediff.AccountDiff{ + { Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -206,10 +198,18 @@ func TestBuilder(t *testing.T) { Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, + { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.BankLeafKey: { + UpdatedAccounts: []statediff.AccountDiff{ + { Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -227,22 +227,14 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block1.Root(), newStateRoot: block2.Root(), - blockNumber: block2.Number().Int64(), + blockNumber: block2.Number(), blockHash: block2Hash, }, &statediff.StateDiff{ - BlockNumber: block2.Number().Int64(), + BlockNumber: block2.Number(), BlockHash: block2.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account2LeafKey: { - Key: testhelpers.Account2LeafKey.Bytes(), - Value: account2, - Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, - Storage: []statediff.StorageDiff{}, - }, - contractLeafKey: { + CreatedAccounts: []statediff.AccountDiff{ + { Key: contractLeafKey.Bytes(), Value: contractAccount, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -257,10 +249,18 @@ func TestBuilder(t *testing.T) { }, }, }, + { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account2, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 72, 248, 70, 128, 130, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + Storage: []statediff.StorageDiff{}, + }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.BankLeafKey: { + UpdatedAccounts: []statediff.AccountDiff{ + { Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -268,15 +268,7 @@ func TestBuilder(t *testing.T) { Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, - testhelpers.Account1LeafKey: { - Key: testhelpers.Account1LeafKey.Bytes(), - Value: account3, - Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, - Storage: []statediff.StorageDiff{}, - }, - burnLeafKey: { + { Key: burnLeafKey.Bytes(), Value: burnAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -284,6 +276,14 @@ func TestBuilder(t *testing.T) { Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, + { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account3, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, }, }, }, @@ -294,24 +294,24 @@ func TestBuilder(t *testing.T) { arguments{ oldStateRoot: block2.Root(), newStateRoot: block3.Root(), - blockNumber: block3.Number().Int64(), + blockNumber: block3.Number(), blockHash: block3.Hash(), }, &statediff.StateDiff{ - BlockNumber: block3.Number().Int64(), + BlockNumber: block3.Number(), BlockHash: block3.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{}, + CreatedAccounts: []statediff.AccountDiff{}, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account2LeafKey: { - Key: testhelpers.Account2LeafKey.Bytes(), - Value: account4, + UpdatedAccounts: []statediff.AccountDiff{ + { + Key: testhelpers.BankLeafKey.Bytes(), + Value: bankAccount3, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, + {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, - contractLeafKey: { + { Key: contractLeafKey.Bytes(), Value: contractAccount2, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -327,12 +327,12 @@ func TestBuilder(t *testing.T) { }, }, }, - testhelpers.BankLeafKey: { - Key: testhelpers.BankLeafKey.Bytes(), - Value: bankAccount3, + { + Key: testhelpers.Account2LeafKey.Bytes(), + Value: account4, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, - {248, 109, 160, 48, 191, 73, 244, 64, 161, 205, 5, 39, 228, 208, 110, 39, 101, 101, 76, 15, 86, 69, 34, 87, 81, 109, 121, 58, 155, 141, 96, 77, 207, 223, 42, 184, 74, 248, 72, 3, 132, 5, 245, 182, 8, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{0, 0, 11, 15, 4, 9, 15, 4, 4, 0, 10, 1, 12, 13, 0, 5, 2, 7, 14, 4, 13, 0, 6, 14, 2, 7, 6, 5, 6, 5, 4, 12, 0, 15, 5, 6, 4, 5, 2, 2, 5, 7, 5, 1, 6, 13, 7, 9, 3, 10, 9, 11, 8, 13, 6, 0, 4, 13, 12, 15, 13, 15, 2, 10, 16}, + {248, 113, 160, 57, 87, 243, 226, 240, 74, 7, 100, 195, 160, 73, 27, 23, 95, 105, 146, 109, 166, 30, 251, 204, 143, 97, 250, 20, 85, 253, 45, 43, 76, 221, 69, 184, 78, 248, 76, 128, 136, 27, 193, 109, 103, 78, 200, 3, 232, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{12, 9, 5, 7, 15, 3, 14, 2, 15, 0, 4, 10, 0, 7, 6, 4, 12, 3, 10, 0, 4, 9, 1, 11, 1, 7, 5, 15, 6, 9, 9, 2, 6, 13, 10, 6, 1, 14, 15, 11, 12, 12, 8, 15, 6, 1, 15, 10, 1, 4, 5, 5, 15, 13, 2, 13, 2, 11, 4, 12, 13, 13, 4, 5, 16}, Storage: []statediff.StorageDiff{}, }, }, @@ -346,36 +346,23 @@ func TestBuilder(t *testing.T) { if err != nil { t.Error(err) } - fields := []string{"BlockNumber", "BlockHash", "DeletedAccounts", "UpdatedAccounts", "CreatedAccounts"} - - for _, field := range fields { - reflectionOfDiff := reflect.ValueOf(diff) - diffValue := reflect.Indirect(reflectionOfDiff).FieldByName(field) - - reflectionOfExpected := reflect.ValueOf(test.expected) - expectedValue := reflect.Indirect(reflectionOfExpected).FieldByName(field) - - diffValueInterface := diffValue.Interface() - expectedValueInterface := expectedValue.Interface() - - if !equals(diffValueInterface, expectedValueInterface) { - t.Logf("Test failed: %s", test.name) - t.Errorf("field: %+v\nactual: %+v\nexpected: %+v", field, diffValueInterface, expectedValueInterface) - } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff rlp: %+v\nexpected state diff rlp: %+v", receivedStateDiffRlp, expectedStateDiffRlp) } } } -func equals(actual, expected interface{}) (success bool) { - if actualByteSlice, ok := actual.([]byte); ok { - if expectedByteSlice, ok := expected.([]byte); ok { - return bytes.Equal(actualByteSlice, expectedByteSlice) - } - } - - return reflect.DeepEqual(actual, expected) -} - /* contract test { diff --git a/statediff/service.go b/statediff/service.go index 2b6b167f9..897032781 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -18,10 +18,11 @@ package statediff import ( "bytes" - "encoding/json" "fmt" "sync" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -63,6 +64,8 @@ type Service struct { QuitChan chan bool // A mapping of rpc.IDs to their subscription channels Subscriptions map[rpc.ID]Subscription + // Cache the last block so that we can avoid having to lookup the next block's parent + lastBlock *types.Block } // NewStateDiffService creates a new StateDiffingService @@ -106,7 +109,13 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { log.Debug("Event received from chainEventCh", "event", chainEvent) currentBlock := chainEvent.Block parentHash := currentBlock.ParentHash() - parentBlock := sds.BlockChain.GetBlockByHash(parentHash) + var parentBlock *types.Block + if sds.lastBlock != nil && bytes.Equal(sds.lastBlock.Hash().Bytes(), currentBlock.ParentHash().Bytes()) { + parentBlock = sds.lastBlock + } else { + parentBlock = sds.BlockChain.GetBlockByHash(parentHash) + } + sds.lastBlock = currentBlock if parentBlock == nil { log.Error("Parent block is nil, skipping this block", "parent block hash", parentHash.String(), @@ -130,7 +139,7 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { // process method builds the state diff payload from the current and parent block and streams it to listening subscriptions func (sds *Service) process(currentBlock, parentBlock *types.Block) error { - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { return err } @@ -138,11 +147,14 @@ func (sds *Service) process(currentBlock, parentBlock *types.Block) error { rlpBuff := new(bytes.Buffer) currentBlock.EncodeRLP(rlpBuff) blockRlp := rlpBuff.Bytes() - stateDiffBytes, _ := json.Marshal(stateDiff) + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + return err + } payload := Payload{ - BlockRlp: blockRlp, - StateDiff: stateDiffBytes, - Err: err, + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, } // If we have any websocket subscription listening in, send the data to them diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index bb2e38f76..034af0415 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -17,6 +17,8 @@ package mocks import ( + "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff" ) @@ -25,14 +27,14 @@ import ( type Builder struct { OldStateRoot common.Hash NewStateRoot common.Hash - BlockNumber int64 + BlockNumber *big.Int BlockHash common.Hash stateDiff statediff.StateDiff builderError error } // BuildStateDiff mock method -func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber int64, blockHash common.Hash) (statediff.StateDiff, error) { +func (builder *Builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (statediff.StateDiff, error) { builder.OldStateRoot = oldStateRoot builder.NewStateRoot = newStateRoot builder.BlockNumber = blockNumber diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index 65ab0f8c8..aaa0d0202 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -18,11 +18,12 @@ package mocks import ( "bytes" - "encoding/json" "errors" "fmt" "sync" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -76,18 +77,21 @@ func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { continue } - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number().Int64(), currentBlock.Hash()) + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } rlpBuff := new(bytes.Buffer) currentBlock.EncodeRLP(rlpBuff) blockRlp := rlpBuff.Bytes() - stateDiffBytes, _ := json.Marshal(stateDiff) + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + log.Error("Error encoding statediff", "block number", currentBlock.Number(), "error", err) + } payload := statediff.Payload{ - BlockRlp: blockRlp, - StateDiff: stateDiffBytes, - Err: err, + BlockRlp: blockRlp, + StateDiffRlp: stateDiffRlp, + Err: err, } // If we have any websocket subscription listening in, send the data to them sds.send(payload) diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index 99420460a..97e4893c6 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -18,8 +18,8 @@ package mocks import ( "bytes" - "encoding/json" "math/big" + "sort" "sync" "testing" @@ -34,7 +34,7 @@ import ( var block0, block1 *types.Block var burnLeafKey = testhelpers.AddressToLeafKey(common.HexToAddress("0x0")) -var emptyAccountDiffEventualMap = make(statediff.AccountDiffsMap) +var emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) var account1, _ = rlp.EncodeToBytes(state.Account{ Nonce: uint64(0), Balance: big.NewInt(10000), @@ -84,18 +84,10 @@ func TestAPI(t *testing.T) { parentBlockChain <- block0 expectedBlockRlp, _ := rlp.EncodeToBytes(block1) expectedStateDiff := statediff.StateDiff{ - BlockNumber: block1.Number().Int64(), + BlockNumber: block1.Number(), BlockHash: block1.Hash(), - CreatedAccounts: statediff.AccountDiffsMap{ - testhelpers.Account1LeafKey: { - Key: testhelpers.Account1LeafKey.Bytes(), - Value: account1, - Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, - {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, - Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, - Storage: []statediff.StorageDiff{}, - }, - burnLeafKey: { + CreatedAccounts: []statediff.AccountDiff{ + { Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -103,10 +95,18 @@ func TestAPI(t *testing.T) { Path: []byte{5, 3, 8, 0, 12, 7, 11, 7, 10, 14, 8, 1, 10, 5, 8, 14, 11, 9, 8, 13, 9, 12, 7, 8, 13, 14, 4, 10, 1, 15, 13, 7, 15, 13, 9, 5, 3, 5, 15, 12, 9, 5, 3, 14, 13, 2, 11, 14, 6, 0, 2, 13, 10, 10, 10, 4, 1, 7, 6, 7, 3, 1, 2, 10, 16}, Storage: []statediff.StorageDiff{}, }, + { + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, }, DeletedAccounts: emptyAccountDiffEventualMap, - UpdatedAccounts: statediff.AccountDiffsMap{ - testhelpers.BankLeafKey: { + UpdatedAccounts: []statediff.AccountDiff{ + { Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -116,17 +116,20 @@ func TestAPI(t *testing.T) { }, }, } - expectedStateDiffBytes, err := json.Marshal(expectedStateDiff) + expectedStateDiffBytes, err := rlp.EncodeToBytes(expectedStateDiff) if err != nil { t.Error(err) } + sort.Slice(expectedStateDiffBytes, func(i, j int) bool { return expectedStateDiffBytes[i] < expectedStateDiffBytes[j] }) + select { case payload := <-payloadChan: if !bytes.Equal(payload.BlockRlp, expectedBlockRlp) { - t.Errorf("payload does not have expected block\r\actual: %v\r\nexpected: %v", payload.BlockRlp, expectedBlockRlp) + t.Errorf("payload does not have expected block\r\actual block rlp: %v\r\nexpected block rlp: %v", payload.BlockRlp, expectedBlockRlp) } - if !bytes.Equal(payload.StateDiff, expectedStateDiffBytes) { - t.Errorf("payload does not have expected state diff\r\actual: %v\r\nexpected: %v", payload.StateDiff, expectedStateDiffBytes) + sort.Slice(payload.StateDiffRlp, func(i, j int) bool { return payload.StateDiffRlp[i] < payload.StateDiffRlp[j] }) + if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffBytes) { + t.Errorf("payload does not have expected state diff\r\actual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateDiffRlp, expectedStateDiffBytes) } if payload.Err != nil { t.Errorf("payload should not contain an error, but does: %v", payload.Err) diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index 1203be611..2f6088f86 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -36,7 +36,7 @@ func AddressToLeafKey(address common.Address) common.Hash { // Test variables var ( - BlockNumber = rand.Int63() + BlockNumber = big.NewInt(rand.Int63()) BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") NewNonceValue = rand.Uint64() @@ -63,26 +63,26 @@ var ( CodeHash: CodeHash, } valueBytes, _ = rlp.EncodeToBytes(testAccount) - CreatedAccountDiffs = statediff.AccountDiffsMap{ - ContractLeafKey: { + CreatedAccountDiffs = []statediff.AccountDiff{ + { Key: ContractLeafKey.Bytes(), Value: valueBytes, Storage: storage, }, - AnotherContractLeafKey: { + { Key: AnotherContractLeafKey.Bytes(), Value: valueBytes, Storage: emptyStorage, }, } - UpdatedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + UpdatedAccountDiffs = []statediff.AccountDiff{{ Key: ContractLeafKey.Bytes(), Value: valueBytes, Storage: storage, }} - DeletedAccountDiffs = statediff.AccountDiffsMap{ContractLeafKey: { + DeletedAccountDiffs = []statediff.AccountDiff{{ Key: ContractLeafKey.Bytes(), Value: valueBytes, Storage: storage, diff --git a/statediff/types.go b/statediff/types.go index aa9460fde..3bcbed333 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -21,6 +21,7 @@ package statediff import ( "encoding/json" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" @@ -34,9 +35,9 @@ type Subscription struct { // Payload packages the data to send to StateDiffingService subscriptions type Payload struct { - BlockRlp []byte `json:"blockRlp" gencodec:"required"` - StateDiff []byte `json:"stateDiff" gencodec:"required"` - Err error `json:"error"` + BlockRlp []byte `json:"blockRlp" gencodec:"required"` + StateDiffRlp []byte `json:"stateDiff" gencodec:"required"` + Err error `json:"error"` } // AccountsMap is a mapping of keccak256(address) => accountWrapper @@ -53,11 +54,11 @@ type accountWrapper struct { // StateDiff is the final output structure from the builder type StateDiff struct { - BlockNumber int64 `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - CreatedAccounts AccountDiffsMap `json:"createdAccounts" gencodec:"required"` - DeletedAccounts AccountDiffsMap `json:"deletedAccounts" gencodec:"required"` - UpdatedAccounts AccountDiffsMap `json:"updatedAccounts" gencodec:"required"` + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + CreatedAccounts []AccountDiff `json:"createdAccounts" gencodec:"required"` + DeletedAccounts []AccountDiff `json:"deletedAccounts" gencodec:"required"` + UpdatedAccounts []AccountDiff `json:"updatedAccounts" gencodec:"required"` encoded []byte err error @@ -81,46 +82,19 @@ func (sd *StateDiff) Encode() ([]byte, error) { return sd.encoded, sd.err } -// AccountDiffsMap is a mapping of keccak256(address) => AccountDiff -type AccountDiffsMap map[common.Hash]AccountDiff - -// AccountDiff holds the data for a single state diff leaf node +// AccountDiff holds the data for a single state diff node type AccountDiff struct { Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` - Storage []StorageDiff `json:"storage" gencodec:"required"` Path []byte `json:"path" gencodec:"required"` + Storage []StorageDiff `json:"storage" gencodec:"required"` } -// StorageDiff holds the data for a single storage diff leaf node +// StorageDiff holds the data for a single storage diff node type StorageDiff struct { Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` Path []byte `json:"path" gencodec:"required"` } - -/* -// State trie leaf is just a short node, below -// that has an rlp encoded account as the value - - -// SO each account diffs map is reall a map of shortnode keys to values -// Flatten to a slice of short nodes? - -// Need to coerce into: - -type TrieNode struct { - // leaf, extension or branch - nodeKind string - - // If leaf or extension: [0] is key, [1] is val. - // If branch: [0] - [16] are children. - elements []interface{} - - // IPLD block information - cid *cid.Cid - rawdata []byte -} -*/ -- 2.45.2 From 5fa40688af2be7d392da8349755a20510e46aeaa Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 16 May 2019 20:58:32 -0500 Subject: [PATCH 13/17] cli parameter to limit statediffing to select account addresses + test --- cmd/geth/main.go | 4 +- cmd/geth/usage.go | 3 +- cmd/utils/flags.go | 19 +- statediff/api.go | 9 +- statediff/builder.go | 270 +++++-------- statediff/builder_test.go | 399 ++++++++++++++------ statediff/config.go | 7 +- statediff/service.go | 17 +- statediff/testhelpers/mocks/service.go | 48 ++- statediff/testhelpers/mocks/service_test.go | 5 + statediff/types.go | 30 +- 11 files changed, 468 insertions(+), 343 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index b88db8791..755100168 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -150,7 +150,9 @@ var ( utils.EVMInterpreterFlag, utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffLeafNodesOnly, + utils.StateDiffAllNodeTypes, + utils.StateDiffStreamBlock, + utils.StateDiffWatchedAddresses, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index e92964aec..d6664d5c9 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -267,8 +267,9 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffLeafNodesOnly, + utils.StateDiffAllNodeTypes, utils.StateDiffWatchedAddresses, + utils.StateDiffStreamBlock, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 13d7ff7dd..b3c0b356a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -765,16 +765,20 @@ var ( } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", - Usage: "Path and proof sets for the state and storage nodes are generated; only works with leaf nodes", + Usage: "Set to true to generate paths and proof sets for diffed state and storage trie lead nodes", } - StateDiffLeafNodesOnly = cli.BoolFlag{ - Name: "statediff.leafs", - Usage: "Consider only leaf nodes of the storage and state tries", + StateDiffAllNodeTypes = cli.BoolFlag{ + Name: "statediff.allnodes", + Usage: "Set to true to consider all node types: leaf, branch, and extension; default (false) processes leaf nodes only", } StateDiffWatchedAddresses = cli.StringSliceFlag{ Name: "statediff.watchedaddresses", Usage: "If provided, state diffing process is restricted to these addresses", } + StateDiffStreamBlock = cli.BoolFlag{ + Name: "statediff.streamblock", + Usage: "Set to true to stream the block data alongside state diff data", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1636,9 +1640,10 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st // RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ - PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), - LeafsOnly: ctx.GlobalBool(StateDiffLeafNodesOnly.Name), - WatchedAddress: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), + StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), + PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), + AllNodes: ctx.GlobalBool(StateDiffAllNodeTypes.Name), + WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum diff --git a/statediff/api.go b/statediff/api.go index 26f03c9c3..498c2f759 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -18,8 +18,6 @@ package statediff import ( "context" - "sync" - "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -36,17 +34,12 @@ const APIVersion = "0.0.1" // are produced by a full node type PublicStateDiffAPI struct { sds IService - - mu sync.Mutex - lastUsed map[string]time.Time // keeps track when a filter was polled for the last time. } // NewPublicStateDiffAPI create a new state diff websocket streaming service. func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { return &PublicStateDiffAPI{ - sds: sds, - lastUsed: make(map[string]time.Time), - mu: sync.Mutex{}, + sds: sds, } } diff --git a/statediff/builder.go b/statediff/builder.go index 26c4faaac..7de9d8beb 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -20,6 +20,7 @@ package statediff import ( + "bytes" "fmt" "math/big" @@ -27,12 +28,15 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) +var nullNode = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") + // Builder interface exposes the method for building a state diff between two blocks type Builder interface { BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) @@ -111,120 +115,71 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block }, nil } +func (sdb *builder) isWatchedAddress(hashKey []byte) bool { + // If we aren't watching any addresses, we are watching everything + if len(sdb.config.WatchedAddresses) == 0 { + return true + } + for _, addrStr := range sdb.config.WatchedAddresses { + addr := common.HexToAddress(addrStr) + addrHashKey := crypto.Keccak256(addr[:]) + if bytes.Equal(addrHashKey, hashKey) { + return true + } + } + return false +} + func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error) { var diffAccounts = make(AccountsMap) it, _ := trie.NewDifferenceIterator(a, b) - - if sdb.config.PathsAndProofs { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { + for { + log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) + if it.Leaf() && sdb.isWatchedAddress(it.LeafKey()) { + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafKeyHash := common.BytesToHash(leafKey) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + // lookup account state + var account state.Account + if err := rlp.DecodeBytes(leafValue, &account); err != nil { + return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) + } + aw := accountWrapper{ + Leaf: true, + Account: &account, + RawKey: leafKey, + RawValue: leafValue, + } + if sdb.config.PathsAndProofs { leafProof := make([][]byte, len(it.LeafProof())) copy(leafProof, it.LeafProof()) leafPath := make([]byte, len(it.Path())) copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) - } - aw := accountWrapper{ - Account: &account, - RawKey: leafKey, - RawValue: leafValue, - Proof: leafProof, - Path: leafPath, - } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw + aw.Proof = leafProof + aw.Path = leafPath } - cont := it.Next(true) - if !cont { - break + // record account to diffs (creation if we are looking at new - old; deletion if old - new) + log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) + diffAccounts[leafKeyHash] = aw + } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err) } + aw := accountWrapper{ + Leaf: false, + RawKey: nodeKey.Bytes(), + RawValue: node, + } + log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node) + diffAccounts[nodeKey] = aw } - } else { - if sdb.config.LeafsOnly { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) - } - aw := accountWrapper{ - Account: &account, - RawKey: leafKey, - RawValue: leafValue, - Proof: nil, - Path: nil, - } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw - } - cont := it.Next(true) - if !cont { - break - } - } - } else { - for { - log.Debug("Current Path and Hash", "path", pathToStr(it), "old hash", it.Hash()) - if it.Leaf() { - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafKeyHash := common.BytesToHash(leafKey) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - // lookup account state - var account state.Account - if err := rlp.DecodeBytes(leafValue, &account); err != nil { - return nil, fmt.Errorf("error looking up account via address %s\r\nerror: %v", leafKeyHash.Hex(), err) - } - aw := accountWrapper{ - Account: &account, - RawKey: leafKey, - RawValue: leafValue, - Proof: nil, - Path: nil, - } - // record account to diffs (creation if we are looking at new - old; deletion if old - new) - log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) - diffAccounts[leafKeyHash] = aw - } else { - nodeKey := it.Hash() - node, err := sdb.stateCache.TrieDB().Node(nodeKey) - if err != nil { - return nil, fmt.Errorf("error looking up intermediate state trie node %s\r\nerror: %v", nodeKey.Hex(), err) - } - aw := accountWrapper{ - Account: nil, - RawKey: nodeKey.Bytes(), - RawValue: node, - Proof: nil, - Path: nil, - } - log.Debug("intermediate state trie node lookup successful", "key", nodeKey.Hex(), "value", node) - diffAccounts[nodeKey] = aw - } - cont := it.Next(true) - if !cont { - break - } - } + cont := it.Next(true) + if !cont { + break } } @@ -244,6 +199,7 @@ func (sdb *builder) buildDiffEventual(accounts AccountsMap) ([]AccountDiff, erro } } accountDiffs = append(accountDiffs, AccountDiff{ + Leaf: val.Leaf, Key: val.RawKey, Value: val.RawValue, Proof: val.Proof, @@ -272,6 +228,7 @@ func (sdb *builder) buildDiffIncremental(creations AccountsMap, deletions Accoun } } updatedAccounts = append(updatedAccounts, AccountDiff{ + Leaf: createdAcc.Leaf, Key: createdAcc.RawKey, Value: createdAcc.RawValue, Proof: createdAcc.Proof, @@ -318,87 +275,44 @@ func (sdb *builder) buildStorageDiffsIncremental(oldSR common.Hash, newSR common func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDiff, error) { storageDiffs := make([]StorageDiff, 0) - if sdb.config.PathsAndProofs { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) + for { + log.Debug("Iterating over state at path ", "path", pathToStr(it)) + if it.Leaf() { + log.Debug("Found leaf in storage", "path", pathToStr(it)) + leafKey := make([]byte, len(it.LeafKey())) + copy(leafKey, it.LeafKey()) + leafValue := make([]byte, len(it.LeafBlob())) + copy(leafValue, it.LeafBlob()) + sd := StorageDiff{ + Leaf: true, + Key: leafKey, + Value: leafValue, + } + if sdb.config.PathsAndProofs { leafProof := make([][]byte, len(it.LeafProof())) copy(leafProof, it.LeafProof()) leafPath := make([]byte, len(it.Path())) copy(leafPath, it.Path()) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: leafPath, - Proof: leafProof, - }) + sd.Proof = leafProof + sd.Path = leafPath } - cont := it.Next(true) - if !cont { - break + storageDiffs = append(storageDiffs, sd) + } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + nodeKey := it.Hash() + node, err := sdb.stateCache.TrieDB().Node(nodeKey) + if err != nil { + return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err) } + storageDiffs = append(storageDiffs, StorageDiff{ + Leaf: false, + Key: nodeKey.Bytes(), + Value: node, + }) + log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node) } - } else { - if sdb.config.LeafsOnly { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: nil, - Proof: nil, - }) - } - cont := it.Next(true) - if !cont { - break - } - } - } else { - for { - log.Debug("Iterating over state at path ", "path", pathToStr(it)) - if it.Leaf() { - log.Debug("Found leaf in storage", "path", pathToStr(it)) - leafKey := make([]byte, len(it.LeafKey())) - copy(leafKey, it.LeafKey()) - leafValue := make([]byte, len(it.LeafBlob())) - copy(leafValue, it.LeafBlob()) - storageDiffs = append(storageDiffs, StorageDiff{ - Key: leafKey, - Value: leafValue, - Path: nil, - Proof: nil, - }) - } else { - nodeKey := it.Hash() - node, err := sdb.stateCache.TrieDB().Node(nodeKey) - if err != nil { - return nil, fmt.Errorf("error looking up intermediate storage trie node %s\r\nerror: %v", nodeKey.Hex(), err) - } - storageDiffs = append(storageDiffs, StorageDiff{ - Key: nodeKey.Bytes(), - Value: node, - Path: nil, - Proof: nil, - }) - log.Debug("intermediate storage trie node lookup successful", "key", nodeKey.Hex(), "value", node) - } - cont := it.Next(true) - if !cont { - break - } - } + cont := it.Next(true) + if !cont { + break } } diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 372241d10..7575b060a 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -32,130 +32,127 @@ import ( ) var ( - contractLeafKey common.Hash - emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) - emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0) - block0Hash, block1Hash, block2Hash, block3Hash common.Hash - block0, block1, block2, block3 *types.Block - builder statediff.Builder - miningReward = int64(2000000000000000000) - burnAddress = common.HexToAddress("0x0") - burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) + contractLeafKey common.Hash + emptyAccountDiffEventualMap = make([]statediff.AccountDiff, 0) + emptyAccountDiffIncrementalMap = make([]statediff.AccountDiff, 0) + block0, block1, block2, block3 *types.Block + builder statediff.Builder + miningReward = int64(2000000000000000000) + burnAddress = common.HexToAddress("0x0") + burnLeafKey = testhelpers.AddressToLeafKey(burnAddress) + + block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") + block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") + block2Hash = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5") + block3Hash = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a") + balanceChange10000 = int64(10000) + balanceChange1000 = int64(1000) + block1BankBalance = int64(99990000) + block1Account1Balance = int64(10000) + block2Account2Balance = int64(1000) + nonce0 = uint64(0) + nonce1 = uint64(1) + nonce2 = uint64(2) + nonce3 = uint64(3) + originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" + originalStorageLocation = common.HexToHash("0") + originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes() + updatedStorageLocation = common.HexToHash("2") + updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes() + originalStorageValue = common.Hex2Bytes("01") + updatedStorageValue = common.Hex2Bytes("03") + + account1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + bankAccount1, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(contractContractRoot), + }) + bankAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1BankBalance - balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce2, + Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + burnAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(miningReward + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + account4, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(block2Account2Balance + miningReward), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) + contractAccount2, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), + Root: common.HexToHash(newContractRoot), + }) + bankAccount3, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce3, + Balance: big.NewInt(99989000), + CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + Root: common.HexToHash(originalContractRoot), + }) ) +type arguments struct { + oldStateRoot common.Hash + newStateRoot common.Hash + blockNumber *big.Int + blockHash common.Hash +} + func TestBuilder(t *testing.T) { _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) defer chain.Stop() - block0Hash = common.HexToHash("0xd1721cfd0b29c36fd7a68f25c128e86413fb666a6e1d68e89b875bd299262661") - block1Hash = common.HexToHash("0xbbe88de60ba33a3f18c0caa37d827bfb70252e19e40a07cd34041696c35ecb1a") - block2Hash = common.HexToHash("0x34ad0fd9bb2911986b75d518c822641079dea823bc6952343ebf05da1062b6f5") - block3Hash = common.HexToHash("0x9872058136c560a6ebed0c0522b8d3016fc21f4fb0fb6585ddd8fd4c54f9909a") - block0 = blockMap[block0Hash] block1 = blockMap[block1Hash] block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] config := statediff.Config{ PathsAndProofs: true, - LeafsOnly: true, + AllNodes: false, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) - type arguments struct { - oldStateRoot common.Hash - newStateRoot common.Hash - blockNumber *big.Int - blockHash common.Hash - } - - var ( - balanceChange10000 = int64(10000) - balanceChange1000 = int64(1000) - block1BankBalance = int64(99990000) - block1Account1Balance = int64(10000) - block2Account2Balance = int64(1000) - nonce0 = uint64(0) - nonce1 = uint64(1) - nonce2 = uint64(2) - nonce3 = uint64(3) - originalContractRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - contractContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" - newContractRoot = "0x71e0d14b2b93e5c7f9748e69e1fe5f17498a1c3ac3cec29f96af13d7f8a4e070" - originalStorageLocation = common.HexToHash("0") - originalStorageKey = crypto.Keccak256Hash(originalStorageLocation[:]).Bytes() - updatedStorageLocation = common.HexToHash("2") - updatedStorageKey = crypto.Keccak256Hash(updatedStorageLocation[:]).Bytes() - originalStorageValue = common.Hex2Bytes("01") - updatedStorageValue = common.Hex2Bytes("03") - account1, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(balanceChange10000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - burnAccount1, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(miningReward), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - bankAccount1, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce1, - Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - account2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(balanceChange1000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - contractAccount, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce1, - Balance: big.NewInt(0), - CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), - Root: common.HexToHash(contractContractRoot), - }) - bankAccount2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce2, - Balance: big.NewInt(block1BankBalance - balanceChange1000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - account3, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce2, - Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - burnAccount2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(miningReward + miningReward), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - account4, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce0, - Balance: big.NewInt(block2Account2Balance + miningReward), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - contractAccount2, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce1, - Balance: big.NewInt(0), - CodeHash: common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea").Bytes(), - Root: common.HexToHash(newContractRoot), - }) - bankAccount3, _ = rlp.EncodeToBytes(state.Account{ - Nonce: nonce3, - Balance: big.NewInt(99989000), - CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), - Root: common.HexToHash(originalContractRoot), - }) - ) - var tests = []struct { name string startingArguments arguments @@ -191,6 +188,7 @@ func TestBuilder(t *testing.T) { BlockHash: block1.Hash(), CreatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -199,6 +197,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: testhelpers.Account1LeafKey.Bytes(), Value: account1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -210,6 +209,7 @@ func TestBuilder(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -235,6 +235,7 @@ func TestBuilder(t *testing.T) { BlockHash: block2.Hash(), CreatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: contractLeafKey.Bytes(), Value: contractAccount, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -242,6 +243,7 @@ func TestBuilder(t *testing.T) { Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, Storage: []statediff.StorageDiff{ { + Leaf: true, Key: originalStorageKey, Value: originalStorageValue, Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}}, @@ -250,6 +252,7 @@ func TestBuilder(t *testing.T) { }, }, { + Leaf: true, Key: testhelpers.Account2LeafKey.Bytes(), Value: account2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -261,6 +264,7 @@ func TestBuilder(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -269,6 +273,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: burnLeafKey.Bytes(), Value: burnAccount2, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -277,6 +282,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: testhelpers.Account1LeafKey.Bytes(), Value: account3, Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -304,6 +310,7 @@ func TestBuilder(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount3, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -312,6 +319,7 @@ func TestBuilder(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: contractLeafKey.Bytes(), Value: contractAccount2, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -319,6 +327,7 @@ func TestBuilder(t *testing.T) { Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, Storage: []statediff.StorageDiff{ { + Leaf: true, Key: updatedStorageKey, Value: updatedStorageValue, Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, @@ -328,6 +337,7 @@ func TestBuilder(t *testing.T) { }, }, { + Leaf: true, Key: testhelpers.Account2LeafKey.Bytes(), Value: account4, Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, @@ -363,6 +373,177 @@ func TestBuilder(t *testing.T) { } } +func TestBuilderWithWatchedAddressList(t *testing.T) { + _, blockMap, chain := testhelpers.MakeChain(3, testhelpers.Genesis) + contractLeafKey = testhelpers.AddressToLeafKey(testhelpers.ContractAddr) + defer chain.Stop() + block0 = blockMap[block0Hash] + block1 = blockMap[block1Hash] + block2 = blockMap[block2Hash] + block3 = blockMap[block3Hash] + config := statediff.Config{ + PathsAndProofs: true, + AllNodes: false, + WatchedAddresses: []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()}, + } + builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) + + var tests = []struct { + name string + startingArguments arguments + expected *statediff.StateDiff + }{ + { + "testEmptyDiff", + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block0.Root(), + blockNumber: block0.Number(), + blockHash: block0Hash, + }, + &statediff.StateDiff{ + BlockNumber: block0.Number(), + BlockHash: block0Hash, + CreatedAccounts: emptyAccountDiffEventualMap, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: emptyAccountDiffIncrementalMap, + }, + }, + { + "testBlock1", + //10000 transferred from testBankAddress to account1Addr + arguments{ + oldStateRoot: block0.Root(), + newStateRoot: block1.Root(), + blockNumber: block1.Number(), + blockHash: block1Hash, + }, + &statediff.StateDiff{ + BlockNumber: block1.Number(), + BlockHash: block1.Hash(), + CreatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account1, + Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 128, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: []statediff.AccountDiff{}, + }, + }, + { + "testBlock2", + //1000 transferred from testBankAddress to account1Addr + //1000 transferred from account1Addr to account2Addr + arguments{ + oldStateRoot: block1.Root(), + newStateRoot: block2.Root(), + blockNumber: block2.Number(), + blockHash: block2Hash, + }, + &statediff.StateDiff{ + BlockNumber: block2.Number(), + BlockHash: block2.Hash(), + CreatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: contractLeafKey.Bytes(), + Value: contractAccount, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 130, 30, 37, 86, 162, 144, 200, 100, 5, 248, 22, 10, 45, 102, 32, 66, 164, 49, 186, 69, 107, 157, 178, 101, 199, 155, 184, 55, 192, 75, 229, 240, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Leaf: true, + Key: originalStorageKey, + Value: originalStorageValue, + Proof: [][]byte{{227, 161, 32, 41, 13, 236, 217, 84, 139, 98, 168, 214, 3, 69, 169, 136, 56, 111, 200, 75, 166, 188, 149, 72, 64, 8, 246, 54, 47, 147, 22, 14, 243, 229, 99, 1}}, + Path: []byte{2, 9, 0, 13, 14, 12, 13, 9, 5, 4, 8, 11, 6, 2, 10, 8, 13, 6, 0, 3, 4, 5, 10, 9, 8, 8, 3, 8, 6, 15, 12, 8, 4, 11, 10, 6, 11, 12, 9, 5, 4, 8, 4, 0, 0, 8, 15, 6, 3, 6, 2, 15, 9, 3, 1, 6, 0, 14, 15, 3, 14, 5, 6, 3, 16}, + }, + }, + }, + }, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: testhelpers.Account1LeafKey.Bytes(), + Value: account3, + Proof: [][]byte{{248, 177, 160, 177, 155, 238, 178, 242, 47, 83, 2, 49, 141, 155, 92, 149, 175, 245, 120, 233, 177, 101, 67, 46, 200, 23, 250, 41, 74, 135, 94, 61, 133, 51, 162, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 114, 57, 32, 11, 115, 232, 140, 238, 165, 222, 121, 226, 208, 2, 192, 216, 67, 198, 179, 31, 181, 27, 208, 243, 99, 202, 48, 148, 207, 107, 106, 177, 128, 128, 128, 128, 128, 160, 10, 173, 165, 125, 110, 240, 77, 112, 149, 100, 135, 237, 25, 228, 116, 7, 195, 9, 210, 166, 208, 148, 101, 23, 244, 238, 84, 84, 211, 249, 138, 137, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 107, 160, 57, 38, 219, 105, 170, 206, 213, 24, 233, 185, 240, 244, 52, 164, 115, 231, 23, 65, 9, 201, 67, 84, 139, 184, 242, 59, 228, 28, 167, 109, 154, 210, 184, 72, 248, 70, 2, 130, 39, 16, 160, 86, 232, 31, 23, 27, 204, 85, 166, 255, 131, 69, 230, 146, 192, 248, 110, 91, 72, 224, 27, 153, 108, 173, 192, 1, 98, 47, 181, 227, 99, 180, 33, 160, 197, 210, 70, 1, 134, 247, 35, 60, 146, 126, 125, 178, 220, 199, 3, 192, 229, 0, 182, 83, 202, 130, 39, 59, 123, 250, 216, 4, 93, 133, 164, 112}}, + Path: []byte{14, 9, 2, 6, 13, 11, 6, 9, 10, 10, 12, 14, 13, 5, 1, 8, 14, 9, 11, 9, 15, 0, 15, 4, 3, 4, 10, 4, 7, 3, 14, 7, 1, 7, 4, 1, 0, 9, 12, 9, 4, 3, 5, 4, 8, 11, 11, 8, 15, 2, 3, 11, 14, 4, 1, 12, 10, 7, 6, 13, 9, 10, 13, 2, 16}, + Storage: []statediff.StorageDiff{}, + }, + }, + }, + }, + { + "testBlock3", + //the contract's storage is changed + //and the block is mined by account 2 + arguments{ + oldStateRoot: block2.Root(), + newStateRoot: block3.Root(), + blockNumber: block3.Number(), + blockHash: block3.Hash(), + }, + &statediff.StateDiff{ + BlockNumber: block3.Number(), + BlockHash: block3.Hash(), + CreatedAccounts: []statediff.AccountDiff{}, + DeletedAccounts: emptyAccountDiffEventualMap, + UpdatedAccounts: []statediff.AccountDiff{ + { + Leaf: true, + Key: contractLeafKey.Bytes(), + Value: contractAccount2, + Proof: [][]byte{{248, 177, 160, 101, 223, 138, 81, 34, 40, 229, 170, 198, 188, 136, 99, 7, 55, 33, 112, 160, 111, 181, 131, 167, 201, 131, 24, 201, 211, 177, 30, 159, 229, 246, 6, 128, 128, 128, 128, 160, 179, 86, 53, 29, 96, 188, 152, 148, 207, 31, 29, 108, 182, 140, 129, 95, 1, 49, 213, 15, 29, 168, 60, 64, 35, 160, 158, 200, 85, 207, 255, 145, 160, 32, 135, 108, 213, 150, 150, 110, 44, 170, 65, 75, 154, 74, 249, 94, 65, 74, 107, 100, 115, 39, 5, 3, 26, 22, 238, 138, 114, 254, 21, 6, 171, 128, 128, 128, 128, 128, 160, 4, 228, 121, 222, 255, 218, 60, 247, 15, 0, 34, 198, 28, 229, 180, 129, 109, 157, 68, 181, 248, 229, 200, 123, 29, 81, 145, 114, 90, 209, 205, 210, 128, 160, 255, 115, 147, 190, 57, 135, 174, 188, 86, 51, 227, 70, 22, 253, 237, 49, 24, 19, 149, 199, 142, 195, 186, 244, 70, 51, 138, 0, 146, 148, 117, 60, 128, 128}, + {248, 105, 160, 49, 20, 101, 138, 116, 217, 204, 159, 122, 207, 44, 92, 214, 150, 195, 73, 77, 124, 52, 77, 120, 191, 236, 58, 221, 13, 145, 236, 78, 141, 28, 69, 184, 70, 248, 68, 1, 128, 160, 113, 224, 209, 75, 43, 147, 229, 199, 249, 116, 142, 105, 225, 254, 95, 23, 73, 138, 28, 58, 195, 206, 194, 159, 150, 175, 19, 215, 248, 164, 224, 112, 160, 117, 63, 152, 168, 212, 50, 139, 21, 99, 110, 70, 246, 111, 44, 180, 188, 134, 1, 0, 170, 23, 150, 124, 193, 69, 252, 209, 125, 29, 71, 16, 234}}, + Path: []byte{6, 1, 1, 4, 6, 5, 8, 10, 7, 4, 13, 9, 12, 12, 9, 15, 7, 10, 12, 15, 2, 12, 5, 12, 13, 6, 9, 6, 12, 3, 4, 9, 4, 13, 7, 12, 3, 4, 4, 13, 7, 8, 11, 15, 14, 12, 3, 10, 13, 13, 0, 13, 9, 1, 14, 12, 4, 14, 8, 13, 1, 12, 4, 5, 16}, + Storage: []statediff.StorageDiff{ + { + Leaf: true, + Key: updatedStorageKey, + Value: updatedStorageValue, + Proof: [][]byte{{248, 81, 128, 128, 160, 79, 197, 241, 58, 178, 249, 186, 12, 45, 168, 139, 1, 81, 171, 14, 124, 244, 216, 93, 8, 204, 164, 92, 205, 146, 60, 106, 183, 99, 35, 235, 40, 128, 160, 205, 69, 114, 89, 105, 97, 21, 35, 94, 100, 199, 130, 35, 52, 214, 33, 41, 226, 241, 96, 68, 37, 167, 218, 100, 148, 243, 95, 196, 91, 229, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, + {226, 160, 48, 87, 135, 250, 18, 168, 35, 224, 242, 183, 99, 28, 196, 27, 59, 168, 130, 139, 51, 33, 202, 129, 17, 17, 250, 117, 205, 58, 163, 187, 90, 206, 3}}, + Path: []byte{4, 0, 5, 7, 8, 7, 15, 10, 1, 2, 10, 8, 2, 3, 14, 0, 15, 2, 11, 7, 6, 3, 1, 12, 12, 4, 1, 11, 3, 11, 10, 8, 8, 2, 8, 11, 3, 3, 2, 1, 12, 10, 8, 1, 1, 1, 1, 1, 15, 10, 7, 5, 12, 13, 3, 10, 10, 3, 11, 11, 5, 10, 12, 14, 16}, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + arguments := test.startingArguments + diff, err := builder.BuildStateDiff(arguments.oldStateRoot, arguments.newStateRoot, arguments.blockNumber, arguments.blockHash) + if err != nil { + t.Error(err) + } + receivedStateDiffRlp, err := rlp.EncodeToBytes(diff) + if err != nil { + t.Error(err) + } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) + if err != nil { + t.Error(err) + } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) + sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) + if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { + t.Logf("Test failed: %s", test.name) + t.Errorf("actual state diff rlp: %+v\nexpected state diff rlp: %+v", receivedStateDiffRlp, expectedStateDiffRlp) + } + } +} + /* contract test { diff --git a/statediff/config.go b/statediff/config.go index 4a568982f..c246cfc81 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -18,7 +18,8 @@ package statediff // Config is used to carry in parameters from CLI configuration type Config struct { - PathsAndProofs bool - LeafsOnly bool - WatchedAddress []string + StreamBlock bool + PathsAndProofs bool + AllNodes bool + WatchedAddresses []string } diff --git a/statediff/service.go b/statediff/service.go index 897032781..5e3f3e59f 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -66,6 +66,8 @@ type Service struct { Subscriptions map[rpc.ID]Subscription // Cache the last block so that we can avoid having to lookup the next block's parent lastBlock *types.Block + // Whether or not the block data is streamed alongside the state diff data in the subscription payload + streamBlock bool } // NewStateDiffService creates a new StateDiffingService @@ -76,6 +78,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Builder: NewBuilder(db, blockChain, config), QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]Subscription), + streamBlock: config.StreamBlock, }, nil } @@ -143,21 +146,23 @@ func (sds *Service) process(currentBlock, parentBlock *types.Block) error { if err != nil { return err } - - rlpBuff := new(bytes.Buffer) - currentBlock.EncodeRLP(rlpBuff) - blockRlp := rlpBuff.Bytes() stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) if err != nil { return err } payload := Payload{ - BlockRlp: blockRlp, StateDiffRlp: stateDiffRlp, Err: err, } + if sds.streamBlock { + rlpBuff := new(bytes.Buffer) + if err = currentBlock.EncodeRLP(rlpBuff); err != nil { + return err + } + payload.BlockRlp = rlpBuff.Bytes() + } - // If we have any websocket subscription listening in, send the data to them + // If we have any websocket subscriptions listening in, send the data to them sds.send(payload) return nil } diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index aaa0d0202..687a7c77d 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -42,6 +42,7 @@ type MockStateDiffService struct { ParentBlockChan chan *types.Block QuitChan chan bool Subscriptions map[rpc.ID]statediff.Subscription + streamBlock bool } // Protocols mock method @@ -76,25 +77,10 @@ func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { "current block number", currentBlock.Number()) continue } - - stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) - if err != nil { + if err := sds.process(currentBlock, parentBlock); err != nil { + println(err.Error()) log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } - rlpBuff := new(bytes.Buffer) - currentBlock.EncodeRLP(rlpBuff) - blockRlp := rlpBuff.Bytes() - stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) - if err != nil { - log.Error("Error encoding statediff", "block number", currentBlock.Number(), "error", err) - } - payload := statediff.Payload{ - BlockRlp: blockRlp, - StateDiffRlp: stateDiffRlp, - Err: err, - } - // If we have any websocket subscription listening in, send the data to them - sds.send(payload) case <-sds.QuitChan: log.Debug("Quitting the statediff block channel") sds.close() @@ -103,6 +89,34 @@ func (sds *MockStateDiffService) Loop(chan core.ChainEvent) { } } +// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions +func (sds *MockStateDiffService) process(currentBlock, parentBlock *types.Block) error { + stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) + if err != nil { + return err + } + + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + if err != nil { + return err + } + payload := statediff.Payload{ + StateDiffRlp: stateDiffRlp, + Err: err, + } + if sds.streamBlock { + rlpBuff := new(bytes.Buffer) + if err = currentBlock.EncodeRLP(rlpBuff); err != nil { + return err + } + payload.BlockRlp = rlpBuff.Bytes() + } + + // If we have any websocket subscription listening in, send the data to them + sds.send(payload) + return nil +} + // Subscribe mock method func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index 97e4893c6..4b4ac95a5 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -66,6 +66,7 @@ func TestAPI(t *testing.T) { serviceQuitChan := make(chan bool) config := statediff.Config{ PathsAndProofs: true, + AllNodes: false, } mockService := MockStateDiffService{ Mutex: sync.Mutex{}, @@ -74,6 +75,7 @@ func TestAPI(t *testing.T) { ParentBlockChan: parentBlockChain, QuitChan: serviceQuitChan, Subscriptions: make(map[rpc.ID]statediff.Subscription), + streamBlock: true, } mockService.Start(nil) id := rpc.NewID() @@ -88,6 +90,7 @@ func TestAPI(t *testing.T) { BlockHash: block1.Hash(), CreatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: burnLeafKey.Bytes(), Value: burnAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -96,6 +99,7 @@ func TestAPI(t *testing.T) { Storage: []statediff.StorageDiff{}, }, { + Leaf: true, Key: testhelpers.Account1LeafKey.Bytes(), Value: account1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, @@ -107,6 +111,7 @@ func TestAPI(t *testing.T) { DeletedAccounts: emptyAccountDiffEventualMap, UpdatedAccounts: []statediff.AccountDiff{ { + Leaf: true, Key: testhelpers.BankLeafKey.Bytes(), Value: bankAccount1, Proof: [][]byte{{248, 113, 160, 87, 118, 82, 182, 37, 183, 123, 219, 91, 247, 123, 196, 63, 49, 37, 202, 215, 70, 77, 103, 157, 21, 117, 86, 82, 119, 211, 97, 27, 128, 83, 231, 128, 128, 128, 128, 160, 254, 136, 159, 16, 229, 219, 143, 44, 43, 243, 85, 146, 129, 82, 161, 127, 110, 59, 185, 154, 146, 65, 172, 109, 132, 199, 126, 98, 100, 80, 156, 121, 128, 128, 128, 128, 128, 128, 128, 128, 160, 17, 219, 12, 218, 52, 168, 150, 218, 190, 182, 131, 155, 176, 106, 56, 244, 149, 20, 207, 164, 134, 67, 89, 132, 235, 1, 59, 125, 249, 238, 133, 197, 128, 128}, diff --git a/statediff/types.go b/statediff/types.go index 3bcbed333..6df398a10 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -23,8 +23,9 @@ import ( "encoding/json" "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" + + "github.com/ethereum/go-ethereum/common" ) // Subscription struct holds our subscription channels @@ -40,18 +41,6 @@ type Payload struct { Err error `json:"error"` } -// AccountsMap is a mapping of keccak256(address) => accountWrapper -type AccountsMap map[common.Hash]accountWrapper - -// AccountWrapper is used to temporary associate the unpacked account with its raw values -type accountWrapper struct { - Account *state.Account - RawKey []byte - RawValue []byte - Proof [][]byte - Path []byte -} - // StateDiff is the final output structure from the builder type StateDiff struct { BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` @@ -84,6 +73,7 @@ func (sd *StateDiff) Encode() ([]byte, error) { // AccountDiff holds the data for a single state diff node type AccountDiff struct { + Leaf bool `json:"leaf" gencodec:"required"` Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` @@ -93,8 +83,22 @@ type AccountDiff struct { // StorageDiff holds the data for a single storage diff node type StorageDiff struct { + Leaf bool `json:"leaf" gencodec:"required"` Key []byte `json:"key" gencodec:"required"` Value []byte `json:"value" gencodec:"required"` Proof [][]byte `json:"proof" gencodec:"required"` Path []byte `json:"path" gencodec:"required"` } + +// AccountsMap is a mapping of keccak256(address) => accountWrapper +type AccountsMap map[common.Hash]accountWrapper + +// AccountWrapper is used to temporary associate the unpacked account with its raw values +type accountWrapper struct { + Account *state.Account + Leaf bool + RawKey []byte + RawValue []byte + Proof [][]byte + Path []byte +} -- 2.45.2 From 65e9874da314da4f5a72115639a86551e7768bbf Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 17 May 2019 10:20:28 -0500 Subject: [PATCH 14/17] review fixes and fixes for issues ran into in integration --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 12 ++++++------ statediff/api.go | 14 ++++++++------ statediff/service.go | 12 ++++++------ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 755100168..8d4e9f9e8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -150,7 +150,7 @@ var ( utils.EVMInterpreterFlag, utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffAllNodeTypes, + utils.StateDiffIntermediateNodes, utils.StateDiffStreamBlock, utils.StateDiffWatchedAddresses, configFileFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index d6664d5c9..49bb0d32d 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -267,7 +267,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, - utils.StateDiffAllNodeTypes, + utils.StateDiffIntermediateNodes, utils.StateDiffWatchedAddresses, utils.StateDiffStreamBlock, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b3c0b356a..677e82f7d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -765,11 +765,11 @@ var ( } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", - Usage: "Set to true to generate paths and proof sets for diffed state and storage trie lead nodes", + Usage: "Set to true to generate paths and proof sets for diffed state and storage trie leaf nodes", } - StateDiffAllNodeTypes = cli.BoolFlag{ - Name: "statediff.allnodes", - Usage: "Set to true to consider all node types: leaf, branch, and extension; default (false) processes leaf nodes only", + StateDiffIntermediateNodes = cli.BoolFlag{ + Name: "statediff.intermediatenodes", + Usage: "Set to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only", } StateDiffWatchedAddresses = cli.StringSliceFlag{ Name: "statediff.watchedaddresses", @@ -777,7 +777,7 @@ var ( } StateDiffStreamBlock = cli.BoolFlag{ Name: "statediff.streamblock", - Usage: "Set to true to stream the block data alongside state diff data", + Usage: "Set to true to stream the block data alongside state diff data in the same subscription payload", } ) @@ -1642,7 +1642,7 @@ func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), - AllNodes: ctx.GlobalBool(StateDiffAllNodeTypes.Name), + AllNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name), WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { diff --git a/statediff/api.go b/statediff/api.go index 498c2f759..75f590daf 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -44,7 +44,7 @@ func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { } // Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created -func (api *PublicStateDiffAPI) Subscribe(ctx context.Context) (*rpc.Subscription, error) { +func (api *PublicStateDiffAPI) Subscribe(ctx context.Context, payloadChan chan Payload) (*rpc.Subscription, error) { // ensure that the RPC connection supports subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -56,19 +56,21 @@ func (api *PublicStateDiffAPI) Subscribe(ctx context.Context) (*rpc.Subscription go func() { // subscribe to events from the state diff service - payloadChannel := make(chan Payload) + payloadChannel := make(chan Payload, 10) quitChan := make(chan bool) api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) - - // loop and await state diff payloads and relay them to the subscriber with then notifier + // loop and await state diff payloads and relay them to the subscriber with the notifier for { select { case packet := <-payloadChannel: if err := notifier.Notify(rpcSub.ID, packet); err != nil { log.Error("Failed to send state diff packet", "err", err) } - case <-rpcSub.Err(): - err := api.sds.Unsubscribe(rpcSub.ID) + case err := <-rpcSub.Err(): + log.Error("State diff service rpcSub error", err) + println("err") + println(err.Error()) + err = api.sds.Unsubscribe(rpcSub.ID) if err != nil { log.Error("Failed to unsubscribe from the state diff service", err) } diff --git a/statediff/service.go b/statediff/service.go index 5e3f3e59f..49cebef5f 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -129,11 +129,11 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } case err := <-errCh: - log.Warn("Error from chain event subscription, breaking loop.", "error", err) + log.Warn("Error from chain event subscription, breaking loop", "error", err) sds.close() return case <-sds.QuitChan: - log.Info("Quitting the statediff block channel") + log.Info("Quitting the statediffing process") sds.close() return } @@ -214,9 +214,9 @@ func (sds *Service) send(payload Payload) { for id, sub := range sds.Subscriptions { select { case sub.PayloadChan <- payload: - log.Info("sending state diff payload to subscription %s", id) + log.Info(fmt.Sprintf("sending state diff payload to subscription %s", id)) default: - log.Info("unable to send payload to subscription %s; channel has no receiver", id) + log.Info(fmt.Sprintf("unable to send payload to subscription %s", id)) } } sds.Unlock() @@ -229,9 +229,9 @@ func (sds *Service) close() { select { case sub.QuitChan <- true: delete(sds.Subscriptions, id) - log.Info("closing subscription %s", id) + log.Info(fmt.Sprintf("closing subscription %s", id)) default: - log.Info("unable to close subscription %s; channel has no receiver", id) + log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) } } sds.Unlock() -- 2.45.2 From 3edcd7690fe60bb2240dd641f9efb5e9bee65ba0 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 5 Jun 2019 13:10:04 -0500 Subject: [PATCH 15/17] review fixes; proper method signature for api; adjust service so that statediff processing is halted/paused until there is at least one subscriber listening for the results --- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 16 +++---- statediff/api.go | 6 +-- statediff/builder.go | 4 +- statediff/builder_test.go | 10 ++--- statediff/config.go | 8 ++-- statediff/service.go | 45 +++++++++++++++---- statediff/service_test.go | 13 +++++- .../testhelpers/mocks/{service.go => api.go} | 0 .../mocks/{service_test.go => api_test.go} | 4 +- statediff/testhelpers/mocks/publisher.go | 36 --------------- 11 files changed, 73 insertions(+), 71 deletions(-) rename statediff/testhelpers/mocks/{service.go => api.go} (100%) rename statediff/testhelpers/mocks/{service_test.go => api_test.go} (99%) delete mode 100644 statediff/testhelpers/mocks/publisher.go diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 49bb0d32d..d667120e7 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -268,8 +268,8 @@ var AppHelpFlagGroups = []flagGroup{ utils.StateDiffFlag, utils.StateDiffPathsAndProofs, utils.StateDiffIntermediateNodes, - utils.StateDiffWatchedAddresses, utils.StateDiffStreamBlock, + utils.StateDiffWatchedAddresses, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 677e82f7d..61084cb62 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -771,14 +771,14 @@ var ( Name: "statediff.intermediatenodes", Usage: "Set to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only", } - StateDiffWatchedAddresses = cli.StringSliceFlag{ - Name: "statediff.watchedaddresses", - Usage: "If provided, state diffing process is restricted to these addresses", - } StateDiffStreamBlock = cli.BoolFlag{ Name: "statediff.streamblock", Usage: "Set to true to stream the block data alongside state diff data in the same subscription payload", } + StateDiffWatchedAddresses = cli.StringSliceFlag{ + Name: "statediff.watchedaddresses", + Usage: "If provided, state diffing process is restricted to these addresses", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1640,10 +1640,10 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st // RegisterStateDiffService configures and registers a service to stream state diff data over RPC func RegisterStateDiffService(stack *node.Node, ctx *cli.Context) { config := statediff.Config{ - StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), - PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), - AllNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name), - WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), + PathsAndProofs: ctx.GlobalBool(StateDiffPathsAndProofs.Name), + IntermediateNodes: ctx.GlobalBool(StateDiffIntermediateNodes.Name), + StreamBlock: ctx.GlobalBool(StateDiffStreamBlock.Name), + WatchedAddresses: ctx.GlobalStringSlice(StateDiffWatchedAddresses.Name), } if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum diff --git a/statediff/api.go b/statediff/api.go index 75f590daf..946be1146 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -43,8 +43,8 @@ func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { } } -// Subscribe is the public method to setup a subscription that fires off state-diff payloads as they are created -func (api *PublicStateDiffAPI) Subscribe(ctx context.Context, payloadChan chan Payload) (*rpc.Subscription, error) { +// Stream is the public method to setup a subscription that fires off state-diff payloads as they are created +func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, error) { // ensure that the RPC connection supports subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -68,8 +68,6 @@ func (api *PublicStateDiffAPI) Subscribe(ctx context.Context, payloadChan chan P } case err := <-rpcSub.Err(): log.Error("State diff service rpcSub error", err) - println("err") - println(err.Error()) err = api.sds.Unsubscribe(rpcSub.ID) if err != nil { log.Error("Failed to unsubscribe from the state diff service", err) diff --git a/statediff/builder.go b/statediff/builder.go index 7de9d8beb..765152e87 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -163,7 +163,7 @@ func (sdb *builder) collectDiffNodes(a, b trie.NodeIterator) (AccountsMap, error // record account to diffs (creation if we are looking at new - old; deletion if old - new) log.Debug("Account lookup successful", "address", leafKeyHash, "account", account) diffAccounts[leafKeyHash] = aw - } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + } else if sdb.config.IntermediateNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { nodeKey := it.Hash() node, err := sdb.stateCache.TrieDB().Node(nodeKey) if err != nil { @@ -297,7 +297,7 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDi sd.Path = leafPath } storageDiffs = append(storageDiffs, sd) - } else if sdb.config.AllNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { + } else if sdb.config.IntermediateNodes && !bytes.Equal(nullNode, it.Hash().Bytes()) { nodeKey := it.Hash() node, err := sdb.stateCache.TrieDB().Node(nodeKey) if err != nil { diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 7575b060a..2c9253de1 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -148,8 +148,8 @@ func TestBuilder(t *testing.T) { block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] config := statediff.Config{ - PathsAndProofs: true, - AllNodes: false, + PathsAndProofs: true, + IntermediateNodes: false, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) @@ -382,9 +382,9 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { block2 = blockMap[block2Hash] block3 = blockMap[block3Hash] config := statediff.Config{ - PathsAndProofs: true, - AllNodes: false, - WatchedAddresses: []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()}, + PathsAndProofs: true, + IntermediateNodes: false, + WatchedAddresses: []string{testhelpers.Account1Addr.Hex(), testhelpers.ContractAddr.Hex()}, } builder = statediff.NewBuilder(testhelpers.Testdb, chain, config) diff --git a/statediff/config.go b/statediff/config.go index c246cfc81..70f09a749 100644 --- a/statediff/config.go +++ b/statediff/config.go @@ -18,8 +18,8 @@ package statediff // Config is used to carry in parameters from CLI configuration type Config struct { - StreamBlock bool - PathsAndProofs bool - AllNodes bool - WatchedAddresses []string + PathsAndProofs bool + IntermediateNodes bool + StreamBlock bool + WatchedAddresses []string } diff --git a/statediff/service.go b/statediff/service.go index 49cebef5f..f3e47d16e 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -20,8 +20,7 @@ import ( "bytes" "fmt" "sync" - - "github.com/ethereum/go-ethereum/rlp" + "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -68,6 +68,8 @@ type Service struct { lastBlock *types.Block // Whether or not the block data is streamed alongside the state diff data in the subscription payload streamBlock bool + // Whether or not we have any subscribers; only if we do, do we processes state diffs + subscribers int32 } // NewStateDiffService creates a new StateDiffingService @@ -110,6 +112,11 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { //Notify chain event channel of events case chainEvent := <-chainEventCh: log.Debug("Event received from chainEventCh", "event", chainEvent) + // if we don't have any subscribers, do not process a statediff + if atomic.LoadInt32(&sds.subscribers) == 0 { + log.Debug("Currently no subscribers to the statediffing service; processing is halted") + continue + } currentBlock := chainEvent.Block parentHash := currentBlock.ParentHash() var parentBlock *types.Block @@ -125,7 +132,7 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { "current block number", currentBlock.Number()) continue } - if err := sds.process(currentBlock, parentBlock); err != nil { + if err := sds.processStateDiff(currentBlock, parentBlock); err != nil { log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) } case err := <-errCh: @@ -140,8 +147,8 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { } } -// process method builds the state diff payload from the current and parent block and streams it to listening subscriptions -func (sds *Service) process(currentBlock, parentBlock *types.Block) error { +// processStateDiff method builds the state diff payload from the current and parent block and sends it to listening subscriptions +func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) error { stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { return err @@ -170,6 +177,9 @@ func (sds *Service) process(currentBlock, parentBlock *types.Block) error { // Subscribe is used by the API to subscribe to the StateDiffingService loop func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") + if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) { + log.Info("State diffing subscription received; beginning statediff processing") + } sds.Lock() sds.Subscriptions[id] = Subscription{ PayloadChan: sub, @@ -187,6 +197,11 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { return fmt.Errorf("cannot unsubscribe; subscription for id %s does not exist", id) } delete(sds.Subscriptions, id) + if len(sds.Subscriptions) == 0 { + if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) { + log.Info("No more subscriptions; halting statediff processing") + } + } sds.Unlock() return nil } @@ -208,7 +223,7 @@ func (sds *Service) Stop() error { return nil } -// send is used to fan out and serve a payload to any subscriptions +// send is used to fan out and serve the statediff payload to all subscriptions func (sds *Service) send(payload Payload) { sds.Lock() for id, sub := range sds.Subscriptions { @@ -216,7 +231,21 @@ func (sds *Service) send(payload Payload) { case sub.PayloadChan <- payload: log.Info(fmt.Sprintf("sending state diff payload to subscription %s", id)) default: - log.Info(fmt.Sprintf("unable to send payload to subscription %s", id)) + log.Info(fmt.Sprintf("unable to send payload to subscription %s; channel has no receiver", id)) + // in this case, try to close the bad subscription and remove it + select { + case sub.QuitChan <- true: + log.Info(fmt.Sprintf("closing subscription %s", id)) + default: + log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) + } + delete(sds.Subscriptions, id) + } + } + // If after removing all bad subscriptions we have none left, halt processing + if len(sds.Subscriptions) == 0 { + if atomic.CompareAndSwapInt32(&sds.subscribers, 1, 0) { + log.Info("No more subscriptions; halting statediff processing") } } sds.Unlock() @@ -228,11 +257,11 @@ func (sds *Service) close() { for id, sub := range sds.Subscriptions { select { case sub.QuitChan <- true: - delete(sds.Subscriptions, id) log.Info(fmt.Sprintf("closing subscription %s", id)) default: log.Info(fmt.Sprintf("unable to close subscription %s; channel has no receiver", id)) } + delete(sds.Subscriptions, id) } sds.Unlock() } diff --git a/statediff/service_test.go b/statediff/service_test.go index d5edee04e..de3df3dd2 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -33,7 +33,7 @@ import ( func TestServiceLoop(t *testing.T) { testErrorInChainEventLoop(t) - testErrorInBlockLoop(t) + //testErrorInBlockLoop(t) } var ( @@ -76,10 +76,21 @@ func testErrorInChainEventLoop(t *testing.T) { QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]statediff.Subscription), } + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan) testRoot2 = common.HexToHash("0xTestRoot2") blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) + // Need to have listeners on the channels or the subscription will be closed and the processing halted + go func() { + select { + case <-payloadChan: + case <-quitChan: + } + }() service.Loop(eventsChannel) + if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { t.Error("Test failure:", t.Name()) t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/api.go similarity index 100% rename from statediff/testhelpers/mocks/service.go rename to statediff/testhelpers/mocks/api.go diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/api_test.go similarity index 99% rename from statediff/testhelpers/mocks/service_test.go rename to statediff/testhelpers/mocks/api_test.go index 4b4ac95a5..22971b4b6 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/api_test.go @@ -65,8 +65,8 @@ func TestAPI(t *testing.T) { parentBlockChain := make(chan *types.Block) serviceQuitChan := make(chan bool) config := statediff.Config{ - PathsAndProofs: true, - AllNodes: false, + PathsAndProofs: true, + IntermediateNodes: false, } mockService := MockStateDiffService{ Mutex: sync.Mutex{}, diff --git a/statediff/testhelpers/mocks/publisher.go b/statediff/testhelpers/mocks/publisher.go deleted file mode 100644 index 6a6018746..000000000 --- a/statediff/testhelpers/mocks/publisher.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mocks - -import "github.com/ethereum/go-ethereum/statediff" - -// Publisher mock -type Publisher struct { - StateDiff *statediff.StateDiff - publisherError error -} - -// PublishStateDiff mock method -func (publisher *Publisher) PublishStateDiff(sd *statediff.StateDiff) (string, error) { - publisher.StateDiff = sd - return "", publisher.publisherError -} - -// SetPublisherError mock method -func (publisher *Publisher) SetPublisherError(err error) { - publisher.publisherError = err -} -- 2.45.2 From 00cc1f89ff57f613e4014865121f66dbfbb37df7 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 6 Jun 2019 21:24:17 -0500 Subject: [PATCH 16/17] adjust buffering to improve stability; doc.go; fix notifier err handling --- cmd/utils/flags.go | 2 +- core/blockchain_test.go | 4 ++-- statediff/api.go | 23 ++++++++++++------- statediff/doc.go | 49 +++++++++++++++++++++++++++++++++++++++++ statediff/service.go | 10 ++++----- 5 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 statediff/doc.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 61084cb62..ec02856c4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -761,7 +761,7 @@ var ( StateDiffFlag = cli.BoolFlag{ Name: "statediff", - Usage: "Enables the calculation of state diffs between each block, persists these state diffs the configured persistence mode.", + Usage: "Enables the processing of state diffs between each block", } StateDiffPathsAndProofs = cli.BoolFlag{ Name: "statediff.pathsandproofs", diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 2480ab55d..be105d2db 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2227,8 +2227,8 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) { func BenchmarkBlockChain_1x1000Executions(b *testing.B) { var ( - numTxs= 1000 - numBlocks= 1 + numTxs = 1000 + numBlocks = 1 ) b.StopTimer() b.ResetTimer() diff --git a/statediff/api.go b/statediff/api.go index 946be1146..a05ef5510 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -56,23 +56,30 @@ func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, e go func() { // subscribe to events from the state diff service - payloadChannel := make(chan Payload, 10) - quitChan := make(chan bool) + payloadChannel := make(chan Payload, chainEventChanSize) + quitChan := make(chan bool, 1) api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) // loop and await state diff payloads and relay them to the subscriber with the notifier for { select { case packet := <-payloadChannel: - if err := notifier.Notify(rpcSub.ID, packet); err != nil { - log.Error("Failed to send state diff packet", "err", err) + if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil { + log.Error("Failed to send state diff packet; error: " + notifyErr.Error()) + unSubErr := api.sds.Unsubscribe(rpcSub.ID) + if unSubErr != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + unSubErr.Error()) + } + return } case err := <-rpcSub.Err(): - log.Error("State diff service rpcSub error", err) - err = api.sds.Unsubscribe(rpcSub.ID) if err != nil { - log.Error("Failed to unsubscribe from the state diff service", err) + log.Error("State diff service rpcSub error: " + err.Error()) + err = api.sds.Unsubscribe(rpcSub.ID) + if err != nil { + log.Error("Failed to unsubscribe from the state diff service; error: " + err.Error()) + } + return } - return case <-quitChan: // don't need to unsubscribe, statediff service does so before sending the quit signal return diff --git a/statediff/doc.go b/statediff/doc.go new file mode 100644 index 000000000..0e6d5f3e1 --- /dev/null +++ b/statediff/doc.go @@ -0,0 +1,49 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go + +Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, +relaying the objects to any rpc subscriptions. + +The service is spun up using the below CLI flags +--statediff: boolean flag, turns on the service +--statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs. +--statediff.intermediatenodes: boolean flag, tells service to include intermediate (branch and extension) nodes; default (false) processes leaf nodes only. +--statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes. +--statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3 + +If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. + +Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, +with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". + +e.g. + +cli, _ := rpc.Dial("ipcPathOrWsURL") +stateDiffPayloadChan := make(chan statediff.Payload, 20000) +rpcSub, err := cli.Subscribe(context.Background(), "statediff", stateDiffPayloadChan, "stream"}) +for { + select { + case stateDiffPayload := <- stateDiffPayloadChan: + processPayload(stateDiffPayload) + case err := <= rpcSub.Err(): + log.Error(err) + } +} +*/ +package statediff diff --git a/statediff/service.go b/statediff/service.go index f3e47d16e..e978fcb8f 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -34,6 +34,8 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) +const chainEventChanSize = 20000 + type blockChain interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription GetBlockByHash(hash common.Hash) *types.Block @@ -127,13 +129,11 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { } sds.lastBlock = currentBlock if parentBlock == nil { - log.Error("Parent block is nil, skipping this block", - "parent block hash", parentHash.String(), - "current block number", currentBlock.Number()) + log.Error(fmt.Sprintf("Parent block is nil, skipping this block (%d)", currentBlock.Number())) continue } if err := sds.processStateDiff(currentBlock, parentBlock); err != nil { - log.Error("Error building statediff", "block number", currentBlock.Number(), "error", err) + log.Error(fmt.Sprintf("Error building statediff for block %d; error: ", currentBlock.Number()) + err.Error()) } case err := <-errCh: log.Warn("Error from chain event subscription, breaking loop", "error", err) @@ -210,7 +210,7 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { func (sds *Service) Start(*p2p.Server) error { log.Info("Starting statediff service") - chainEventCh := make(chan core.ChainEvent, 10) + chainEventCh := make(chan core.ChainEvent, chainEventChanSize) go sds.Loop(chainEventCh) return nil -- 2.45.2 From ca79f6ef9877a5d401c7f180809157923da4e93b Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 18 Jul 2019 13:55:37 -0500 Subject: [PATCH 17/17] relay receipts with the rest of the data + review fixes/changes --- core/blockchain.go | 4 +- statediff/api.go | 18 ++--- statediff/builder.go | 20 ++---- statediff/doc.go | 17 +++-- statediff/helpers.go | 31 ++------- statediff/service.go | 38 ++++++----- statediff/service_test.go | 81 +++++++++++++++++++---- statediff/testhelpers/mocks/api.go | 11 ++- statediff/testhelpers/mocks/api_test.go | 3 - statediff/testhelpers/mocks/blockchain.go | 24 +++++-- statediff/types.go | 49 +++++++------- 11 files changed, 175 insertions(+), 121 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 2a930cccf..d88110f5d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1345,7 +1345,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. break } if bc.cacheConfig.ProcessingStateDiffs { - if !bc.allowedRootToBeDereferenced(root.(common.Hash)) { + if !bc.rootAllowedToBeDereferenced(root.(common.Hash)) { bc.triegc.Push(root, number) break } else { @@ -1414,7 +1414,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. // since we need the state tries of the current block and its parent in-memory // in order to process statediffs, we should avoid dereferencing roots until // its statediff and its child have been processed -func (bc *BlockChain) allowedRootToBeDereferenced(root common.Hash) bool { +func (bc *BlockChain) rootAllowedToBeDereferenced(root common.Hash) bool { diffProcessedForSelfAndChildCount := 2 count := bc.stateDiffsProcessed[root] return count >= diffProcessedForSelfAndChildCount diff --git a/statediff/api.go b/statediff/api.go index a05ef5510..52c604f97 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -29,21 +29,21 @@ const APIName = "statediff" // APIVersion is the version of the state diffing service API const APIVersion = "0.0.1" -// PublicStateDiffAPI provides the a websocket service +// PublicStateDiffAPI provides an RPC subscription interface // that can be used to stream out state diffs as they // are produced by a full node type PublicStateDiffAPI struct { sds IService } -// NewPublicStateDiffAPI create a new state diff websocket streaming service. +// NewPublicStateDiffAPI creates an rpc subscription interface for the underlying statediff service func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI { return &PublicStateDiffAPI{ sds: sds, } } -// Stream is the public method to setup a subscription that fires off state-diff payloads as they are created +// Stream is the public method to setup a subscription that fires off statediff service payloads as they are created func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, error) { // ensure that the RPC connection supports subscriptions notifier, supported := rpc.NotifierFromContext(ctx) @@ -51,19 +51,19 @@ func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, e return nil, rpc.ErrNotificationsUnsupported } - // create subscription and start waiting for statediff events + // create subscription and start waiting for events rpcSub := notifier.CreateSubscription() go func() { - // subscribe to events from the state diff service + // subscribe to events from the statediff service payloadChannel := make(chan Payload, chainEventChanSize) quitChan := make(chan bool, 1) api.sds.Subscribe(rpcSub.ID, payloadChannel, quitChan) - // loop and await state diff payloads and relay them to the subscriber with the notifier + // loop and await payloads and relay them to the subscriber with the notifier for { select { - case packet := <-payloadChannel: - if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil { + case payload := <-payloadChannel: + if notifyErr := notifier.Notify(rpcSub.ID, payload); notifyErr != nil { log.Error("Failed to send state diff packet; error: " + notifyErr.Error()) unSubErr := api.sds.Unsubscribe(rpcSub.ID) if unSubErr != nil { @@ -81,7 +81,7 @@ func (api *PublicStateDiffAPI) Stream(ctx context.Context) (*rpc.Subscription, e return } case <-quitChan: - // don't need to unsubscribe, statediff service does so before sending the quit signal + // don't need to unsubscribe, service does so before sending the quit signal return } } diff --git a/statediff/builder.go b/statediff/builder.go index 765152e87..bbd523a5f 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -25,7 +25,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" @@ -49,7 +48,7 @@ type builder struct { stateCache state.Database } -// NewBuilder is used to create a state diff builder +// NewBuilder is used to create a statediff builder func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) Builder { return &builder{ chainDB: db, @@ -58,7 +57,7 @@ func NewBuilder(db ethdb.Database, blockChain *core.BlockChain, config Config) B } } -// BuildStateDiff builds a StateDiff object from two blocks +// BuildStateDiff builds a statediff object from two blocks func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, blockNumber *big.Int, blockHash common.Hash) (StateDiff, error) { // Generate tries for old and new states sdb.stateCache = sdb.blockChain.StateCache() @@ -115,8 +114,9 @@ func (sdb *builder) BuildStateDiff(oldStateRoot, newStateRoot common.Hash, block }, nil } +// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch func (sdb *builder) isWatchedAddress(hashKey []byte) bool { - // If we aren't watching any addresses, we are watching everything + // If we aren't watching any specific addresses, we are watching everything if len(sdb.config.WatchedAddresses) == 0 { return true } @@ -318,15 +318,3 @@ func (sdb *builder) buildStorageDiffsFromTrie(it trie.NodeIterator) ([]StorageDi return storageDiffs, nil } - -func (sdb *builder) addressByPath(path []byte) (*common.Address, error) { - log.Debug("Looking up address from path", "path", hexutil.Encode(append([]byte("secure-key-"), path...))) - addrBytes, err := sdb.chainDB.Get(append([]byte("secure-key-"), hexToKeyBytes(path)...)) - if err != nil { - log.Error("Error looking up address via path", "path", hexutil.Encode(append([]byte("secure-key-"), path...)), "error", err) - return nil, err - } - addr := common.BytesToAddress(addrBytes) - log.Debug("Address found", "Address", addr) - return &addr, nil -} diff --git a/statediff/doc.go b/statediff/doc.go index 0e6d5f3e1..35c48c02d 100644 --- a/statediff/doc.go +++ b/statediff/doc.go @@ -15,11 +15,11 @@ // along with the go-ethereum library. If not, see . /* -This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go - Package statediff provides an auxiliary service that processes state diff objects from incoming chain events, relaying the objects to any rpc subscriptions. +This work is adapted from work by Charles Crain at https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go + The service is spun up using the below CLI flags --statediff: boolean flag, turns on the service --statediff.streamblock: boolean flag, configures the service to associate and stream out the rest of the block data with the state diffs. @@ -27,7 +27,16 @@ The service is spun up using the below CLI flags --statediff.pathsandproofs: boolean flag, tells service to generate paths and proofs for the diffed storage and state trie leaf nodes. --statediff.watchedaddresses: string slice flag, used to limit the state diffing process to the given addresses. Usage: --statediff.watchedaddresses=addr1 --statediff.watchedaddresses=addr2 --statediff.watchedaddresses=addr3 -If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. +If you wish to use the websocket endpoint to subscribe to the statediff service, be sure to open up the Websocket RPC server with the `--ws` flag. The IPC-RPC server is turned on by default. + +The statediffing services works only with `--syncmode="full", but -importantly- does not require garbage collection to be turned off (does not require an archival node). + +e.g. + +$ ./geth --statediff --statediff.streamblock --ws --syncmode "full" + +This starts up the geth node in full sync mode, starts up the statediffing service, and opens up the websocket endpoint to subscribe to the service. +Because the "streamblock" flag has been turned on, the service will strean out block data (headers, transactions, and receipts) along with the diffed state and storage leafs. Rpc subscriptions to the service can be created using the rpc.Client.Subscribe() method, with the "statediff" namespace, a statediff.Payload channel, and the name of the statediff api's rpc method- "stream". @@ -41,7 +50,7 @@ for { select { case stateDiffPayload := <- stateDiffPayloadChan: processPayload(stateDiffPayload) - case err := <= rpcSub.Err(): + case err := <- rpcSub.Err(): log.Error(err) } } diff --git a/statediff/helpers.go b/statediff/helpers.go index 89852b55c..2c5ddeb45 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -37,7 +37,7 @@ func sortKeys(data AccountsMap) []string { return keys } -// BytesToNiblePath +// bytesToNiblePath converts the byte representation of a path to its string representation func bytesToNiblePath(path []byte) string { if hasTerm(path) { path = path[:len(path)-1] @@ -53,6 +53,8 @@ func bytesToNiblePath(path []byte) string { return nibblePath } +// findIntersection finds the set of strings from both arrays that are equivalent (same key as same index) +// this is used to find which keys have been both "deleted" and "created" i.e. they were updated func findIntersection(a, b []string) []string { lenA := len(a) lenB := len(b) @@ -63,13 +65,13 @@ func findIntersection(a, b []string) []string { } for { switch strings.Compare(a[iOfA], b[iOfB]) { - // a[iOfA] < b[iOfB] + // -1 when a[iOfA] < b[iOfB] case -1: iOfA++ if iOfA >= lenA { return updates } - // a[iOfA] == b[iOfB] + // 0 when a[iOfA] == b[iOfB] case 0: updates = append(updates, a[iOfA]) iOfA++ @@ -77,7 +79,7 @@ func findIntersection(a, b []string) []string { if iOfA >= lenA || iOfB >= lenB { return updates } - // a[iOfA] > b[iOfB] + // 1 when a[iOfA] > b[iOfB] case 1: iOfB++ if iOfB >= lenB { @@ -88,30 +90,11 @@ func findIntersection(a, b []string) []string { } +// pathToStr converts the NodeIterator path to a string representation func pathToStr(it trie.NodeIterator) string { return bytesToNiblePath(it.Path()) } -// Duplicated from trie/encoding.go -func hexToKeyBytes(hex []byte) []byte { - if hasTerm(hex) { - hex = hex[:len(hex)-1] - } - if len(hex)&1 != 0 { - panic("can't convert hex key of odd length") - } - key := make([]byte, (len(hex)+1)/2) - decodeNibbles(hex, key) - - return key -} - -func decodeNibbles(nibbles []byte, bytes []byte) { - for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { - bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] - } -} - // hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 diff --git a/statediff/service.go b/statediff/service.go index e978fcb8f..d3eab1065 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -40,6 +40,7 @@ type blockChain interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription GetBlockByHash(hash common.Hash) *types.Block AddToStateDiffProcessedCollection(hash common.Hash) + GetReceiptsByHash(hash common.Hash) types.Receipts } // IService is the state-diffing service interface @@ -69,12 +70,12 @@ type Service struct { // Cache the last block so that we can avoid having to lookup the next block's parent lastBlock *types.Block // Whether or not the block data is streamed alongside the state diff data in the subscription payload - streamBlock bool + StreamBlock bool // Whether or not we have any subscribers; only if we do, do we processes state diffs subscribers int32 } -// NewStateDiffService creates a new StateDiffingService +// NewStateDiffService creates a new statediff.Service func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Config) (*Service, error) { return &Service{ Mutex: sync.Mutex{}, @@ -82,7 +83,7 @@ func NewStateDiffService(db ethdb.Database, blockChain *core.BlockChain, config Builder: NewBuilder(db, blockChain, config), QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]Subscription), - streamBlock: config.StreamBlock, + StreamBlock: config.StreamBlock, }, nil } @@ -91,7 +92,7 @@ func (sds *Service) Protocols() []p2p.Protocol { return []p2p.Protocol{} } -// APIs returns the RPC descriptors the StateDiffingService offers +// APIs returns the RPC descriptors the statediff.Service offers func (sds *Service) APIs() []rpc.API { return []rpc.API{ { @@ -108,7 +109,6 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { chainEventSub := sds.BlockChain.SubscribeChainEvent(chainEventCh) defer chainEventSub.Unsubscribe() errCh := chainEventSub.Err() - for { select { //Notify chain event channel of events @@ -147,7 +147,7 @@ func (sds *Service) Loop(chainEventCh chan core.ChainEvent) { } } -// processStateDiff method builds the state diff payload from the current and parent block and sends it to listening subscriptions +// processStateDiff method builds the state diff payload from the current and parent block before sending it to listening subscriptions func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) error { stateDiff, err := sds.Builder.BuildStateDiff(parentBlock.Root(), currentBlock.Root(), currentBlock.Number(), currentBlock.Hash()) if err != nil { @@ -159,22 +159,26 @@ func (sds *Service) processStateDiff(currentBlock, parentBlock *types.Block) err } payload := Payload{ StateDiffRlp: stateDiffRlp, - Err: err, } - if sds.streamBlock { - rlpBuff := new(bytes.Buffer) - if err = currentBlock.EncodeRLP(rlpBuff); err != nil { + if sds.StreamBlock { + blockBuff := new(bytes.Buffer) + if err = currentBlock.EncodeRLP(blockBuff); err != nil { return err } - payload.BlockRlp = rlpBuff.Bytes() + payload.BlockRlp = blockBuff.Bytes() + receiptBuff := new(bytes.Buffer) + receipts := sds.BlockChain.GetReceiptsByHash(currentBlock.Hash()) + if err = rlp.Encode(receiptBuff, receipts); err != nil { + return err + } + payload.ReceiptsRlp = receiptBuff.Bytes() } - // If we have any websocket subscriptions listening in, send the data to them sds.send(payload) return nil } -// Subscribe is used by the API to subscribe to the StateDiffingService loop +// Subscribe is used by the API to subscribe to the service loop func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool) { log.Info("Subscribing to the statediff service") if atomic.CompareAndSwapInt32(&sds.subscribers, 0, 1) { @@ -188,7 +192,7 @@ func (sds *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- boo sds.Unlock() } -// Unsubscribe is used to unsubscribe to the StateDiffingService loop +// Unsubscribe is used to unsubscribe from the service loop func (sds *Service) Unsubscribe(id rpc.ID) error { log.Info("Unsubscribing from the statediff service") sds.Lock() @@ -206,7 +210,7 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { return nil } -// Start is used to begin the StateDiffingService +// Start is used to begin the service func (sds *Service) Start(*p2p.Server) error { log.Info("Starting statediff service") @@ -216,14 +220,14 @@ func (sds *Service) Start(*p2p.Server) error { return nil } -// Stop is used to close down the StateDiffingService +// Stop is used to close down the service func (sds *Service) Stop() error { log.Info("Stopping statediff service") close(sds.QuitChan) return nil } -// send is used to fan out and serve the statediff payload to all subscriptions +// send is used to fan out and serve the payloads to all subscriptions func (sds *Service) send(payload Payload) { sds.Lock() for id, sub := range sds.Subscriptions { diff --git a/statediff/service_test.go b/statediff/service_test.go index de3df3dd2..6119f6ecb 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -21,11 +21,13 @@ import ( "math/big" "math/rand" "reflect" + "sync" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" @@ -33,7 +35,7 @@ import ( func TestServiceLoop(t *testing.T) { testErrorInChainEventLoop(t) - //testErrorInBlockLoop(t) + testErrorInBlockLoop(t) } var ( @@ -61,6 +63,12 @@ var ( testBlock2 = types.NewBlock(&header2, nil, nil, nil) testBlock3 = types.NewBlock(&header3, nil, nil, nil) + receiptRoot1 = common.HexToHash("0x05") + receiptRoot2 = common.HexToHash("0x06") + receiptRoot3 = common.HexToHash("0x07") + testReceipts1 = []*types.Receipt{types.NewReceipt(receiptRoot1.Bytes(), false, 1000), types.NewReceipt(receiptRoot2.Bytes(), false, 2000)} + testReceipts2 = []*types.Receipt{types.NewReceipt(receiptRoot3.Bytes(), false, 3000)} + event1 = core.ChainEvent{Block: testBlock1} event2 = core.ChainEvent{Block: testBlock2} event3 = core.ChainEvent{Block: testBlock3} @@ -71,43 +79,79 @@ func testErrorInChainEventLoop(t *testing.T) { builder := mocks.Builder{} blockChain := mocks.BlockChain{} service := statediff.Service{ + Mutex: sync.Mutex{}, Builder: &builder, BlockChain: &blockChain, QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]statediff.Subscription), + StreamBlock: true, } - payloadChan := make(chan statediff.Payload) + payloadChan := make(chan statediff.Payload, 2) quitChan := make(chan bool) service.Subscribe(rpc.NewID(), payloadChan, quitChan) testRoot2 = common.HexToHash("0xTestRoot2") - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, parentBlock2}) + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockMapping[parentBlock2.Hash()] = parentBlock2 + blockChain.SetParentBlocksToReturn(blockMapping) blockChain.SetChainEvents([]core.ChainEvent{event1, event2, event3}) - // Need to have listeners on the channels or the subscription will be closed and the processing halted + blockChain.SetReceiptsForHash(testBlock1.Hash(), testReceipts1) + blockChain.SetReceiptsForHash(testBlock2.Hash(), testReceipts2) + + payloads := make([]statediff.Payload, 0, 2) + wg := sync.WaitGroup{} go func() { - select { - case <-payloadChan: - case <-quitChan: + wg.Add(1) + for i := 0; i < 2; i++ { + select { + case payload := <-payloadChan: + payloads = append(payloads, payload) + case <-quitChan: + } } + wg.Done() }() + service.Loop(eventsChannel) + wg.Wait() + if len(payloads) != 2 { + t.Error("Test failure:", t.Name()) + t.Logf("Actual number of payloads does not equal expected.\nactual: %+v\nexpected: 3", len(payloads)) + } + + testReceipts1Rlp, err := rlp.EncodeToBytes(testReceipts1) + if err != nil { + t.Error(err) + } + testReceipts2Rlp, err := rlp.EncodeToBytes(testReceipts2) + if err != nil { + t.Error(err) + } + expectedReceiptsRlp := [][]byte{testReceipts1Rlp, testReceipts2Rlp, nil} + for i, payload := range payloads { + if !bytes.Equal(payload.ReceiptsRlp, expectedReceiptsRlp[i]) { + t.Error("Test failure:", t.Name()) + t.Logf("Actual receipt rlp for payload %d does not equal expected.\nactual: %+v\nexpected: %+v", i, payload.ReceiptsRlp, expectedReceiptsRlp[i]) + } + } if !reflect.DeepEqual(builder.BlockHash, testBlock2.Hash()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) + t.Logf("Actual blockhash does not equal expected.\nactual:%+v\nexpected: %+v", builder.BlockHash, testBlock2.Hash()) } if !bytes.Equal(builder.OldStateRoot.Bytes(), parentBlock2.Root().Bytes()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) + t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.OldStateRoot, parentBlock2.Root()) } if !bytes.Equal(builder.NewStateRoot.Bytes(), testBlock2.Root().Bytes()) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) + t.Logf("Actual root does not equal expected.\nactual:%+v\nexpected: %+v", builder.NewStateRoot, testBlock2.Root()) } //look up the parent block from its hash expectedHashes := []common.Hash{testBlock1.ParentHash(), testBlock2.ParentHash()} if !reflect.DeepEqual(blockChain.ParentHashesLookedUp, expectedHashes) { t.Error("Test failure:", t.Name()) - t.Logf("Actual does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) + t.Logf("Actual parent hash does not equal expected.\nactual:%+v\nexpected: %+v", blockChain.ParentHashesLookedUp, expectedHashes) } } @@ -121,9 +165,20 @@ func testErrorInBlockLoop(t *testing.T) { QuitChan: make(chan bool), Subscriptions: make(map[rpc.ID]statediff.Subscription), } - - blockChain.SetParentBlocksToReturn([]*types.Block{parentBlock1, nil}) + payloadChan := make(chan statediff.Payload) + quitChan := make(chan bool) + service.Subscribe(rpc.NewID(), payloadChan, quitChan) + blockMapping := make(map[common.Hash]*types.Block) + blockMapping[parentBlock1.Hash()] = parentBlock1 + blockChain.SetParentBlocksToReturn(blockMapping) blockChain.SetChainEvents([]core.ChainEvent{event1, event2}) + // Need to have listeners on the channels or the subscription will be closed and the processing halted + go func() { + select { + case <-payloadChan: + case <-quitChan: + } + }() service.Loop(eventsChannel) if !bytes.Equal(builder.BlockHash.Bytes(), testBlock1.Hash().Bytes()) { diff --git a/statediff/testhelpers/mocks/api.go b/statediff/testhelpers/mocks/api.go index 687a7c77d..3b43ab7dd 100644 --- a/statediff/testhelpers/mocks/api.go +++ b/statediff/testhelpers/mocks/api.go @@ -102,7 +102,6 @@ func (sds *MockStateDiffService) process(currentBlock, parentBlock *types.Block) } payload := statediff.Payload{ StateDiffRlp: stateDiffRlp, - Err: err, } if sds.streamBlock { rlpBuff := new(bytes.Buffer) @@ -119,7 +118,7 @@ func (sds *MockStateDiffService) process(currentBlock, parentBlock *types.Block) // Subscribe mock method func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Payload, quitChan chan<- bool) { - log.Info("Subscribing to the statediff service") + log.Info("Subscribing to the mock statediff service") sds.Lock() sds.Subscriptions[id] = statediff.Subscription{ PayloadChan: sub, @@ -130,7 +129,7 @@ func (sds *MockStateDiffService) Subscribe(id rpc.ID, sub chan<- statediff.Paylo // Unsubscribe mock method func (sds *MockStateDiffService) Unsubscribe(id rpc.ID) error { - log.Info("Unsubscribing from the statediff service") + log.Info("Unsubscribing from the mock statediff service") sds.Lock() _, ok := sds.Subscriptions[id] if !ok { @@ -170,9 +169,9 @@ func (sds *MockStateDiffService) close() { // Start mock method func (sds *MockStateDiffService) Start(server *p2p.Server) error { - log.Info("Starting statediff service") + log.Info("Starting mock statediff service") if sds.ParentBlockChan == nil || sds.BlockChan == nil { - return errors.New("mock StateDiffingService requires preconfiguration with a MockParentBlockChan and MockBlockChan") + return errors.New("MockStateDiffingService needs to be configured with a MockParentBlockChan and MockBlockChan") } chainEventCh := make(chan core.ChainEvent, 10) go sds.Loop(chainEventCh) @@ -182,7 +181,7 @@ func (sds *MockStateDiffService) Start(server *p2p.Server) error { // Stop mock method func (sds *MockStateDiffService) Stop() error { - log.Info("Stopping statediff service") + log.Info("Stopping mock statediff service") close(sds.QuitChan) return nil } diff --git a/statediff/testhelpers/mocks/api_test.go b/statediff/testhelpers/mocks/api_test.go index 22971b4b6..b76ba4328 100644 --- a/statediff/testhelpers/mocks/api_test.go +++ b/statediff/testhelpers/mocks/api_test.go @@ -136,9 +136,6 @@ func TestAPI(t *testing.T) { if !bytes.Equal(payload.StateDiffRlp, expectedStateDiffBytes) { t.Errorf("payload does not have expected state diff\r\actual state diff rlp: %v\r\nexpected state diff rlp: %v", payload.StateDiffRlp, expectedStateDiffBytes) } - if payload.Err != nil { - t.Errorf("payload should not contain an error, but does: %v", payload.Err) - } case <-quitChan: t.Errorf("channel quit before delivering payload") } diff --git a/statediff/testhelpers/mocks/blockchain.go b/statediff/testhelpers/mocks/blockchain.go index f2c097d38..508435236 100644 --- a/statediff/testhelpers/mocks/blockchain.go +++ b/statediff/testhelpers/mocks/blockchain.go @@ -30,16 +30,20 @@ import ( // BlockChain is a mock blockchain for testing type BlockChain struct { ParentHashesLookedUp []common.Hash - parentBlocksToReturn []*types.Block + parentBlocksToReturn map[common.Hash]*types.Block callCount int ChainEvents []core.ChainEvent + Receipts map[common.Hash]types.Receipts } // AddToStateDiffProcessedCollection mock method func (blockChain *BlockChain) AddToStateDiffProcessedCollection(hash common.Hash) {} // SetParentBlocksToReturn mock method -func (blockChain *BlockChain) SetParentBlocksToReturn(blocks []*types.Block) { +func (blockChain *BlockChain) SetParentBlocksToReturn(blocks map[common.Hash]*types.Block) { + if blockChain.parentBlocksToReturn == nil { + blockChain.parentBlocksToReturn = make(map[common.Hash]*types.Block) + } blockChain.parentBlocksToReturn = blocks } @@ -49,10 +53,9 @@ func (blockChain *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { var parentBlock *types.Block if len(blockChain.parentBlocksToReturn) > 0 { - parentBlock = blockChain.parentBlocksToReturn[blockChain.callCount] + parentBlock = blockChain.parentBlocksToReturn[hash] } - blockChain.callCount++ return parentBlock } @@ -84,3 +87,16 @@ func (blockChain *BlockChain) SubscribeChainEvent(ch chan<- core.ChainEvent) eve return subscription } + +// SetReceiptsForHash mock method +func (blockChain *BlockChain) SetReceiptsForHash(hash common.Hash, receipts types.Receipts) { + if blockChain.Receipts == nil { + blockChain.Receipts = make(map[common.Hash]types.Receipts) + } + blockChain.Receipts[hash] = receipts +} + +// GetReceiptsByHash mock method +func (blockChain *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { + return blockChain.Receipts[hash] +} diff --git a/statediff/types.go b/statediff/types.go index 6df398a10..3bfb598a5 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -34,17 +34,38 @@ type Subscription struct { QuitChan chan<- bool } -// Payload packages the data to send to StateDiffingService subscriptions +// Payload packages the data to send to statediff subscriptions type Payload struct { - BlockRlp []byte `json:"blockRlp" gencodec:"required"` + BlockRlp []byte `json:"blockRlp"` + ReceiptsRlp []byte `json:"receiptsRlp"` StateDiffRlp []byte `json:"stateDiff" gencodec:"required"` - Err error `json:"error"` + + encoded []byte + err error +} + +func (sd *Payload) ensureEncoded() { + if sd.encoded == nil && sd.err == nil { + sd.encoded, sd.err = json.Marshal(sd) + } +} + +// Length to implement Encoder interface for Payload +func (sd *Payload) Length() int { + sd.ensureEncoded() + return len(sd.encoded) +} + +// Encode to implement Encoder interface for Payload +func (sd *Payload) Encode() ([]byte, error) { + sd.ensureEncoded() + return sd.encoded, sd.err } // StateDiff is the final output structure from the builder type StateDiff struct { - BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` + BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` CreatedAccounts []AccountDiff `json:"createdAccounts" gencodec:"required"` DeletedAccounts []AccountDiff `json:"deletedAccounts" gencodec:"required"` UpdatedAccounts []AccountDiff `json:"updatedAccounts" gencodec:"required"` @@ -53,24 +74,6 @@ type StateDiff struct { err error } -func (sd *StateDiff) ensureEncoded() { - if sd.encoded == nil && sd.err == nil { - sd.encoded, sd.err = json.Marshal(sd) - } -} - -// Length to implement Encoder interface for StateDiff -func (sd *StateDiff) Length() int { - sd.ensureEncoded() - return len(sd.encoded) -} - -// Encode to implement Encoder interface for StateDiff -func (sd *StateDiff) Encode() ([]byte, error) { - sd.ensureEncoded() - return sd.encoded, sd.err -} - // AccountDiff holds the data for a single state diff node type AccountDiff struct { Leaf bool `json:"leaf" gencodec:"required"` -- 2.45.2