From 1da5e0ebb0bf702f1e4d811b9dab9b3f4589e9e5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 24 Jun 2019 16:16:44 +0200 Subject: [PATCH] core/state, cmd/geth: streaming json output for dump command (#15475) * core/state, cmd/geth: streaming json output dump cmd + optional code+storage * dump: add option to continue even if preimages are missing * core, evm: lint nits * cmd: use local flags for dump, omit empty code/storage * core/state: fix state dump test --- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 2 +- cmd/geth/chaincmd.go | 21 ++++++- cmd/utils/flags.go | 16 +++++ core/state/dump.go | 124 +++++++++++++++++++++++++++++++-------- core/state/state_test.go | 19 +++--- eth/api.go | 4 +- 7 files changed, 143 insertions(+), 45 deletions(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index bc5d00cfb..08f36dadc 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -209,7 +209,7 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump())) + fmt.Println(string(statedb.Dump(false, false, true))) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index b3c69d9b9..cef2aedb5 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() if ctx.GlobalBool(DumpFlag.Name) && state != nil { - dump := state.RawDump() + dump := state.RawDump(false, false, true) result.State = &dump } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c91545c7f..49e6a0594 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -162,6 +162,10 @@ Remove blockchain and state databases`, utils.DataDirFlag, utils.CacheFlag, utils.SyncModeFlag, + utils.IterativeOutputFlag, + utils.ExcludeCodeFlag, + utils.ExcludeStorageFlag, + utils.IncludeIncompletesFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -287,7 +291,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000) fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs)) - if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) { + if ctx.GlobalBool(utils.NoCompactionFlag.Name) { return nil } @@ -504,6 +508,7 @@ func dump(ctx *cli.Context) error { defer stack.Close() chain, chainDb := utils.MakeChain(ctx, stack) + defer chainDb.Close() for _, arg := range ctx.Args() { var block *types.Block if hashish(arg) { @@ -520,10 +525,20 @@ func dump(ctx *cli.Context) error { if err != nil { utils.Fatalf("could not create new state: %v", err) } - fmt.Printf("%s\n", state.Dump()) + excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name) + excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name) + includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name) + if ctx.Bool(utils.IterativeOutputFlag.Name) { + state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout)) + } else { + if includeMissing { + fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" + + " otherwise the accounts will overwrite each other in the resulting mapping.") + } + fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false)) + } } } - chainDb.Close() return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8c5a0c99a..8d94539ab 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -196,6 +196,22 @@ var ( Name: "ulc.trusted", Usage: "List of trusted ULC servers", } + IterativeOutputFlag = cli.BoolFlag{ + Name: "iterative", + Usage: "Print streaming JSON iteratively, delimited by newlines", + } + ExcludeStorageFlag = cli.BoolFlag{ + Name: "nostorage", + Usage: "Exclude storage entries (save db lookups)", + } + IncludeIncompletesFlag = cli.BoolFlag{ + Name: "incompletes", + Usage: "Include accounts for which we don't have the address (missing preimage)", + } + ExcludeCodeFlag = cli.BoolFlag{ + Name: "nocode", + Usage: "Exclude contract code (save db lookups)", + } defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", diff --git a/core/state/dump.go b/core/state/dump.go index 072dbbf05..51d3e5554 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -21,61 +21,133 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) +// DumpAccount represents an account in the state type DumpAccount struct { - Balance string `json:"balance"` - Nonce uint64 `json:"nonce"` - Root string `json:"root"` - CodeHash string `json:"codeHash"` - Code string `json:"code"` - Storage map[string]string `json:"storage"` + Balance string `json:"balance"` + Nonce uint64 `json:"nonce"` + Root string `json:"root"` + CodeHash string `json:"codeHash"` + Code string `json:"code,omitempty"` + Storage map[common.Hash]string `json:"storage,omitempty"` + Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode + SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key + } +// Dump represents the full dump in a collected format, as one large map type Dump struct { - Root string `json:"root"` - Accounts map[string]DumpAccount `json:"accounts"` + Root string `json:"root"` + Accounts map[common.Address]DumpAccount `json:"accounts"` } -func (self *StateDB) RawDump() Dump { - dump := Dump{ - Root: fmt.Sprintf("%x", self.trie.Hash()), - Accounts: make(map[string]DumpAccount), - } +// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively +type iterativeDump json.Encoder +// Collector interface which the state trie calls during iteration +type collector interface { + onRoot(common.Hash) + onAccount(common.Address, DumpAccount) +} + +func (self *Dump) onRoot(root common.Hash) { + self.Root = fmt.Sprintf("%x", root) +} + +func (self *Dump) onAccount(addr common.Address, account DumpAccount) { + self.Accounts[addr] = account +} + +func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) { + dumpAccount := &DumpAccount{ + Balance: account.Balance, + Nonce: account.Nonce, + Root: account.Root, + CodeHash: account.CodeHash, + Code: account.Code, + Storage: account.Storage, + SecureKey: account.SecureKey, + Address: nil, + } + if addr != (common.Address{}) { + dumpAccount.Address = &addr + } + (*json.Encoder)(&self).Encode(dumpAccount) +} +func (self iterativeDump) onRoot(root common.Hash) { + (*json.Encoder)(&self).Encode(struct { + Root common.Hash `json:"root"` + }{root}) +} + +func (self *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) { + emptyAddress := (common.Address{}) + missingPreimages := 0 + c.onRoot(self.trie.Hash()) it := trie.NewIterator(self.trie.NodeIterator(nil)) for it.Next() { - addr := self.trie.GetKey(it.Key) var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } - - obj := newObject(nil, common.BytesToAddress(addr), data) + addr := common.BytesToAddress(self.trie.GetKey(it.Key)) + obj := newObject(nil, addr, data) account := DumpAccount{ Balance: data.Balance.String(), Nonce: data.Nonce, Root: common.Bytes2Hex(data.Root[:]), CodeHash: common.Bytes2Hex(data.CodeHash), - Code: common.Bytes2Hex(obj.Code(self.db)), - Storage: make(map[string]string), } - storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil)) - for storageIt.Next() { - account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) + if emptyAddress == addr { + // Preimage missing + missingPreimages++ + if excludeMissingPreimages { + continue + } + account.SecureKey = it.Key } - dump.Accounts[common.Bytes2Hex(addr)] = account + if !excludeCode { + account.Code = common.Bytes2Hex(obj.Code(self.db)) + } + if !excludeStorage { + account.Storage = make(map[common.Hash]string) + storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil)) + for storageIt.Next() { + account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) + } + } + c.onAccount(addr, account) + } + if missingPreimages > 0 { + log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } - return dump } -func (self *StateDB) Dump() []byte { - json, err := json.MarshalIndent(self.RawDump(), "", " ") +// RawDump returns the entire state an a single large object +func (self *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump { + dump := &Dump{ + Accounts: make(map[common.Address]DumpAccount), + } + self.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages) + return *dump +} + +// Dump returns a JSON string representing the entire state as a single json-object +func (self *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte { + dump := self.RawDump(excludeCode, excludeStorage, excludeMissingPreimages) + json, err := json.MarshalIndent(dump, "", " ") if err != nil { fmt.Println("dump err", err) } - return json } + +// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout +func (self *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { + self.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages) +} diff --git a/core/state/state_test.go b/core/state/state_test.go index 606f2a6f6..d6ff714ee 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -52,33 +52,28 @@ func (s *StateSuite) TestDump(c *checker.C) { s.state.Commit(false) // check that dump contains the state objects that are in trie - got := string(s.state.Dump()) + got := string(s.state.Dump(false, false, true)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { - "0000000000000000000000000000000000000001": { + "0x0000000000000000000000000000000000000001": { "balance": "22", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - "code": "", - "storage": {} + "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" }, - "0000000000000000000000000000000000000002": { + "0x0000000000000000000000000000000000000002": { "balance": "44", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - "code": "", - "storage": {} + "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" }, - "0000000000000000000000000000000000000102": { + "0x0000000000000000000000000000000000000102": { "balance": "0", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", - "code": "03030303030303", - "storage": {} + "code": "03030303030303" } } }` diff --git a/eth/api.go b/eth/api.go index bdbbd1ba3..8afa21a38 100644 --- a/eth/api.go +++ b/eth/api.go @@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error // both the pending block as well as the pending state from // the miner and operate on those _, stateDb := api.eth.miner.Pending() - return stateDb.RawDump(), nil + return stateDb.RawDump(false, false, true), nil } var block *types.Block if blockNr == rpc.LatestBlockNumber { @@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error if err != nil { return state.Dump{}, err } - return stateDb.RawDump(), nil + return stateDb.RawDump(false, false, true), nil } // PrivateDebugAPI is the collection of Ethereum full node APIs exposed over