From 17d92233d9e64b642fed9a992556f7ff7d6fda18 Mon Sep 17 00:00:00 2001 From: Nick Johnson Date: Tue, 17 Jan 2017 11:19:50 +0000 Subject: [PATCH] cmd/geth, core: add support for recording SHA3 preimages (#3543) --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 8 ++++- core/bench_test.go | 5 +-- core/block_validator_test.go | 3 +- core/blockchain.go | 10 ++++-- core/blockchain_test.go | 26 +++++++------- core/chain_makers.go | 2 +- core/chain_makers_test.go | 3 +- core/dao_test.go | 13 +++---- core/database_util.go | 45 +++++++++++++++++++++---- core/state/journal.go | 7 ++++ core/state/statedb.go | 26 +++++++++++++- core/state_processor.go | 2 +- core/vm/instructions.go | 7 +++- core/vm/interface.go | 1 + core/vm/noop.go | 1 + core/vm/vm.go | 2 ++ eth/api.go | 6 ++++ eth/backend.go | 6 ++-- eth/handler_test.go | 3 +- eth/helper_test.go | 3 +- internal/web3ext/web3ext.go | 6 ++++ les/helper_test.go | 3 +- light/odr_test.go | 2 +- light/txpool_test.go | 3 +- light/vm_env.go | 2 ++ tests/block_test_util.go | 3 +- 29 files changed, 156 insertions(+), 46 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 7bff1a125..c26662ea1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -61,7 +61,7 @@ type SimulatedBackend struct { func NewSimulatedBackend(accounts ...core.GenesisAccount) *SimulatedBackend { database, _ := ethdb.NewMemDatabase() core.WriteGenesisBlockForTesting(database, accounts...) - blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux)) + blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux), vm.Config{}) backend := &SimulatedBackend{database: database, blockchain: blockchain} backend.rollback() return backend diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a6b6331df..ac34d3035 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -132,6 +132,7 @@ func init() { utils.VMForceJitFlag, utils.VMJitCacheFlag, utils.VMEnableJitFlag, + utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.RPCCORSDomainFlag, utils.EthStatsURLFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 853307604..5ebfc8919 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -155,6 +155,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.VMEnableJitFlag, utils.VMForceJitFlag, utils.VMJitCacheFlag, + utils.VMEnableDebugFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 01114a957..08b4db578 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" @@ -230,6 +231,10 @@ var ( Name: "jitvm", Usage: "Enable the JIT VM", } + VMEnableDebugFlag = cli.BoolFlag{ + Name: "vmdebug", + Usage: "Record information useful for VM and contract debugging", + } // Logging and debug settings EthStatsURLFlag = cli.StringFlag{ Name: "ethstats", @@ -741,6 +746,7 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) { GpobaseCorrectionFactor: ctx.GlobalInt(GpobaseCorrectionFactorFlag.Name), SolcPath: ctx.GlobalString(SolcPathFlag.Name), AutoDAG: ctx.GlobalBool(AutoDAGFlag.Name) || ctx.GlobalBool(MiningEnabledFlag.Name), + EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name), } // Override any default configs in dev mode or the test net @@ -912,7 +918,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai if !ctx.GlobalBool(FakePoWFlag.Name) { pow = ethash.New() } - chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux)) + chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux), vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}) if err != nil { Fatalf("Could not start chainmanager: %v", err) } diff --git a/core/bench_test.go b/core/bench_test.go index 5785748a1..353d217fd 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -168,7 +169,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. evmux := new(event.TypeMux) - chainman, _ := NewBlockChain(db, ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, FakePow{}, evmux) + chainman, _ := NewBlockChain(db, ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, FakePow{}, evmux, vm.Config{}) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() @@ -278,7 +279,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux)) + chain, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 413c3cc8e..01931efd2 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -39,7 +40,7 @@ func proc() (Validator, *BlockChain) { var mux event.TypeMux WriteTestNetGenesisBlock(db) - blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &mux) + blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &mux, vm.Config{}) if err != nil { fmt.Println(err) } diff --git a/core/blockchain.go b/core/blockchain.go index 6462c17fa..2e522d97c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -107,12 +107,13 @@ type BlockChain struct { pow pow.PoW processor Processor // block processor interface validator Validator // block and state validator interface + vmConfig vm.Config } // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialiser the default Ethereum Validator and // Processor. -func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux) (*BlockChain, error) { +func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux, vmConfig vm.Config) (*BlockChain, error) { bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) blockCache, _ := lru.New(blockCacheLimit) @@ -128,6 +129,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.P blockCache: blockCache, futureBlocks: futureBlocks, pow: pow, + vmConfig: vmConfig, } bc.SetValidator(NewBlockValidator(config, bc, pow)) bc.SetProcessor(NewStateProcessor(config, bc)) @@ -954,7 +956,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { return i, err } // Process block using the parent state as reference point. - receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, vm.Config{}) + receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, self.vmConfig) if err != nil { self.reportBlock(block, receipts, err) return i, err @@ -1004,6 +1006,10 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil { return i, err } + // Write hash preimages + if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil { + return i, err + } case SideStatTy: if glog.V(logger.Detail) { glog.Infof("inserted forked block #%d [%x…] (TD=%v) in %9v: %3d txs %d uncles.", block.Number(), block.Hash().Bytes()[0:4], block.Difficulty(), common.PrettyDuration(time.Since(bstart)), len(block.Transactions()), len(block.Uncles())) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index a5a83ba60..8f1383acd 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -53,7 +53,7 @@ func thePow() pow.PoW { func theBlockChain(db ethdb.Database, t *testing.T) *BlockChain { var eventMux event.TypeMux WriteTestNetGenesisBlock(db) - blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &eventMux) + blockchain, err := NewBlockChain(db, testChainConfig(), thePow(), &eventMux, vm.Config{}) if err != nil { t.Error("failed creating blockchain:", err) t.FailNow() @@ -614,7 +614,7 @@ func testReorgBadHashes(t *testing.T, full bool) { defer func() { delete(BadHashes, headers[3].Hash()) }() } // Create a new chain manager and check it rolled back the state - ncm, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux)) + ncm, err := NewBlockChain(db, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -735,7 +735,7 @@ func TestFastVsFullChains(t *testing.T) { archiveDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds}) - archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) @@ -743,7 +743,7 @@ func TestFastVsFullChains(t *testing.T) { // Fast import the chain as a non-archive node to test fastDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds}) - fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -819,7 +819,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds}) - archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + archive, _ := NewBlockChain(archiveDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) @@ -831,7 +831,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds}) - fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + fast, _ := NewBlockChain(fastDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -850,7 +850,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(lightDb, GenesisAccount{address, funds}) - light, _ := NewBlockChain(lightDb, testChainConfig(), FakePow{}, new(event.TypeMux)) + light, _ := NewBlockChain(lightDb, testChainConfig(), FakePow{}, new(event.TypeMux), vm.Config{}) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) @@ -916,7 +916,7 @@ func TestChainTxReorgs(t *testing.T) { }) // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -990,7 +990,7 @@ func TestLogReorgs(t *testing.T) { ) evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) subs := evmux.Subscribe(RemovedLogsEvent{}) chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 2, func(i int, gen *BlockGen) { @@ -1027,7 +1027,7 @@ func TestReorgSideEvent(t *testing.T) { ) evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 3, func(i int, gen *BlockGen) {}) if _, err := blockchain.InsertChain(chain); err != nil { @@ -1103,7 +1103,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { ) evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, testChainConfig(), FakePow{}, evmux, vm.Config{}) chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *BlockGen) {}) @@ -1146,7 +1146,7 @@ func TestEIP155Transition(t *testing.T) { mux event.TypeMux ) - blockchain, _ := NewBlockChain(db, config, FakePow{}, &mux) + blockchain, _ := NewBlockChain(db, config, FakePow{}, &mux, vm.Config{}) blocks, _ := GenerateChain(config, genesis, db, 4, func(i int, block *BlockGen) { var ( tx *types.Transaction @@ -1250,7 +1250,7 @@ func TestEIP161AccountRemoval(t *testing.T) { } mux event.TypeMux - blockchain, _ = NewBlockChain(db, config, FakePow{}, &mux) + blockchain, _ = NewBlockChain(db, config, FakePow{}, &mux, vm.Config{}) ) blocks, _ := GenerateChain(config, genesis, db, 3, func(i int, block *BlockGen) { var ( diff --git a/core/chain_makers.go b/core/chain_makers.go index 4a838a5aa..8b3b015a8 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -256,7 +256,7 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockChain, error) { // Initialize a fresh chain with only a genesis block genesis, _ := WriteTestNetGenesisBlock(db) - blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux, vm.Config{}) // Create and inject the requested chain if n == 0 { return db, blockchain, nil diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 942f4ace2..2796817c0 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -21,6 +21,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -81,7 +82,7 @@ func ExampleGenerateChain() { // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - blockchain, _ := NewBlockChain(db, chainConfig, FakePow{}, evmux) + blockchain, _ := NewBlockChain(db, chainConfig, FakePow{}, evmux, vm.Config{}) if i, err := blockchain.InsertChain(chain); err != nil { fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err) return diff --git a/core/dao_test.go b/core/dao_test.go index f461131f4..b8b4c71cf 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -20,6 +20,7 @@ import ( "math/big" "testing" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -39,12 +40,12 @@ func TestDAOForkRangeExtradata(t *testing.T) { proDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(proDb) proConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true} - proBc, _ := NewBlockChain(proDb, proConf, new(FakePow), new(event.TypeMux)) + proBc, _ := NewBlockChain(proDb, proConf, new(FakePow), new(event.TypeMux), vm.Config{}) conDb, _ := ethdb.NewMemDatabase() WriteGenesisBlockForTesting(conDb) conConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false} - conBc, _ := NewBlockChain(conDb, conConf, new(FakePow), new(event.TypeMux)) + conBc, _ := NewBlockChain(conDb, conConf, new(FakePow), new(event.TypeMux), vm.Config{}) if _, err := proBc.InsertChain(prefix); err != nil { t.Fatalf("pro-fork: failed to import chain prefix: %v", err) @@ -57,7 +58,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux)) + bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { @@ -78,7 +79,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a no-fork block, and try to feed into the pro-fork chain db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux)) + bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { @@ -100,7 +101,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux)) + bc, _ := NewBlockChain(db, conConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { @@ -116,7 +117,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() WriteGenesisBlockForTesting(db) - bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux)) + bc, _ = NewBlockChain(db, proConf, new(FakePow), new(event.TypeMux), vm.Config{}) blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1)) for j := 0; j < len(blocks)/2; j++ { diff --git a/core/database_util.go b/core/database_util.go index 2060b8b6a..229f21b5b 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -39,12 +40,13 @@ var ( headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") - headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header - tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td - numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash - blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) - bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body - blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header + tdSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + tdSuffix -> td + numSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + numSuffix -> hash + blockHashPrefix = []byte("H") // blockHashPrefix + hash -> num (uint64 big endian) + bodyPrefix = []byte("b") // bodyPrefix + num (uint64 big endian) + hash -> block body + blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage txMetaSuffix = []byte{0x01} receiptsPrefix = []byte("receipts-") @@ -66,6 +68,9 @@ var ( ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms + + preimageCounter = metrics.NewCounter("db/preimage/total") + preimageHitCounter = metrics.NewCounter("db/preimage/hits") ) // encodeBlockNumber encodes a block number as big endian uint64 @@ -595,6 +600,34 @@ func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom { return types.BytesToBloom(bloomDat) } +// PreimageTable returns a Database instance with the key prefix for preimage entries. +func PreimageTable(db ethdb.Database) ethdb.Database { + return ethdb.NewTable(db, preimagePrefix) +} + +// WritePreimages writes the provided set of preimages to the database. `number` is the +// current block number, and is used for debug messages only. +func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash][]byte) error { + table := PreimageTable(db) + batch := table.NewBatch() + hitCount := 0 + for hash, preimage := range preimages { + if _, err := table.Get(hash.Bytes()); err != nil { + batch.Put(hash.Bytes(), preimage) + hitCount += 1 + } + } + preimageCounter.Inc(int64(len(preimages))) + preimageHitCounter.Inc(int64(hitCount)) + if hitCount > 0 { + if err := batch.Write(); err != nil { + return fmt.Errorf("preimage write fail for block %d: %v", number, err) + } + glog.V(logger.Debug).Infof("%d preimages in block %d, including %d new", len(preimages), number, hitCount) + } + return nil +} + // GetBlockChainVersion reads the version number from db. func GetBlockChainVersion(db ethdb.Database) int { var vsn uint diff --git a/core/state/journal.go b/core/state/journal.go index d1e73e7d0..68d07fa03 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -67,6 +67,9 @@ type ( addLogChange struct { txhash common.Hash } + addPreimageChange struct { + hash common.Hash + } touchChange struct { account *common.Address prev bool @@ -127,3 +130,7 @@ func (ch addLogChange) undo(s *StateDB) { s.logs[ch.txhash] = logs[:len(logs)-1] } } + +func (ch addPreimageChange) undo(s *StateDB) { + delete(s.preimages, ch.hash) +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 063e2b469..bbccba9fb 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -75,6 +75,8 @@ type StateDB struct { logs map[common.Hash][]*types.Log logSize uint + preimages map[common.Hash][]byte + // Journal of state modifications. This is the backbone of // Snapshot and RevertToSnapshot. journal journal @@ -99,6 +101,7 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { stateObjectsDirty: make(map[common.Address]struct{}), refund: new(big.Int), logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), }, nil } @@ -120,6 +123,7 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) { stateObjectsDirty: make(map[common.Address]struct{}), refund: new(big.Int), logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), }, nil } @@ -141,6 +145,7 @@ func (self *StateDB) Reset(root common.Hash) error { self.txIndex = 0 self.logs = make(map[common.Hash][]*types.Log) self.logSize = 0 + self.preimages = make(map[common.Hash][]byte) self.clearJournalAndRefund() return nil @@ -199,6 +204,21 @@ func (self *StateDB) Logs() []*types.Log { return logs } +// AddPreimage records a SHA3 preimage seen by the VM. +func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) { + if _, ok := self.preimages[hash]; !ok { + self.journal = append(self.journal, addPreimageChange{hash: hash}) + pi := make([]byte, len(preimage)) + copy(pi, preimage) + self.preimages[hash] = pi + } +} + +// Preimages returns a list of SHA3 preimages that have been submitted. +func (self *StateDB) Preimages() map[common.Hash][]byte { + return self.preimages +} + func (self *StateDB) AddRefund(gas *big.Int) { self.journal = append(self.journal, refundChange{prev: new(big.Int).Set(self.refund)}) self.refund.Add(self.refund, gas) @@ -477,8 +497,9 @@ func (self *StateDB) Copy() *StateDB { refund: new(big.Int).Set(self.refund), logs: make(map[common.Hash][]*types.Log, len(self.logs)), logSize: self.logSize, + preimages: make(map[common.Hash][]byte), } - // Copy the dirty states and logs + // Copy the dirty states, logs, and preimages for addr := range self.stateObjectsDirty { state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty) state.stateObjectsDirty[addr] = struct{}{} @@ -487,6 +508,9 @@ func (self *StateDB) Copy() *StateDB { state.logs[hash] = make([]*types.Log, len(logs)) copy(state.logs[hash], logs) } + for hash, preimage := range self.preimages { + state.preimages[hash] = preimage + } return state } diff --git a/core/state_processor.go b/core/state_processor.go index 4f6ca651e..6485e9abd 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -99,7 +99,7 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, gp *GasPool, s context := NewEVMContext(msg, header, bc) // Create a new environment which holds all relevant information // about the transaction and calling mechanisms. - vmenv := vm.NewEVM(context, statedb, config, vm.Config{}) + vmenv := vm.NewEVM(context, statedb, config, cfg) // Apply the transaction to the current state (included in the env) _, gas, err := ApplyMessage(vmenv, msg, gp) if err != nil { diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 5bfa73a30..3b1b06cca 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -247,7 +247,12 @@ func opMulmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *S func opSha3(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() - hash := crypto.Keccak256(memory.Get(offset.Int64(), size.Int64())) + data := memory.Get(offset.Int64(), size.Int64()) + hash := crypto.Keccak256(data) + + if env.vmConfig.EnablePreimageRecording { + env.StateDB.AddPreimage(common.BytesToHash(hash), data) + } stack.push(common.BytesToBig(hash)) return nil, nil diff --git a/core/vm/interface.go b/core/vm/interface.go index 8617b2d0f..6f15112ee 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -60,6 +60,7 @@ type StateDB interface { Snapshot() int AddLog(*types.Log) + AddPreimage(common.Hash, []byte) } // Account represents a contract or basic ethereum account. diff --git a/core/vm/noop.go b/core/vm/noop.go index ef6837273..7835eeaf3 100644 --- a/core/vm/noop.go +++ b/core/vm/noop.go @@ -67,3 +67,4 @@ func (NoopStateDB) Empty(common.Address) bool { return f func (NoopStateDB) RevertToSnapshot(int) {} func (NoopStateDB) Snapshot() int { return 0 } func (NoopStateDB) AddLog(*types.Log) {} +func (NoopStateDB) AddPreimage(common.Hash, []byte) {} diff --git a/core/vm/vm.go b/core/vm/vm.go index 56081f12c..a5f48750d 100644 --- a/core/vm/vm.go +++ b/core/vm/vm.go @@ -44,6 +44,8 @@ type Config struct { NoRecursion bool // Disable gas metering DisableGasMetering bool + // Enable recording of SHA3/keccak preimages + EnablePreimageRecording bool // JumpTable contains the EVM instruction table. This // may me left uninitialised and will be set the default // table. diff --git a/eth/api.go b/eth/api.go index 023e9a9bb..07df0b79e 100644 --- a/eth/api.go +++ b/eth/api.go @@ -560,3 +560,9 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common. } return nil, errors.New("database inconsistency") } + +// Preimage is a debug API function that returns the preimage for a sha3 hash, if known. +func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { + db := core.PreimageTable(api.eth.ChainDb()) + return db.Get(hash.Bytes()) +} diff --git a/eth/backend.go b/eth/backend.go index dec8c0c6e..02514c25b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -97,8 +98,7 @@ type Config struct { GpobaseStepUp int GpobaseCorrectionFactor int - EnableJit bool - ForceJit bool + EnablePreimageRecording bool TestGenesisBlock *types.Block // Genesis block to seed the chain database with (testing only!) TestGenesisState ethdb.Database // Genesis state to seed the database with (testing only!) @@ -218,7 +218,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { glog.V(logger.Info).Infoln("Chain config:", eth.chainConfig) - eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.pow, eth.EventMux()) + eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.pow, eth.EventMux(), vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}) if err != nil { if err == core.ErrNoGenesis { return nil, fmt.Errorf(`No chain found. Please initialise a new chain using the "init" subcommand.`) diff --git a/eth/handler_test.go b/eth/handler_test.go index 22a4ddf50..8a5d7173b 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" @@ -469,7 +470,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool db, _ = ethdb.NewMemDatabase() genesis = core.WriteGenesisBlockForTesting(db) config = ¶ms.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked} - blockchain, _ = core.NewBlockChain(db, config, pow, evmux) + blockchain, _ = core.NewBlockChain(db, config, pow, evmux, vm.Config{}) ) pm, err := NewProtocolManager(config, false, NetworkId, 1000, evmux, new(testTxPool), pow, blockchain, db) if err != nil { diff --git a/eth/helper_test.go b/eth/helper_test.go index a718a6d21..0861c884b 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -56,7 +57,7 @@ func newTestProtocolManager(fastSync bool, blocks int, generator func(int, *core db, _ = ethdb.NewMemDatabase() genesis = core.WriteGenesisBlockForTesting(db, testBank) chainConfig = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0)} // homestead set to 0 because of chain maker - blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux) + blockchain, _ = core.NewBlockChain(db, chainConfig, pow, evmux, vm.Config{}) ) chain, _ := core.GenerateChain(chainConfig, genesis, db, blocks, generator) if _, err := blockchain.InsertChain(chain); err != nil { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 12bdb6996..a265d2406 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -385,6 +385,12 @@ web3._extend({ call: 'debug_traceTransaction', params: 2, inputFormatter: [null, null] + }), + new web3._extend.Method({ + name: 'preimage', + call: 'debug_preimage', + params: 1, + inputFormatter: [null] }) ], properties: [] diff --git a/les/helper_test.go b/les/helper_test.go index 3d6bf3c29..e0b7558ee 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -143,7 +144,7 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor odr = NewLesOdr(db) chain, _ = light.NewLightChain(odr, chainConfig, pow, evmux) } else { - blockchain, _ := core.NewBlockChain(db, chainConfig, pow, evmux) + blockchain, _ := core.NewBlockChain(db, chainConfig, pow, evmux, vm.Config{}) gchain, _ := core.GenerateChain(chainConfig, genesis, db, blocks, generator) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) diff --git a/light/odr_test.go b/light/odr_test.go index 2dcfa40d9..a2f969acb 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -251,7 +251,7 @@ func testChainOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { ) core.WriteGenesisBlockForTesting(ldb, core.GenesisAccount{Address: testBankAddress, Balance: testBankFunds}) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux) + blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux, vm.Config{}) chainConfig := ¶ms.ChainConfig{HomesteadBlock: new(big.Int)} gchain, _ := core.GenerateChain(chainConfig, genesis, sdb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { diff --git a/light/txpool_test.go b/light/txpool_test.go index e5a4670aa..d8f04e617 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -88,7 +89,7 @@ func TestTxPool(t *testing.T) { ) core.WriteGenesisBlockForTesting(ldb, core.GenesisAccount{Address: testBankAddress, Balance: testBankFunds}) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux) + blockchain, _ := core.NewBlockChain(sdb, testChainConfig(), pow, evmux, vm.Config{}) chainConfig := ¶ms.ChainConfig{HomesteadBlock: new(big.Int)} gchain, _ := core.GenerateChain(chainConfig, genesis, sdb, poolTestBlocks, txPoolTestChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { diff --git a/light/vm_env.go b/light/vm_env.go index 1b225c8de..d2cc7e960 100644 --- a/light/vm_env.go +++ b/light/vm_env.go @@ -45,6 +45,8 @@ func (s *VMState) Error() error { func (s *VMState) AddLog(log *types.Log) {} +func (s *VMState) AddPreimage(hash common.Hash, preimage []byte) {} + // errHandler handles and stores any state error that happens during execution. func (s *VMState) errHandler(err error) { if err != nil && s.err == nil { diff --git a/tests/block_test_util.go b/tests/block_test_util.go index ea63c9996..470eb7cb7 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -172,7 +173,7 @@ func runBlockTest(homesteadBlock, daoForkBlock, gasPriceFork *big.Int, test *Blo core.WriteHeadBlockHash(db, test.Genesis.Hash()) evmux := new(event.TypeMux) config := ¶ms.ChainConfig{HomesteadBlock: homesteadBlock, DAOForkBlock: daoForkBlock, DAOForkSupport: true, EIP150Block: gasPriceFork} - chain, err := core.NewBlockChain(db, config, ethash.NewShared(), evmux) + chain, err := core.NewBlockChain(db, config, ethash.NewShared(), evmux, vm.Config{}) if err != nil { return err }