Merge Geth v1.10.21 as well as update to Plugeth-Utils v0.0.18

core/asm/asm.go

@@ -109,7 +109,7 @@ func PrintDisassembled(code string) error {
 	it := NewInstructionIterator(script)
 	for it.Next() {
 		if it.Arg() != nil && 0 < len(it.Arg()) {
-			fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())
+			fmt.Printf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg())
 		} else {
 			fmt.Printf("%05x: %v\n", it.PC(), it.Op())
 		}
@@ -124,7 +124,7 @@ func Disassemble(script []byte) ([]string, error) {
 	it := NewInstructionIterator(script)
 	for it.Next() {
 		if it.Arg() != nil && 0 < len(it.Arg()) {
-			instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()))
+			instrs = append(instrs, fmt.Sprintf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg()))
 		} else {
 			instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op()))
 		}

core/beacon/types.go

@@ -42,7 +42,7 @@ type payloadAttributesMarshaling struct {
 
 //go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go
 
-// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md
+// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/tree/main/src/engine/specification.md
 type ExecutableDataV1 struct {
 	ParentHash    common.Hash    `json:"parentHash"    gencodec:"required"`
 	FeeRecipient  common.Address `json:"feeRecipient"  gencodec:"required"`
@@ -148,6 +148,13 @@ func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) {
 	if len(params.ExtraData) > 32 {
 		return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData))
 	}
+	if len(params.LogsBloom) != 256 {
+		return nil, fmt.Errorf("invalid logsBloom length: %v", len(params.LogsBloom))
+	}
+	// Check that baseFeePerGas is not negative or too big
+	if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) {
+		return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas)
+	}
 	header := &types.Header{
 		ParentHash:  params.ParentHash,
 		UncleHash:   types.EmptyUncleHash,

core/block_validator_test.go

@@ -107,7 +107,8 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
 			Alloc: map[common.Address]GenesisAccount{
 				addr: {Balance: big.NewInt(1)},
 			},
-			BaseFee: big.NewInt(params.InitialBaseFee),
+			BaseFee:    big.NewInt(params.InitialBaseFee),
+			Difficulty: new(big.Int),
 		}
 		copy(genspec.ExtraData[32:], addr[:])
 		genesis := genspec.MustCommit(testdb)

core/blockchain.go

@@ -51,6 +51,7 @@ var (
 	headHeaderGauge         = metrics.NewRegisteredGauge("chain/head/header", nil)
 	headFastBlockGauge      = metrics.NewRegisteredGauge("chain/head/receipt", nil)
 	headFinalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil)
+	headSafeBlockGauge      = metrics.NewRegisteredGauge("chain/head/safe", nil)
 
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)
 	accountHashTimer   = metrics.NewRegisteredTimer("chain/account/hashes", nil)
@@ -191,6 +192,7 @@ type BlockChain struct {
 	currentBlock          atomic.Value // Current head of the block chain
 	currentFastBlock      atomic.Value // Current head of the fast-sync chain (may be above the block chain!)
 	currentFinalizedBlock atomic.Value // Current finalized head
+	currentSafeBlock      atomic.Value // Current safe head
 
 	stateCache    state.Database // State database to reuse between imports (contains state cache)
 	bodyCache     *lru.Cache     // Cache for the most recent block bodies
@@ -267,6 +269,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	bc.currentBlock.Store(nilBlock)
 	bc.currentFastBlock.Store(nilBlock)
 	bc.currentFinalizedBlock.Store(nilBlock)
+	bc.currentSafeBlock.Store(nilBlock)
 
 	// Initialize the chain with ancient data if it isn't empty.
 	var txIndexBlock uint64
@@ -464,11 +467,15 @@ func (bc *BlockChain) loadLastState() error {
 		}
 	}
 
-	// Restore the last known finalized block
+	// Restore the last known finalized block and safe block
+	// Note: the safe block is not stored on disk and it is set to the last
+	// known finalized block on startup
 	if head := rawdb.ReadFinalizedBlockHash(bc.db); head != (common.Hash{}) {
 		if block := bc.GetBlockByHash(head); block != nil {
 			bc.currentFinalizedBlock.Store(block)
 			headFinalizedBlockGauge.Update(int64(block.NumberU64()))
+			bc.currentSafeBlock.Store(block)
+			headSafeBlockGauge.Update(int64(block.NumberU64()))
 		}
 	}
 	// Issue a status log for the user
@@ -504,8 +511,23 @@ func (bc *BlockChain) SetHead(head uint64) error {
 // SetFinalized sets the finalized block.
 func (bc *BlockChain) SetFinalized(block *types.Block) {
 	bc.currentFinalizedBlock.Store(block)
-	rawdb.WriteFinalizedBlockHash(bc.db, block.Hash())
-	headFinalizedBlockGauge.Update(int64(block.NumberU64()))
+	if block != nil {
+		rawdb.WriteFinalizedBlockHash(bc.db, block.Hash())
+		headFinalizedBlockGauge.Update(int64(block.NumberU64()))
+	} else {
+		rawdb.WriteFinalizedBlockHash(bc.db, common.Hash{})
+		headFinalizedBlockGauge.Update(0)
+	}
+}
+
+// SetSafe sets the safe block.
+func (bc *BlockChain) SetSafe(block *types.Block) {
+	bc.currentSafeBlock.Store(block)
+	if block != nil {
+		headSafeBlockGauge.Update(int64(block.NumberU64()))
+	} else {
+		headSafeBlockGauge.Update(0)
+	}
 }
 
 // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition
@@ -663,6 +685,16 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo
 	bc.txLookupCache.Purge()
 	bc.futureBlocks.Purge()
 
+	// Clear safe block, finalized block if needed
+	if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.NumberU64() {
+		log.Warn("SetHead invalidated safe block")
+		bc.SetSafe(nil)
+	}
+	if finalized := bc.CurrentFinalizedBlock(); finalized != nil && head < finalized.NumberU64() {
+		log.Error("SetHead invalidated finalized block")
+		bc.SetFinalized(nil)
+	}
+
 	return rootNumber, bc.loadLastState()
 }
 
@@ -674,7 +706,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x..]", hash[:4])
 	}
-	if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil {
+	if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil {
 		return err
 	}
 
@@ -739,22 +771,25 @@ func (bc *BlockChain) Export(w io.Writer) error {
 
 // ExportN writes a subset of the active chain to the given writer.
 func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
-	if !bc.chainmu.TryLock() {
-		return errChainStopped
-	}
-	defer bc.chainmu.Unlock()
-
 	if first > last {
 		return fmt.Errorf("export failed: first (%d) is greater than last (%d)", first, last)
 	}
 	log.Info("Exporting batch of blocks", "count", last-first+1)
 
-	start, reported := time.Now(), time.Now()
+	var (
+		parentHash common.Hash
+		start      = time.Now()
+		reported   = time.Now()
+	)
 	for nr := first; nr <= last; nr++ {
 		block := bc.GetBlockByNumber(nr)
 		if block == nil {
 			return fmt.Errorf("export failed on #%d: not found", nr)
 		}
+		if nr > first && block.ParentHash() != parentHash {
+			return fmt.Errorf("export failed: chain reorg during export")
+		}
+		parentHash = block.Hash()
 		if err := block.EncodeRLP(w); err != nil {
 			return err
 		}
@@ -1975,8 +2010,8 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		oldChain    types.Blocks
 		commonBlock *types.Block
 
-		deletedTxs types.Transactions
-		addedTxs   types.Transactions
+		deletedTxs []common.Hash
+		addedTxs   []common.Hash
 
 		deletedLogs [][]*types.Log
 		rebirthLogs [][]*types.Log
@@ -1986,7 +2021,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		// Old chain is longer, gather all transactions and logs as deleted ones
 		for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = bc.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) {
 			oldChain = append(oldChain, oldBlock)
-			deletedTxs = append(deletedTxs, oldBlock.Transactions()...)
+			for _, tx := range oldBlock.Transactions() {
+				deletedTxs = append(deletedTxs, tx.Hash())
+			}
 
 			// Collect deleted logs for notification
 			logs := bc.collectLogs(oldBlock.Hash(), true)
@@ -2016,7 +2053,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		}
 		// Remove an old block as well as stash away a new block
 		oldChain = append(oldChain, oldBlock)
-		deletedTxs = append(deletedTxs, oldBlock.Transactions()...)
+		for _, tx := range oldBlock.Transactions() {
+			deletedTxs = append(deletedTxs, tx.Hash())
+		}
 
 		// Collect deleted logs for notification
 		logs := bc.collectLogs(oldBlock.Hash(), true)
@@ -2035,6 +2074,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 			return fmt.Errorf("invalid new chain")
 		}
 	}
+
 	// Ensure the user sees large reorgs
 	if len(oldChain) > 0 && len(newChain) > 0 {
 		logFn := log.Info
@@ -2054,7 +2094,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 	} else if len(newChain) > 0 {
 		// Special case happens in the post merge stage that current head is
 		// the ancestor of new head while these two blocks are not consecutive
-		log.Info("Extend chain", "add", len(newChain), "number", newChain[0].NumberU64(), "hash", newChain[0].Hash())
+		log.Info("Extend chain", "add", len(newChain), "number", newChain[0].Number(), "hash", newChain[0].Hash())
 		blockReorgAddMeter.Mark(int64(len(newChain)))
 	} else {
 		// len(newChain) == 0 && len(oldChain) > 0
@@ -2067,22 +2107,26 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		// Insert the block in the canonical way, re-writing history
 		bc.writeHeadBlock(newChain[i])
 
-		// Collect reborn logs due to chain reorg
-		logs := bc.collectLogs(newChain[i].Hash(), false)
-		if len(logs) > 0 {
-			rebirthLogs = append(rebirthLogs, logs)
-		}
 		// Collect the new added transactions.
-		addedTxs = append(addedTxs, newChain[i].Transactions()...)
+		for _, tx := range newChain[i].Transactions() {
+			addedTxs = append(addedTxs, tx.Hash())
+		}
 	}
+
 	// Delete useless indexes right now which includes the non-canonical
 	// transaction indexes, canonical chain indexes which above the head.
 	indexesBatch := bc.db.NewBatch()
-	for _, tx := range types.TxDifference(deletedTxs, addedTxs) {
-		rawdb.DeleteTxLookupEntry(indexesBatch, tx.Hash())
+	for _, tx := range types.HashDifference(deletedTxs, addedTxs) {
+		rawdb.DeleteTxLookupEntry(indexesBatch, tx)
+	}
+
+	// Delete all hash markers that are not part of the new canonical chain.
+	// Because the reorg function does not handle new chain head, all hash
+	// markers greater than or equal to new chain head should be deleted.
+	number := commonBlock.NumberU64()
+	if len(newChain) > 1 {
+		number = newChain[1].NumberU64()
 	}
-	// Delete any canonical number assignments above the new head
-	number := bc.CurrentBlock().NumberU64()
 	for i := number + 1; ; i++ {
 		hash := rawdb.ReadCanonicalHash(bc.db, i)
 		if hash == (common.Hash{}) {
@@ -2093,6 +2137,15 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 	if err := indexesBatch.Write(); err != nil {
 		log.Crit("Failed to delete useless indexes", "err", err)
 	}
+
+	// Collect the logs
+	for i := len(newChain) - 1; i >= 1; i-- {
+		// Collect reborn logs due to chain reorg
+		logs := bc.collectLogs(newChain[i].Hash(), false)
+		if len(logs) > 0 {
+			rebirthLogs = append(rebirthLogs, logs)
+		}
+	}
 	// If any logs need to be fired, do it now. In theory we could avoid creating
 	// this goroutine if there are no events to fire, but realistcally that only
 	// ever happens if we're reorging empty blocks, which will only happen on idle
@@ -2345,7 +2398,7 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e
 Chain config: %v
 
 Number: %v
-Hash: 0x%x
+Hash: %#x
 %v
 
 Error: %v
@@ -2377,3 +2430,11 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i
 	_, err := bc.hc.InsertHeaderChain(chain, start, bc.forker)
 	return 0, err
 }
+
+// SetBlockValidatorAndProcessorForTesting sets the current validator and processor.
+// This method can be used to force an invalid blockchain to be verified for tests.
+// This method is unsafe and should only be used before block import starts.
+func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Processor) {
+	bc.validator = v
+	bc.processor = p
+}

core/blockchain_reader.go

@@ -55,6 +55,12 @@ func (bc *BlockChain) CurrentFinalizedBlock() *types.Block {
 	return bc.currentFinalizedBlock.Load().(*types.Block)
 }
 
+// CurrentSafeBlock retrieves the current safe block of the canonical
+// chain. The block is retrieved from the blockchain's internal cache.
+func (bc *BlockChain) CurrentSafeBlock() *types.Block {
+	return bc.currentSafeBlock.Load().(*types.Block)
+}
+
 // HasHeader checks if a block header is present in the database or not, caching
 // it if present.
 func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool {

core/blockchain_sethead_test.go

@@ -51,6 +51,7 @@ type rewindTest struct {
 	expHeadBlock       uint64 // Block number of the expected head full block
 }
 
+//nolint:unused
 func (tt *rewindTest) dump(crash bool) string {
 	buffer := new(strings.Builder)
 

core/blockchain_snapshot_test.go

@@ -150,6 +150,7 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [
 	}
 }
 
+//nolint:unused
 func (basic *snapshotTestBasic) dump() string {
 	buffer := new(strings.Builder)
 
@@ -341,54 +342,6 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) {
 	snaptest.verify(t, newchain, blocks)
 }
 
-// restartCrashSnapshotTest is the test type used to test this scenario:
-// - have a complete snapshot
-// - restart chain
-// - insert more blocks with enabling the snapshot
-// - commit the snapshot
-// - crash
-// - restart again
-type restartCrashSnapshotTest struct {
-	snapshotTestBasic
-	newBlocks int
-}
-
-func (snaptest *restartCrashSnapshotTest) test(t *testing.T) {
-	// It's hard to follow the test case, visualize the input
-	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
-	// fmt.Println(tt.dump())
-	chain, blocks := snaptest.prepare(t)
-
-	// Firstly, stop the chain properly, with all snapshot journal
-	// and state committed.
-	chain.Stop()
-
-	newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
-	if err != nil {
-		t.Fatalf("Failed to recreate chain: %v", err)
-	}
-	newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {})
-	newchain.InsertChain(newBlocks)
-
-	// Commit the entire snapshot into the disk if requested. Note only
-	// (a) snapshot root and (b) snapshot generator will be committed,
-	// the diff journal is not.
-	newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0)
-
-	// Simulate the blockchain crash
-	// Don't call chain.Stop here, so that no snapshot
-	// journal and latest state will be committed
-
-	// Restart the chain after the crash
-	newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
-	if err != nil {
-		t.Fatalf("Failed to recreate chain: %v", err)
-	}
-	defer newchain.Stop()
-
-	snaptest.verify(t, newchain, blocks)
-}
-
 // wipeCrashSnapshotTest is the test type used to test this scenario:
 // - have a complete snapshot
 // - restart, insert more blocks without enabling the snapshot
@@ -431,7 +384,7 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
 		SnapshotLimit:  256,
 		SnapshotWait:   false, // Don't wait rebuild
 	}
-	newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
+	_, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}

core/blockchain_test.go

@@ -503,7 +503,7 @@ func TestReorgShortBlocks(t *testing.T)  { testReorgShort(t, true) }
 func testReorgShort(t *testing.T, full bool) {
 	// Create a long easy chain vs. a short heavy one. Due to difficulty adjustment
 	// we need a fairly long chain of blocks with different difficulties for a short
-	// one to become heavyer than a long one. The 96 is an empirical value.
+	// one to become heavier than a long one. The 96 is an empirical value.
 	easy := make([]int64, 96)
 	for i := 0; i < len(easy); i++ {
 		easy[i] = 60
@@ -1235,7 +1235,6 @@ func TestSideLogRebirth(t *testing.T) {
 	chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) {
 		if i == 1 {
 			gen.OffsetTime(-9) // higher block difficulty
-
 		}
 	})
 	if _, err := blockchain.InsertChain(chain); err != nil {
@@ -1364,7 +1363,6 @@ done:
 		t.Errorf("unexpected event fired: %v", e)
 	case <-time.After(250 * time.Millisecond):
 	}
-
 }
 
 // Tests if the canonical block can be fetched from the database during chain insertion.
@@ -2576,6 +2574,7 @@ func TestTransactionIndices(t *testing.T) {
 			t.Fatalf("failed to create temp freezer db: %v", err)
 		}
 		gspec.MustCommit(ancientDb)
+		l := l
 		chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l)
 		if err != nil {
 			t.Fatalf("failed to create tester chain: %v", err)
@@ -2601,6 +2600,7 @@ func TestTransactionIndices(t *testing.T) {
 	limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */}
 	tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0}
 	for i, l := range limit {
+		l := l
 		chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l)
 		if err != nil {
 			t.Fatalf("failed to create tester chain: %v", err)
@@ -2751,7 +2751,6 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in
 		b.StopTimer()
 		if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
 			b.Fatalf("Transactions were not included, expected %d, got %d", numTxs*numBlocks, got)
-
 		}
 	}
 }
@@ -2762,7 +2761,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) {
 		numBlocks = 1
 	)
 	recipientFn := func(nonce uint64) common.Address {
-		return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce))
+		return common.BigToAddress(new(big.Int).SetUint64(1337 + nonce))
 	}
 	dataFn := func(nonce uint64) []byte {
 		return nil
@@ -2779,7 +2778,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) {
 	b.ResetTimer()
 
 	recipientFn := func(nonce uint64) common.Address {
-		return common.BigToAddress(big.NewInt(0).SetUint64(1337))
+		return common.BigToAddress(new(big.Int).SetUint64(1337))
 	}
 	dataFn := func(nonce uint64) []byte {
 		return nil
@@ -2796,7 +2795,7 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
 	b.ResetTimer()
 
 	recipientFn := func(nonce uint64) common.Address {
-		return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de))
+		return common.BigToAddress(new(big.Int).SetUint64(0xc0de))
 	}
 	dataFn := func(nonce uint64) []byte {
 		return nil
@@ -3520,7 +3519,6 @@ func TestEIP2718Transition(t *testing.T) {
 		vm.GasQuickStep*2 + params.WarmStorageReadCostEIP2929 + params.ColdSloadCostEIP2929
 	if block.GasUsed() != expected {
 		t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expected, block.GasUsed())
-
 	}
 }
 
@@ -3758,3 +3756,112 @@ func TestSetCanonical(t *testing.T) {
 	chain.SetCanonical(canon[TriesInMemory-1])
 	verify(canon[TriesInMemory-1])
 }
+
+// TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted
+// correctly in case reorg is called.
+func TestCanonicalHashMarker(t *testing.T) {
+	var cases = []struct {
+		forkA int
+		forkB int
+	}{
+		// ForkA: 10 blocks
+		// ForkB: 1 blocks
+		//
+		// reorged:
+		//      markers [2, 10] should be deleted
+		//      markers [1] should be updated
+		{10, 1},
+
+		// ForkA: 10 blocks
+		// ForkB: 2 blocks
+		//
+		// reorged:
+		//      markers [3, 10] should be deleted
+		//      markers [1, 2] should be updated
+		{10, 2},
+
+		// ForkA: 10 blocks
+		// ForkB: 10 blocks
+		//
+		// reorged:
+		//      markers [1, 10] should be updated
+		{10, 10},
+
+		// ForkA: 10 blocks
+		// ForkB: 11 blocks
+		//
+		// reorged:
+		//      markers [1, 11] should be updated
+		{10, 11},
+	}
+	for _, c := range cases {
+		var (
+			db    = rawdb.NewMemoryDatabase()
+			gspec = &Genesis{
+				Config:  params.TestChainConfig,
+				Alloc:   GenesisAlloc{},
+				BaseFee: big.NewInt(params.InitialBaseFee),
+			}
+			genesis = gspec.MustCommit(db)
+			engine  = ethash.NewFaker()
+		)
+		forkA, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkA, func(i int, gen *BlockGen) {})
+		forkB, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkB, func(i int, gen *BlockGen) {})
+
+		// Initialize test chain
+		diskdb := rawdb.NewMemoryDatabase()
+		gspec.MustCommit(diskdb)
+		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil)
+		if err != nil {
+			t.Fatalf("failed to create tester chain: %v", err)
+		}
+		// Insert forkA and forkB, the canonical should on forkA still
+		if n, err := chain.InsertChain(forkA); err != nil {
+			t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+		}
+		if n, err := chain.InsertChain(forkB); err != nil {
+			t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+		}
+
+		verify := func(head *types.Block) {
+			if chain.CurrentBlock().Hash() != head.Hash() {
+				t.Fatalf("Unexpected block hash, want %x, got %x", head.Hash(), chain.CurrentBlock().Hash())
+			}
+			if chain.CurrentFastBlock().Hash() != head.Hash() {
+				t.Fatalf("Unexpected fast block hash, want %x, got %x", head.Hash(), chain.CurrentFastBlock().Hash())
+			}
+			if chain.CurrentHeader().Hash() != head.Hash() {
+				t.Fatalf("Unexpected head header, want %x, got %x", head.Hash(), chain.CurrentHeader().Hash())
+			}
+			if !chain.HasState(head.Root()) {
+				t.Fatalf("Lost block state %v %x", head.Number(), head.Hash())
+			}
+		}
+
+		// Switch canonical chain to forkB if necessary
+		if len(forkA) < len(forkB) {
+			verify(forkB[len(forkB)-1])
+		} else {
+			verify(forkA[len(forkA)-1])
+			chain.SetCanonical(forkB[len(forkB)-1])
+			verify(forkB[len(forkB)-1])
+		}
+
+		// Ensure all hash markers are updated correctly
+		for i := 0; i < len(forkB); i++ {
+			block := forkB[i]
+			hash := chain.GetCanonicalHash(block.NumberU64())
+			if hash != block.Hash() {
+				t.Fatalf("Unexpected canonical hash %d", block.NumberU64())
+			}
+		}
+		if c.forkA > c.forkB {
+			for i := uint64(c.forkB) + 1; i <= uint64(c.forkA); i++ {
+				hash := chain.GetCanonicalHash(i)
+				if hash != (common.Hash{}) {
+					t.Fatalf("Unexpected canonical hash %d", i)
+				}
+			}
+		}
+	}
+}

core/bloombits/matcher_test.go

@@ -124,13 +124,13 @@ func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
 
 // testMatcherDiffBatches runs the given matches test in single-delivery and also
 // in batches delivery mode, verifying that all kinds of deliveries are handled
-// correctly withn.
+// correctly within.
 func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
 	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
 	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
 
 	if singleton != batched {
-		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
+		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
 	}
 }
 

core/evm.go

@@ -31,7 +31,7 @@ type ChainContext interface {
 	// Engine retrieves the chain's consensus engine.
 	Engine() consensus.Engine
 
-	// GetHeader returns the hash corresponding to their hash.
+	// GetHeader returns the header corresponding to the hash/number argument pair.
 	GetHeader(common.Hash, uint64) *types.Header
 }
 
@@ -84,6 +84,11 @@ func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash
 	var cache []common.Hash
 
 	return func(n uint64) common.Hash {
+		if ref.Number.Uint64() <= n {
+			// This situation can happen if we're doing tracing and using
+			// block overrides.
+			return common.Hash{}
+		}
 		// If there's no hash cache yet, make one
 		if len(cache) == 0 {
 			cache = append(cache, ref.ParentHash)

core/forkid/forkid_test.go

@@ -31,7 +31,7 @@ import (
 // the correct fork ID.
 func TestCreation(t *testing.T) {
 	mergeConfig := *params.MainnetChainConfig
-	mergeConfig.MergeForkBlock = big.NewInt(15000000)
+	mergeConfig.MergeNetsplitBlock = big.NewInt(18000000)
 	type testcase struct {
 		head uint64
 		want ID
@@ -68,8 +68,10 @@ func TestCreation(t *testing.T) {
 				{12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block
 				{12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block
 				{13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block
-				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}},        // First Arrow Glacier block
-				{20000000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}},        // Future Arrow Glacier block
+				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block
+				{15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block
+				{15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}},        // First Gray Glacier block
+				{20000000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}},        // Future Gray Glacier block
 			},
 		},
 		// Ropsten test cases
@@ -136,6 +138,16 @@ func TestCreation(t *testing.T) {
 				{6000000, ID{Hash: checksumToBytes(0xB8C6299D), Next: 0}},       // Future London block
 			},
 		},
+		// Sepolia test cases
+		{
+			params.SepoliaChainConfig,
+			params.SepoliaGenesisHash,
+			[]testcase{
+				{0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}},       // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin and first London block
+				{1735370, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Last London block
+				{1735371, ID{Hash: checksumToBytes(0xb96cbd13), Next: 0}},       // First MergeNetsplit block
+			},
+		},
 		// Merge test cases
 		{
 			&mergeConfig,
@@ -163,9 +175,11 @@ func TestCreation(t *testing.T) {
 				{12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block
 				{12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block
 				{13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block
-				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15000000}}, // First Arrow Glacier block
-				{15000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}},        // First Merge Start block
-				{20000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}},        // Future Merge Start block
+				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block
+				{15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block
+				{15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 18000000}}, // First Gray Glacier block
+				{18000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}},        // First Merge Start block
+				{20000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}},        // Future Merge Start block
 			},
 		},
 	}
@@ -242,11 +256,11 @@ func TestValidation(t *testing.T) {
 		// Local is mainnet Petersburg, remote is Rinkeby Petersburg.
 		{7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale},
 
-		// Local is mainnet Arrow Glacier, far in the future. Remote announces Gopherium (non existing fork)
+		// Local is mainnet Gray Glacier, far in the future. Remote announces Gopherium (non existing fork)
 		// at some future block 88888888, for itself, but past block for local. Local is incompatible.
 		//
 		// This case detects non-upgraded nodes with majority hash power (typical Ropsten mess).
-		{88888888, ID{Hash: checksumToBytes(0x20c327fc), Next: 88888888}, ErrLocalIncompatibleOrStale},
+		{88888888, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale},
 
 		// Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing
 		// fork) at block 7279999, before Petersburg. Local is incompatible.

core/genesis.go

@@ -233,10 +233,22 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig
 	return SetupGenesisBlockWithOverride(db, genesis, nil, nil)
 }
 
-func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideArrowGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) {
+func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideGrayGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) {
 	if genesis != nil && genesis.Config == nil {
 		return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
 	}
+
+	applyOverrides := func(config *params.ChainConfig) {
+		if config != nil {
+			if overrideTerminalTotalDifficulty != nil {
+				config.TerminalTotalDifficulty = overrideTerminalTotalDifficulty
+			}
+			if overrideGrayGlacier != nil {
+				config.GrayGlacierBlock = overrideGrayGlacier
+			}
+		}
+	}
+
 	// Just commit the new block if there is no stored genesis block.
 	stored := rawdb.ReadCanonicalHash(db, 0)
 	if (stored == common.Hash{}) {
@@ -250,6 +262,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
 		if err != nil {
 			return genesis.Config, common.Hash{}, err
 		}
+		applyOverrides(genesis.Config)
 		return genesis.Config, block.Hash(), nil
 	}
 	// We have the genesis block in database(perhaps in ancient database)
@@ -268,6 +281,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
 		if err != nil {
 			return genesis.Config, hash, err
 		}
+		applyOverrides(genesis.Config)
 		return genesis.Config, block.Hash(), nil
 	}
 	// Check whether the genesis block is already written.
@@ -279,12 +293,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
 	}
 	// Get the existing chain configuration.
 	newcfg := genesis.configOrDefault(stored)
-	if overrideArrowGlacier != nil {
-		newcfg.ArrowGlacierBlock = overrideArrowGlacier
-	}
-	if overrideTerminalTotalDifficulty != nil {
-		newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty
-	}
+	applyOverrides(newcfg)
 	if err := newcfg.CheckConfigForkOrder(); err != nil {
 		return newcfg, common.Hash{}, err
 	}
@@ -301,12 +310,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
 	// apply the overrides.
 	if genesis == nil && stored != params.MainnetGenesisHash {
 		newcfg = storedcfg
-		if overrideArrowGlacier != nil {
-			newcfg.ArrowGlacierBlock = overrideArrowGlacier
-		}
-		if overrideTerminalTotalDifficulty != nil {
-			newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty
-		}
+		applyOverrides(newcfg)
 	}
 	// Check config compatibility and write the config. Compatibility errors
 	// are returned to the caller unless we're already at block zero.

core/rawdb/database.go

@@ -208,7 +208,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 			// are contiguous, otherwise we might end up with a non-functional freezer.
 			if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
 				// Subsequent header after the freezer limit is missing from the database.
-				// Reject startup is the database has a more recent head.
+				// Reject startup if the database has a more recent head.
 				if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
 					return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
 				}

core/rawdb/freezer_test.go

@@ -277,7 +277,7 @@ func TestFreezerReadonlyValidate(t *testing.T) {
 
 	// Re-openening as readonly should fail when validating
 	// table lengths.
-	f, err = NewFreezer(dir, "", true, 2049, tables)
+	_, err = NewFreezer(dir, "", true, 2049, tables)
 	if err == nil {
 		t.Fatal("readonly freezer should fail with differing table lengths")
 	}

core/rawdb/freezer_utils_test.go

@@ -43,7 +43,7 @@ func TestCopyFrom(t *testing.T) {
 		{"foo", "bar", 8, true},
 	}
 	for _, c := range cases {
-		os.WriteFile(c.src, content, 0644)
+		os.WriteFile(c.src, content, 0600)
 
 		if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error {
 			if !c.writePrefix {

core/state/database.go

@@ -133,7 +133,7 @@ type cachingDB struct {
 
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
-	tr, err := trie.NewSecure(root, db.db)
+	tr, err := trie.NewSecure(common.Hash{}, root, db.db)
 	if err != nil {
 		return nil, err
 	}
@@ -142,7 +142,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
-	tr, err := trie.NewSecure(root, db.db)
+	tr, err := trie.NewSecure(addrHash, root, db.db)
 	if err != nil {
 		return nil, err
 	}

core/state/pruner/bloom.go

@@ -39,7 +39,7 @@ func (f stateBloomHasher) BlockSize() int                    { panic("not implem
 func (f stateBloomHasher) Size() int                         { return 8 }
 func (f stateBloomHasher) Sum64() uint64                     { return binary.BigEndian.Uint64(f) }
 
-// stateBloom is a bloom filter used during the state convesion(snapshot->state).
+// stateBloom is a bloom filter used during the state conversion(snapshot->state).
 // The keys of all generated entries will be recorded here so that in the pruning
 // stage the entries belong to the specific version can be avoided for deletion.
 //
@@ -100,7 +100,7 @@ func (bloom *stateBloom) Commit(filename, tempname string) error {
 	}
 	f.Close()
 
-	// Move the teporary file into it's final location
+	// Move the temporary file into it's final location
 	return os.Rename(tempname, filename)
 }
 

core/state/pruner/pruner.go

@@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
 	if genesis == nil {
 		return errors.New("missing genesis block")
 	}
-	t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db))
+	t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db))
 	if err != nil {
 		return err
 	}
@@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
 				return err
 			}
 			if acc.Root != emptyRoot {
-				storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db))
+				storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db))
 				if err != nil {
 					return err
 				}

core/state/snapshot/conversion.go

@@ -43,7 +43,7 @@ type trieKV struct {
 type (
 	// trieGeneratorFn is the interface of trie generation which can
 	// be implemented by different trie algorithm.
-	trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash))
+	trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash))
 
 	// leafCallbackFn is the callback invoked at the leaves of the trie,
 	// returns the subtrie root with the specified subtrie identifier.
@@ -253,7 +253,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		generatorFn(db, in, out)
+		generatorFn(db, account, in, out)
 	}()
 	// Spin up a go-routine for progress logging
 	if report && stats != nil {
@@ -360,8 +360,8 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
 	return stop(nil)
 }
 
-func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) {
-	t := trie.NewStackTrie(db)
+func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) {
+	t := trie.NewStackTrieWithOwner(db, owner)
 	for leaf := range in {
 		t.TryUpdate(leaf.key[:], leaf.value)
 	}

core/state/snapshot/difflayer_test.go

@@ -332,7 +332,6 @@ func BenchmarkFlatten(b *testing.B) {
 				value := make([]byte, 32)
 				rand.Read(value)
 				accStorage[randomHash()] = value
-
 			}
 			storage[accountKey] = accStorage
 		}
@@ -382,7 +381,6 @@ func BenchmarkJournal(b *testing.B) {
 				value := make([]byte, 32)
 				rand.Read(value)
 				accStorage[randomHash()] = value
-
 			}
 			storage[accountKey] = accStorage
 		}

core/state/snapshot/disklayer_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -515,11 +514,7 @@ func TestDiskMidAccountPartialMerge(t *testing.T) {
 // TestDiskSeek tests that seek-operations work on the disk layer
 func TestDiskSeek(t *testing.T) {
 	// Create some accounts in the disk layer
-	diskdb, err := leveldb.New(t.TempDir(), 256, 0, "", false)
-	if err != nil {
-		t.Fatal(err)
-	}
-	db := rawdb.NewDatabase(diskdb)
+	db := rawdb.NewMemoryDatabase()
 	defer db.Close()
 
 	// Fill even keys [0,2,4...]

core/state/snapshot/generate.go

@@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error)
 //
 // The proof result will be returned if the range proving is finished, otherwise
 // the error will be returned to abort the entire procedure.
-func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
+func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
 	var (
 		keys     [][]byte
 		vals     [][]byte
@@ -234,7 +234,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix
 
 	// The snap state is exhausted, pass the entire key/val set for verification
 	if origin == nil && !diskMore {
-		stackTr := trie.NewStackTrie(nil)
+		stackTr := trie.NewStackTrieWithOwner(nil, owner)
 		for i, key := range keys {
 			stackTr.TryUpdate(key, vals[i])
 		}
@@ -248,7 +248,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix
 		return &proofResult{keys: keys, vals: vals}, nil
 	}
 	// Snap state is chunked, generate edge proofs for verification.
-	tr, err := trie.New(root, dl.triedb)
+	tr, err := trie.New(owner, root, dl.triedb)
 	if err != nil {
 		ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
 		return nil, errMissingTrie
@@ -313,9 +313,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
 // generateRange generates the state segment with particular prefix. Generation can
 // either verify the correctness of existing state through range-proof and skip
 // generation, or iterate trie to regenerate state on demand.
-func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
+func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
 	// Use range prover to check the validity of the flat state in the range
-	result, err := dl.proveRange(ctx, root, prefix, kind, origin, max, valueConvertFn)
+	result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn)
 	if err != nil {
 		return false, nil, err
 	}
@@ -363,7 +363,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref
 	if len(result.keys) > 0 {
 		snapNodeCache = memorydb.New()
 		snapTrieDb := trie.NewDatabase(snapNodeCache)
-		snapTrie, _ := trie.New(common.Hash{}, snapTrieDb)
+		snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb)
 		for i, key := range result.keys {
 			snapTrie.Update(key, result.vals[i])
 		}
@@ -374,7 +374,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref
 	// if it's already opened with some nodes resolved.
 	tr := result.tr
 	if tr == nil {
-		tr, err = trie.New(root, dl.triedb)
+		tr, err = trie.New(owner, root, dl.triedb)
 		if err != nil {
 			ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
 			return false, nil, errMissingTrie
@@ -537,7 +537,7 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash,
 	// Loop for re-generating the missing storage slots.
 	var origin = common.CopyBytes(storeMarker)
 	for {
-		exhausted, last, err := dl.generateRange(ctx, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
+		exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
 		if err != nil {
 			return err // The procedure it aborted, either by external signal or internal error.
 		}
@@ -637,7 +637,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
 	}
 	origin := common.CopyBytes(accMarker)
 	for {
-		exhausted, last, err := dl.generateRange(ctx, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
+		exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
 		if err != nil {
 			return err // The procedure it aborted, either by external signal or internal error.
 		}

core/state/snapshot/generate_test.go

@@ -26,61 +26,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"golang.org/x/crypto/sha3"
 )
 
-// Tests that snapshot generation from an empty database.
-func TestGeneration(t *testing.T) {
-	// We can't use statedb to make a test trie (circular dependency), so make
-	// a fake one manually. We're going with a small account trie of 3 accounts,
-	// two of which also has the same 3-slot storage trie attached.
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
-	stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
-	stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
-	stTrie.Commit(nil)                              // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
-
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ := rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
-
-	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
-
-	acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
-	root, _, _ := accTrie.Commit(nil)    // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
-	triedb.Commit(root, false, nil)
-
-	if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
-		t.Fatalf("have %#x want %#x", have, want)
-	}
-	snap := generateSnapshot(diskdb, triedb, 16, root)
-	select {
-	case <-snap.genPending:
-		// Snapshot generation succeeded
-
-	case <-time.After(3 * time.Second):
-		t.Errorf("Snapshot generation failed")
-	}
-	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
-}
-
 func hashData(input []byte) common.Hash {
 	var hasher = sha3.NewLegacyKeccak256()
 	var hash common.Hash
@@ -90,48 +41,25 @@ func hashData(input []byte) common.Hash {
 	return hash
 }
 
-// Tests that snapshot generation with existent flat state.
-func TestGenerateExistentState(t *testing.T) {
+// Tests that snapshot generation from an empty database.
+func TestGeneration(t *testing.T) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
-	stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
-	stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
-	stTrie.Commit(nil)                              // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+	var helper = newHelper()
+	stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
 
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ := rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
-	rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val)
-	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1"))
-	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2"))
-	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3"))
+	helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+	helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+	helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
 
-	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
-	diskdb.Put(hashData([]byte("acc-2")).Bytes(), val)
-	rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val)
+	helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 
-	acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
-	rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val)
-	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1"))
-	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2"))
-	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3"))
-
-	root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
-	triedb.Commit(root, false, nil)
-
-	snap := generateSnapshot(diskdb, triedb, 16, root)
+	root, snap := helper.CommitAndGenerate()
+	if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
+		t.Fatalf("have %#x want %#x", have, want)
+	}
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -140,6 +68,43 @@ func TestGenerateExistentState(t *testing.T) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
+
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+// Tests that snapshot generation with existent flat state.
+func TestGenerateExistentState(t *testing.T) {
+	// We can't use statedb to make a test trie (circular dependency), so make
+	// a fake one manually. We're going with a small account trie of 3 accounts,
+	// two of which also has the same 3-slot storage trie attached.
+	var helper = newHelper()
+
+	stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+	helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+	helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+
+	helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+	helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+
+	stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
+	helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
+	helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+
+	root, snap := helper.CommitAndGenerate()
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(3 * time.Second):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+
 	// Signal abortion to the generator and wait for it to tear down
 	stop := make(chan *generatorStats)
 	snap.genAbort <- stop
@@ -163,7 +128,6 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
 			}
 			return hash, nil
 		}, newGenerateStats(), true)
-
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -171,20 +135,20 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
 		t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot)
 	}
 	if err := CheckDanglingStorage(snap.diskdb); err != nil {
-		t.Fatalf("Detected dangling storages %v", err)
+		t.Fatalf("Detected dangling storages: %v", err)
 	}
 }
 
 type testHelper struct {
-	diskdb  *memorydb.Database
+	diskdb  ethdb.Database
 	triedb  *trie.Database
 	accTrie *trie.SecureTrie
 }
 
 func newHelper() *testHelper {
-	diskdb := memorydb.New()
+	diskdb := rawdb.NewMemoryDatabase()
 	triedb := trie.NewDatabase(diskdb)
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
 	return &testHelper{
 		diskdb:  diskdb,
 		triedb:  triedb,
@@ -215,18 +179,28 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string)
 	}
 }
 
-func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte {
-	stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb)
+func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte {
+	stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb)
 	for i, k := range keys {
 		stTrie.Update([]byte(k), []byte(vals[i]))
 	}
-	root, _, _ := stTrie.Commit(nil)
+	var root common.Hash
+	if !commit {
+		root = stTrie.Hash()
+	} else {
+		root, _, _ = stTrie.Commit(nil)
+	}
 	return root.Bytes()
 }
 
-func (t *testHelper) Generate() (common.Hash, *diskLayer) {
+func (t *testHelper) Commit() common.Hash {
 	root, _, _ := t.accTrie.Commit(nil)
 	t.triedb.Commit(root, false, nil)
+	return root
+}
+
+func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
+	root := t.Commit()
 	snap := generateSnapshot(t.diskdb, t.triedb, 16, root)
 	return root, snap
 }
@@ -249,26 +223,29 @@ func (t *testHelper) Generate() (common.Hash, *diskLayer) {
 //   - extra slots in the end
 func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
 	helper := newHelper()
-	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
 
 	// Account one, empty root but non-empty database
 	helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
 	helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
 
 	// Account two, non empty root but empty database
+	stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 
 	// Miss slots
 	{
 		// Account three, non empty root but misses slots in the beginning
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"})
 
 		// Account four, non empty root but misses slots in the middle
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"})
 
 		// Account five, non empty root but misses slots in the end
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"})
 	}
@@ -276,18 +253,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
 	// Wrong storage slots
 	{
 		// Account six, non empty root but wrong slots in the beginning
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"})
 
 		// Account seven, non empty root but wrong slots in the middle
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"})
 
 		// Account eight, non empty root but wrong slots in the end
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"})
 
 		// Account 9, non empty root but rotated slots
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"})
 	}
@@ -295,19 +276,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
 	// Extra storage slots
 	{
 		// Account 10, non empty root but extra slots in the beginning
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"})
 
 		// Account 11, non empty root but extra slots in the middle
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"})
 
 		// Account 12, non empty root but extra slots in the end
+		helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"})
 	}
 
-	root, snap := helper.Generate()
+	root, snap := helper.CommitAndGenerate()
 	t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0
 
 	select {
@@ -331,7 +315,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
 // - extra accounts
 func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
 	helper := newHelper()
-	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+
+	helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 
 	// Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6]
 	// Extra accounts [acc-0, acc-5, acc-7]
@@ -359,7 +348,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
 		helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()})          // after the end
 	}
 
-	root, snap := helper.Generate()
+	root, snap := helper.CommitAndGenerate()
 	t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8
 
 	select {
@@ -383,29 +372,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// without any storage slots to keep the test smaller.
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	tr, _ := trie.NewSecure(common.Hash{}, triedb)
-	acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ := rlp.EncodeToBytes(acc)
-	tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
+	helper := newHelper()
 
-	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+	helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
+	helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+	helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
 
-	acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
-	tr.Commit(nil)                  // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
+	root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
 
 	// Delete an account trie leaf and ensure the generator chokes
-	triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil)
-	diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
+	helper.triedb.Commit(root, false, nil)
+	helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
 
-	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"))
+	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -427,45 +406,30 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
-	stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
-	stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
-	stTrie.Commit(nil)                              // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+	helper := newHelper()
 
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ := rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
-
-	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
-
-	acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
-	accTrie.Commit(nil)                  // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
+	stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+	helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})                                                // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+	helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})                                     // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+	stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
+	root, _, _ := helper.accTrie.Commit(nil)
 
 	// We can only corrupt the disk database, so flush the tries out
-	triedb.Reference(
-		common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
+	helper.triedb.Reference(
+		common.BytesToHash(stRoot),
 		common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
 	)
-	triedb.Reference(
-		common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
+	helper.triedb.Reference(
+		common.BytesToHash(stRoot),
 		common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
 	)
-	triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil)
+	helper.triedb.Commit(root, false, nil)
 
 	// Delete a storage trie root and ensure the generator chokes
-	diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes())
+	helper.diskdb.Delete(stRoot)
 
-	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"))
+	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -486,45 +450,31 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
-	stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
-	stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
-	stTrie.Commit(nil)                              // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+	helper := newHelper()
 
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ := rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+	stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+	helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})                                                // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+	helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})                                     // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+	stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+	helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
 
-	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
-
-	acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
-	val, _ = rlp.EncodeToBytes(acc)
-	accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
-	accTrie.Commit(nil)                  // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
+	root, _, _ := helper.accTrie.Commit(nil)
 
 	// We can only corrupt the disk database, so flush the tries out
-	triedb.Reference(
-		common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
+	helper.triedb.Reference(
+		common.BytesToHash(stRoot),
 		common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
 	)
-	triedb.Reference(
-		common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
+	helper.triedb.Reference(
+		common.BytesToHash(stRoot),
 		common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
 	)
-	triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil)
+	helper.triedb.Commit(root, false, nil)
 
 	// Delete a storage trie leaf and ensure the generator chokes
-	diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
+	helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
 
-	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"))
+	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -539,56 +489,51 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 	<-stop
 }
 
-func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie {
-	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	for i := 0; i < n; i++ {
-		k := fmt.Sprintf("key-%d", i)
-		v := fmt.Sprintf("val-%d", i)
-		stTrie.Update([]byte(k), []byte(v))
-	}
-	stTrie.Commit(nil)
-	return stTrie
-}
-
 // Tests that snapshot generation when an extra account with storage exists in the snap state.
 func TestGenerateWithExtraAccounts(t *testing.T) {
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-		stTrie = getStorageTrie(5, triedb)
-	)
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	{ // Account one in the trie
-		acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	helper := newHelper()
+	{
+		// Account one in the trie
+		stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
+			[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
+			[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
+			true,
+		)
+		acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
-		accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+		helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+
 		// Identical in the snap
 		key := hashData([]byte("acc-1"))
-		rawdb.WriteAccountSnapshot(diskdb, key, val)
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5"))
+		rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1"))
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2"))
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3"))
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4"))
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5"))
 	}
-	{ // Account two exists only in the snapshot
-		acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	{
+		// Account two exists only in the snapshot
+		stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")),
+			[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
+			[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
+			true,
+		)
+		acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
 		key := hashData([]byte("acc-2"))
-		rawdb.WriteAccountSnapshot(diskdb, key, val)
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3"))
+		rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1"))
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2"))
+		rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3"))
 	}
-	root, _, _ := accTrie.Commit(nil)
-	t.Logf("root: %x", root)
-	triedb.Commit(root, false, nil)
+	root := helper.Commit()
+
 	// To verify the test: If we now inspect the snap db, there should exist extraneous storage items
-	if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
+	if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
 		t.Fatalf("expected snap storage to exist")
 	}
-
-	snap := generateSnapshot(diskdb, triedb, 16, root)
+	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -597,12 +542,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
+
 	// Signal abortion to the generator and wait for it to tear down
 	stop := make(chan *generatorStats)
 	snap.genAbort <- stop
 	<-stop
 	// If we now inspect the snap db, there should exist no extraneous storage items
-	if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
+	if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
 		t.Fatalf("expected slot to be removed, got %v", string(data))
 	}
 }
@@ -616,37 +562,36 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
 	if false {
 		enableLogging()
 	}
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-		stTrie = getStorageTrie(3, triedb)
-	)
-	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
-	{ // Account one in the trie
-		acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	helper := newHelper()
+	{
+		// Account one in the trie
+		stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
+			[]string{"key-1", "key-2", "key-3"},
+			[]string{"val-1", "val-2", "val-3"},
+			true,
+		)
+		acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
-		accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+		helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+
 		// Identical in the snap
 		key := hashData([]byte("acc-1"))
-		rawdb.WriteAccountSnapshot(diskdb, key, val)
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
-		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
+		rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
+		rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
+		rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
+		rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
 	}
-	{ // 100 accounts exist only in snapshot
+	{
+		// 100 accounts exist only in snapshot
 		for i := 0; i < 1000; i++ {
 			//acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
 			acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
 			val, _ := rlp.EncodeToBytes(acc)
 			key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
-			rawdb.WriteAccountSnapshot(diskdb, key, val)
+			rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
 		}
 	}
-	root, _, _ := accTrie.Commit(nil)
-	t.Logf("root: %x", root)
-	triedb.Commit(root, false, nil)
-
-	snap := generateSnapshot(diskdb, triedb, 16, root)
+	root, snap := helper.CommitAndGenerate()
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -675,31 +620,22 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
 	if false {
 		enableLogging()
 	}
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	accTrie, _ := trie.New(common.Hash{}, triedb)
+	helper := newHelper()
 	{
 		acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
-		accTrie.Update(common.HexToHash("0x03").Bytes(), val)
-		accTrie.Update(common.HexToHash("0x07").Bytes(), val)
+		helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
+		helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val)
 
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val)
 	}
-
-	root, _, _ := accTrie.Commit(nil)
-	t.Logf("root: %x", root)
-	triedb.Commit(root, false, nil)
-
-	snap := generateSnapshot(diskdb, triedb, 16, root)
+	root, snap := helper.CommitAndGenerate()
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -721,29 +657,20 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
 	if false {
 		enableLogging()
 	}
-	var (
-		diskdb = memorydb.New()
-		triedb = trie.NewDatabase(diskdb)
-	)
-	accTrie, _ := trie.New(common.Hash{}, triedb)
+	helper := newHelper()
 	{
 		acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
-		accTrie.Update(common.HexToHash("0x03").Bytes(), val)
+		helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
 
 		junk := make([]byte, 100)
 		copy(junk, []byte{0xde, 0xad})
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk)
-		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk)
+		rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk)
 	}
-
-	root, _, _ := accTrie.Commit(nil)
-	t.Logf("root: %x", root)
-	triedb.Commit(root, false, nil)
-
-	snap := generateSnapshot(diskdb, triedb, 16, root)
+	root, snap := helper.CommitAndGenerate()
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -757,7 +684,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
 	snap.genAbort <- stop
 	<-stop
 	// If we now inspect the snap db, there should exist no extraneous storage items
-	if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
+	if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
 		t.Fatalf("expected slot to be removed, got %v", string(data))
 	}
 }
@@ -767,13 +694,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
 	accountCheckRange = 10
 	storageCheckRange = 20
 	helper := newHelper()
-	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
 	// Add 1K accounts to the trie
 	for i := 0; i < 400; i++ {
+		stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 		helper.addTrieAccount(fmt.Sprintf("acc-%d", i),
 			&Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 	}
-	root, snap := helper.Generate()
+	root, snap := helper.CommitAndGenerate()
 	t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
 
 	select {
@@ -802,12 +729,12 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
 	helper := newHelper()
 	stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
 	stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
-	stRoot := helper.makeStorageTrie(stKeys, stVals)
 	// We add 8 accounts, each one is missing exactly one of the storage slots. This means
 	// we don't have to order the keys and figure out exactly which hash-key winds up
 	// on the sensitive spots at the boundaries
 	for i := 0; i < 8; i++ {
 		accKey := fmt.Sprintf("acc-%d", i)
+		stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true)
 		helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()})
 		var moddedKeys []string
 		var moddedVals []string
@@ -819,8 +746,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
 		}
 		helper.addSnapStorage(accKey, moddedKeys, moddedVals)
 	}
-
-	root, snap := helper.Generate()
+	root, snap := helper.CommitAndGenerate()
 	t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff
 
 	select {
@@ -899,7 +825,7 @@ func populateDangling(disk ethdb.KeyValueStore) {
 // This test will populate some dangling storages to see if they can be cleaned up.
 func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
 	var helper = newHelper()
-	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+	stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 
 	helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 	helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
@@ -910,7 +836,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
 
 	populateDangling(helper.diskdb)
 
-	root, snap := helper.Generate()
+	root, snap := helper.CommitAndGenerate()
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -932,7 +858,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
 // This test will populate some dangling storages to see if they can be cleaned up.
 func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
 	var helper = newHelper()
-	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+	stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 
 	helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
 	helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
@@ -940,7 +866,7 @@ func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
 
 	populateDangling(helper.diskdb)
 
-	root, snap := helper.Generate()
+	root, snap := helper.CommitAndGenerate()
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded

core/state/snapshot/journal.go

@@ -108,44 +108,15 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou
 	// So if there is no journal, or the journal is invalid(e.g. the journal
 	// is not matched with disk layer; or the it's the legacy-format journal,
 	// etc.), we just discard all diffs and try to recover them later.
-	journal := rawdb.ReadSnapshotJournal(db)
-	if len(journal) == 0 {
-		log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "missing")
-		return base, generator, nil
-	}
-	r := rlp.NewStream(bytes.NewReader(journal), 0)
-
-	// Firstly, resolve the first element as the journal version
-	version, err := r.Uint()
+	var current snapshot = base
+	err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error {
+		current = newDiffLayer(current, root, destructSet, accountData, storageData)
+		return nil
+	})
 	if err != nil {
-		log.Warn("Failed to resolve the journal version", "error", err)
 		return base, generator, nil
 	}
-	if version != journalVersion {
-		log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version)
-		return base, generator, nil
-	}
-	// Secondly, resolve the disk layer root, ensure it's continuous
-	// with disk layer. Note now we can ensure it's the snapshot journal
-	// correct version, so we expect everything can be resolved properly.
-	var root common.Hash
-	if err := r.Decode(&root); err != nil {
-		return nil, journalGenerator{}, errors.New("missing disk layer root")
-	}
-	// The diff journal is not matched with disk, discard them.
-	// It can happen that Geth crashes without persisting the latest
-	// diff journal.
-	if !bytes.Equal(root.Bytes(), base.root.Bytes()) {
-		log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "unmatched")
-		return base, generator, nil
-	}
-	// Load all the snapshot diffs from the journal
-	snapshot, err := loadDiffLayer(base, r)
-	if err != nil {
-		return nil, journalGenerator{}, err
-	}
-	log.Debug("Loaded snapshot journal", "diskroot", base.root, "diffhead", snapshot.Root())
-	return snapshot, generator, nil
+	return current, generator, nil
 }
 
 // loadSnapshot loads a pre-existing state snapshot backed by a key-value store.
@@ -218,57 +189,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
 	return snapshot, false, nil
 }
 
-// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new
-// diff and verifying that it can be linked to the requested parent.
-func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
-	// Read the next diff journal entry
-	var root common.Hash
-	if err := r.Decode(&root); err != nil {
-		// The first read may fail with EOF, marking the end of the journal
-		if err == io.EOF {
-			return parent, nil
-		}
-		return nil, fmt.Errorf("load diff root: %v", err)
-	}
-	var destructs []journalDestruct
-	if err := r.Decode(&destructs); err != nil {
-		return nil, fmt.Errorf("load diff destructs: %v", err)
-	}
-	destructSet := make(map[common.Hash]struct{})
-	for _, entry := range destructs {
-		destructSet[entry.Hash] = struct{}{}
-	}
-	var accounts []journalAccount
-	if err := r.Decode(&accounts); err != nil {
-		return nil, fmt.Errorf("load diff accounts: %v", err)
-	}
-	accountData := make(map[common.Hash][]byte)
-	for _, entry := range accounts {
-		if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
-			accountData[entry.Hash] = entry.Blob
-		} else {
-			accountData[entry.Hash] = nil
-		}
-	}
-	var storage []journalStorage
-	if err := r.Decode(&storage); err != nil {
-		return nil, fmt.Errorf("load diff storage: %v", err)
-	}
-	storageData := make(map[common.Hash]map[common.Hash][]byte)
-	for _, entry := range storage {
-		slots := make(map[common.Hash][]byte)
-		for i, key := range entry.Keys {
-			if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
-				slots[key] = entry.Vals[i]
-			} else {
-				slots[key] = nil
-			}
-		}
-		storageData[entry.Hash] = slots
-	}
-	return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r)
-}
-
 // Journal terminates any in-progress snapshot generation, also implicitly pushing
 // the progress into the database.
 func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
@@ -345,3 +265,96 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
 	log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root())
 	return base, nil
 }
+
+// journalCallback is a function which is invoked by iterateJournal, every
+// time a difflayer is loaded from disk.
+type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error
+
+// iterateJournal iterates through the journalled difflayers, loading them from
+// the database, and invoking the callback for each loaded layer.
+// The order is incremental; starting with the bottom-most difflayer, going towards
+// the most recent layer.
+// This method returns error either if there was some error reading from disk,
+// OR if the callback returns an error when invoked.
+func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
+	journal := rawdb.ReadSnapshotJournal(db)
+	if len(journal) == 0 {
+		log.Warn("Loaded snapshot journal", "diffs", "missing")
+		return nil
+	}
+	r := rlp.NewStream(bytes.NewReader(journal), 0)
+	// Firstly, resolve the first element as the journal version
+	version, err := r.Uint64()
+	if err != nil {
+		log.Warn("Failed to resolve the journal version", "error", err)
+		return errors.New("failed to resolve journal version")
+	}
+	if version != journalVersion {
+		log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version)
+		return errors.New("wrong journal version")
+	}
+	// Secondly, resolve the disk layer root, ensure it's continuous
+	// with disk layer. Note now we can ensure it's the snapshot journal
+	// correct version, so we expect everything can be resolved properly.
+	var parent common.Hash
+	if err := r.Decode(&parent); err != nil {
+		return errors.New("missing disk layer root")
+	}
+	if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent {
+		log.Warn("Loaded snapshot journal", "diskroot", baseRoot, "diffs", "unmatched")
+		return fmt.Errorf("mismatched disk and diff layers")
+	}
+	for {
+		var (
+			root        common.Hash
+			destructs   []journalDestruct
+			accounts    []journalAccount
+			storage     []journalStorage
+			destructSet = make(map[common.Hash]struct{})
+			accountData = make(map[common.Hash][]byte)
+			storageData = make(map[common.Hash]map[common.Hash][]byte)
+		)
+		// Read the next diff journal entry
+		if err := r.Decode(&root); err != nil {
+			// The first read may fail with EOF, marking the end of the journal
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return fmt.Errorf("load diff root: %v", err)
+		}
+		if err := r.Decode(&destructs); err != nil {
+			return fmt.Errorf("load diff destructs: %v", err)
+		}
+		if err := r.Decode(&accounts); err != nil {
+			return fmt.Errorf("load diff accounts: %v", err)
+		}
+		if err := r.Decode(&storage); err != nil {
+			return fmt.Errorf("load diff storage: %v", err)
+		}
+		for _, entry := range destructs {
+			destructSet[entry.Hash] = struct{}{}
+		}
+		for _, entry := range accounts {
+			if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
+				accountData[entry.Hash] = entry.Blob
+			} else {
+				accountData[entry.Hash] = nil
+			}
+		}
+		for _, entry := range storage {
+			slots := make(map[common.Hash][]byte)
+			for i, key := range entry.Keys {
+				if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
+					slots[key] = entry.Vals[i]
+				} else {
+					slots[key] = nil
+				}
+			}
+			storageData[entry.Hash] = slots
+		}
+		if err := callback(parent, root, destructSet, accountData, storageData); err != nil {
+			return err
+		}
+		parent = root
+	}
+}

core/state/snapshot/utils.go

@@ -18,9 +18,7 @@ package snapshot
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
-	"io"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -34,7 +32,7 @@ import (
 // storage also has corresponding account data.
 func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error {
 	if err := checkDanglingDiskStorage(chaindb); err != nil {
-		return err
+		log.Error("Database check error", "err", err)
 	}
 	return checkDanglingMemStorage(chaindb)
 }
@@ -75,81 +73,80 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error {
 // checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled
 // snapshot difflayers.
 func checkDanglingMemStorage(db ethdb.KeyValueStore) error {
-	var (
-		start   = time.Now()
-		journal = rawdb.ReadSnapshotJournal(db)
-	)
-	if len(journal) == 0 {
-		log.Warn("Loaded snapshot journal", "diffs", "missing")
+	start := time.Now()
+	log.Info("Checking dangling journalled storage")
+	err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error {
+		for accHash := range storage {
+			if _, ok := accounts[accHash]; !ok {
+				log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root)
+			}
+		}
 		return nil
-	}
-	r := rlp.NewStream(bytes.NewReader(journal), 0)
-	// Firstly, resolve the first element as the journal version
-	version, err := r.Uint()
+	})
 	if err != nil {
-		log.Warn("Failed to resolve the journal version", "error", err)
-		return nil
-	}
-	if version != journalVersion {
-		log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version)
-		return nil
-	}
-	// Secondly, resolve the disk layer root, ensure it's continuous
-	// with disk layer. Note now we can ensure it's the snapshot journal
-	// correct version, so we expect everything can be resolved properly.
-	var root common.Hash
-	if err := r.Decode(&root); err != nil {
-		return errors.New("missing disk layer root")
-	}
-	// The diff journal is not matched with disk, discard them.
-	// It can happen that Geth crashes without persisting the latest
-	// diff journal.
-	// Load all the snapshot diffs from the journal
-	if err := checkDanglingJournalStorage(r); err != nil {
+		log.Info("Failed to resolve snapshot journal", "err", err)
 		return err
 	}
 	log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start)))
 	return nil
 }
 
-// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new
-// diff and verifying that it can be linked to the requested parent.
-func checkDanglingJournalStorage(r *rlp.Stream) error {
-	for {
-		// Read the next diff journal entry
-		var root common.Hash
-		if err := r.Decode(&root); err != nil {
-			// The first read may fail with EOF, marking the end of the journal
-			if err == io.EOF {
-				return nil
-			}
-			return fmt.Errorf("load diff root: %v", err)
-		}
-		var destructs []journalDestruct
-		if err := r.Decode(&destructs); err != nil {
-			return fmt.Errorf("load diff destructs: %v", err)
-		}
-		var accounts []journalAccount
-		if err := r.Decode(&accounts); err != nil {
-			return fmt.Errorf("load diff accounts: %v", err)
-		}
-		accountData := make(map[common.Hash][]byte)
-		for _, entry := range accounts {
-			if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
-				accountData[entry.Hash] = entry.Blob
-			} else {
-				accountData[entry.Hash] = nil
-			}
-		}
-		var storage []journalStorage
-		if err := r.Decode(&storage); err != nil {
-			return fmt.Errorf("load diff storage: %v", err)
-		}
-		for _, entry := range storage {
-			if _, ok := accountData[entry.Hash]; !ok {
-				log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", entry.Hash), "root", root)
-				return fmt.Errorf("dangling journal snapshot storage account %#x", entry.Hash)
-			}
+// CheckJournalAccount shows information about an account, from the disk layer and
+// up through the diff layers.
+func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error {
+	// Look up the disk layer first
+	baseRoot := rawdb.ReadSnapshotRoot(db)
+	fmt.Printf("Disklayer: Root: %x\n", baseRoot)
+	if data := rawdb.ReadAccountSnapshot(db, hash); data != nil {
+		account := new(Account)
+		if err := rlp.DecodeBytes(data, account); err != nil {
+			panic(err)
 		}
+		fmt.Printf("\taccount.nonce: %d\n", account.Nonce)
+		fmt.Printf("\taccount.balance: %x\n", account.Balance)
+		fmt.Printf("\taccount.root: %x\n", account.Root)
+		fmt.Printf("\taccount.codehash: %x\n", account.CodeHash)
 	}
+	// Check storage
+	{
+		it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength)
+		fmt.Printf("\tStorage:\n")
+		for it.Next() {
+			slot := it.Key()[33:]
+			fmt.Printf("\t\t%x: %x\n", slot, it.Value())
+		}
+		it.Release()
+	}
+	var depth = 0
+
+	return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error {
+		_, a := accounts[hash]
+		_, b := destructs[hash]
+		_, c := storage[hash]
+		depth++
+		if !a && !b && !c {
+			return nil
+		}
+		fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot)
+		if data, ok := accounts[hash]; ok {
+			account := new(Account)
+			if err := rlp.DecodeBytes(data, account); err != nil {
+				panic(err)
+			}
+			fmt.Printf("\taccount.nonce: %d\n", account.Nonce)
+			fmt.Printf("\taccount.balance: %x\n", account.Balance)
+			fmt.Printf("\taccount.root: %x\n", account.Root)
+			fmt.Printf("\taccount.codehash: %x\n", account.CodeHash)
+		}
+		if _, ok := destructs[hash]; ok {
+			fmt.Printf("\t Destructed!")
+		}
+		if data, ok := storage[hash]; ok {
+			fmt.Printf("\tStorage\n")
+			for k, v := range data {
+				fmt.Printf("\t\t%x: %x\n", k, v)
+			}
+		}
+		return nil
+	})
 }

core/state/state_object.go

@@ -49,7 +49,7 @@ func (s Storage) String() (str string) {
 }
 
 func (s Storage) Copy() Storage {
-	cpy := make(Storage)
+	cpy := make(Storage, len(s))
 	for key, value := range s {
 		cpy[key] = value
 	}
@@ -154,7 +154,7 @@ func (s *stateObject) getTrie(db Database) Trie {
 		if s.data.Root != emptyRoot && s.db.prefetcher != nil {
 			// When the miner is creating the pending state, there is no
 			// prefetcher
-			s.trie = s.db.prefetcher.trie(s.data.Root)
+			s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
 		}
 		if s.trie == nil {
 			var err error
@@ -295,7 +295,7 @@ func (s *stateObject) finalise(prefetch bool) {
 		}
 	}
 	if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot {
-		s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch)
+		s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch)
 	}
 	if len(s.dirtyStorage) > 0 {
 		s.dirtyStorage = make(Storage)
@@ -352,7 +352,7 @@ func (s *stateObject) updateTrie(db Database) Trie {
 		usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure
 	}
 	if s.db.prefetcher != nil {
-		s.db.prefetcher.used(s.data.Root, usedStorage)
+		s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage)
 	}
 	if len(s.pendingStorage) > 0 {
 		s.pendingStorage = make(Storage)

core/state/statedb.go

@@ -62,11 +62,14 @@ func (n *proofList) Delete(key []byte) error {
 // * Contracts
 // * Accounts
 type StateDB struct {
-	db           Database
-	prefetcher   *triePrefetcher
-	originalRoot common.Hash // The pre-state root, before any changes were made
-	trie         Trie
-	hasher       crypto.KeccakState
+	db         Database
+	prefetcher *triePrefetcher
+	trie       Trie
+	hasher     crypto.KeccakState
+
+	// originalRoot is the pre-state root, before any changes were made.
+	// It will be updated when the Commit is called.
+	originalRoot common.Hash
 
 	snaps         *snapshot.Tree
 	snap          snapshot.Snapshot
@@ -657,6 +660,7 @@ func (s *StateDB) Copy() *StateDB {
 	state := &StateDB{
 		db:                  s.db,
 		trie:                s.db.CopyTrie(s.trie),
+		originalRoot:        s.originalRoot,
 		stateObjects:        make(map[common.Address]*stateObject, len(s.journal.dirties)),
 		stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)),
 		stateObjectsDirty:   make(map[common.Address]struct{}, len(s.journal.dirties)),
@@ -819,7 +823,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) {
 		addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure
 	}
 	if s.prefetcher != nil && len(addressesToPrefetch) > 0 {
-		s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch)
+		s.prefetcher.prefetch(common.Hash{}, s.originalRoot, addressesToPrefetch)
 	}
 	// Invalidate journal because reverting across transactions is not allowed.
 	s.clearJournalAndRefund()
@@ -860,7 +864,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 	// _untouched_. We can check with the prefetcher, if it can give us a trie
 	// which has the same root, but also has some content loaded into it.
 	if prefetcher != nil {
-		if trie := prefetcher.trie(s.originalRoot); trie != nil {
+		if trie := prefetcher.trie(common.Hash{}, s.originalRoot); trie != nil {
 			s.trie = trie
 		}
 	}
@@ -876,7 +880,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 		usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure
 	}
 	if prefetcher != nil {
-		prefetcher.used(s.originalRoot, usedAddrs)
+		prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs)
 	}
 	if len(s.stateObjectsPending) > 0 {
 		s.stateObjectsPending = make(map[common.Address]struct{})
@@ -948,7 +952,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 	// The onleaf func is called _serially_, so we can reuse the same account
 	// for unmarshalling every time.
 	var account types.StateAccount
-	root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error {
+	root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash, _ []byte) error {
 		if err := rlp.DecodeBytes(leaf, &account); err != nil {
 			return nil
 		}
@@ -997,6 +1001,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 		}
 		s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
 	}
+	s.originalRoot = root
 	return root, err
 }
 

core/state/statedb_test.go

@@ -699,7 +699,6 @@ func TestDeleteCreateRevert(t *testing.T) {
 // the Commit operation fails with an error
 // If we are missing trie nodes, we should not continue writing to the trie
 func TestMissingTrieNodes(t *testing.T) {
-
 	// Create an initial state with a few accounts
 	memDb := rawdb.NewMemoryDatabase()
 	db := NewDatabase(memDb)

core/state/sync.go

@@ -27,20 +27,20 @@ import (
 )
 
 // NewStateSync create a new state trie download scheduler.
-func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(paths [][]byte, leaf []byte) error) *trie.Sync {
+func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error) *trie.Sync {
 	// Register the storage slot callback if the external callback is specified.
-	var onSlot func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error
+	var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error
 	if onLeaf != nil {
-		onSlot = func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error {
-			return onLeaf(paths, leaf)
+		onSlot = func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error {
+			return onLeaf(keys, leaf)
 		}
 	}
 	// Register the account callback to connect the state trie and the storage
 	// trie belongs to the contract.
 	var syncer *trie.Sync
-	onAccount := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error {
+	onAccount := func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error {
 		if onLeaf != nil {
-			if err := onLeaf(paths, leaf); err != nil {
+			if err := onLeaf(keys, leaf); err != nil {
 				return err
 			}
 		}
@@ -48,8 +48,8 @@ func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(p
 		if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil {
 			return err
 		}
-		syncer.AddSubTrie(obj.Root, hexpath, parent, onSlot)
-		syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), hexpath, parent)
+		syncer.AddSubTrie(obj.Root, path, parent, parentPath, onSlot)
+		syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent, parentPath)
 		return nil
 	}
 	syncer = trie.NewSync(root, database, onAccount)

core/state/sync_test.go

@@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
 	if v, _ := db.Get(root[:]); v == nil {
 		return nil // Consider a non existent state consistent.
 	}
-	trie, err := trie.New(root, trie.NewDatabase(db))
+	trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db))
 	if err != nil {
 		return err
 	}
@@ -134,8 +134,8 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
 func TestEmptyStateSync(t *testing.T) {
 	empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
 	sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), nil)
-	if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 {
-		t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes)
+	if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
+		t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
 	}
 }
 
@@ -160,66 +160,93 @@ func TestIterativeStateSyncBatchedByPath(t *testing.T) {
 	testIterativeStateSync(t, 100, false, true)
 }
 
+// stateElement represents the element in the state trie(bytecode or trie node).
+type stateElement struct {
+	path     string
+	hash     common.Hash
+	code     common.Hash
+	syncPath trie.SyncPath
+}
+
 func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
 	// Create a random state to copy
 	srcDb, srcRoot, srcAccounts := makeTestState()
 	if commit {
 		srcDb.TrieDB().Commit(srcRoot, false, nil)
 	}
-	srcTrie, _ := trie.New(srcRoot, srcDb.TrieDB())
+	srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB())
 
 	// Create a destination state and sync with the scheduler
 	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb, nil)
 
-	nodes, paths, codes := sched.Missing(count)
 	var (
-		hashQueue []common.Hash
-		pathQueue []trie.SyncPath
+		nodeElements []stateElement
+		codeElements []stateElement
 	)
-	if !bypath {
-		hashQueue = append(append(hashQueue[:0], nodes...), codes...)
-	} else {
-		hashQueue = append(hashQueue[:0], codes...)
-		pathQueue = append(pathQueue[:0], paths...)
+	paths, nodes, codes := sched.Missing(count)
+	for i := 0; i < len(paths); i++ {
+		nodeElements = append(nodeElements, stateElement{
+			path:     paths[i],
+			hash:     nodes[i],
+			syncPath: trie.NewSyncPath([]byte(paths[i])),
+		})
 	}
-	for len(hashQueue)+len(pathQueue) > 0 {
-		results := make([]trie.SyncResult, len(hashQueue)+len(pathQueue))
-		for i, hash := range hashQueue {
-			data, err := srcDb.TrieDB().Node(hash)
+	for i := 0; i < len(codes); i++ {
+		codeElements = append(codeElements, stateElement{
+			code: codes[i],
+		})
+	}
+	for len(nodeElements)+len(codeElements) > 0 {
+		var (
+			nodeResults = make([]trie.NodeSyncResult, len(nodeElements))
+			codeResults = make([]trie.CodeSyncResult, len(codeElements))
+		)
+		for i, element := range codeElements {
+			data, err := srcDb.ContractCode(common.Hash{}, element.code)
 			if err != nil {
-				data, err = srcDb.ContractCode(common.Hash{}, hash)
+				t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code)
 			}
-			if err != nil {
-				t.Fatalf("failed to retrieve node data for hash %x", hash)
-			}
-			results[i] = trie.SyncResult{Hash: hash, Data: data}
+			codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data}
 		}
-		for i, path := range pathQueue {
-			if len(path) == 1 {
-				data, _, err := srcTrie.TryGetNode(path[0])
-				if err != nil {
-					t.Fatalf("failed to retrieve node data for path %x: %v", path, err)
+		for i, node := range nodeElements {
+			if bypath {
+				if len(node.syncPath) == 1 {
+					data, _, err := srcTrie.TryGetNode(node.syncPath[0])
+					if err != nil {
+						t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[0], err)
+					}
+					nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data}
+				} else {
+					var acc types.StateAccount
+					if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil {
+						t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err)
+					}
+					stTrie, err := trie.New(common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB())
+					if err != nil {
+						t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err)
+					}
+					data, _, err := stTrie.TryGetNode(node.syncPath[1])
+					if err != nil {
+						t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[1], err)
+					}
+					nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data}
 				}
-				results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data}
 			} else {
-				var acc types.StateAccount
-				if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil {
-					t.Fatalf("failed to decode account on path %x: %v", path, err)
-				}
-				stTrie, err := trie.New(acc.Root, srcDb.TrieDB())
+				data, err := srcDb.TrieDB().Node(node.hash)
 				if err != nil {
-					t.Fatalf("failed to retriev storage trie for path %x: %v", path, err)
+					t.Fatalf("failed to retrieve node data for key %v", []byte(node.path))
 				}
-				data, _, err := stTrie.TryGetNode(path[1])
-				if err != nil {
-					t.Fatalf("failed to retrieve node data for path %x: %v", path, err)
-				}
-				results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data}
+				nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data}
 			}
 		}
-		for _, result := range results {
-			if err := sched.Process(result); err != nil {
+		for _, result := range codeResults {
+			if err := sched.ProcessCode(result); err != nil {
+				t.Errorf("failed to process result %v", err)
+			}
+		}
+		for _, result := range nodeResults {
+			if err := sched.ProcessNode(result); err != nil {
 				t.Errorf("failed to process result %v", err)
 			}
 		}
@@ -229,12 +256,20 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
 		}
 		batch.Write()
 
-		nodes, paths, codes = sched.Missing(count)
-		if !bypath {
-			hashQueue = append(append(hashQueue[:0], nodes...), codes...)
-		} else {
-			hashQueue = append(hashQueue[:0], codes...)
-			pathQueue = append(pathQueue[:0], paths...)
+		paths, nodes, codes = sched.Missing(count)
+		nodeElements = nodeElements[:0]
+		for i := 0; i < len(paths); i++ {
+			nodeElements = append(nodeElements, stateElement{
+				path:     paths[i],
+				hash:     nodes[i],
+				syncPath: trie.NewSyncPath([]byte(paths[i])),
+			})
+		}
+		codeElements = codeElements[:0]
+		for i := 0; i < len(codes); i++ {
+			codeElements = append(codeElements, stateElement{
+				code: codes[i],
+			})
 		}
 	}
 	// Cross check that the two states are in sync
@@ -251,26 +286,58 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb, nil)
 
-	nodes, _, codes := sched.Missing(0)
-	queue := append(append([]common.Hash{}, nodes...), codes...)
-
-	for len(queue) > 0 {
+	var (
+		nodeElements []stateElement
+		codeElements []stateElement
+	)
+	paths, nodes, codes := sched.Missing(0)
+	for i := 0; i < len(paths); i++ {
+		nodeElements = append(nodeElements, stateElement{
+			path:     paths[i],
+			hash:     nodes[i],
+			syncPath: trie.NewSyncPath([]byte(paths[i])),
+		})
+	}
+	for i := 0; i < len(codes); i++ {
+		codeElements = append(codeElements, stateElement{
+			code: codes[i],
+		})
+	}
+	for len(nodeElements)+len(codeElements) > 0 {
 		// Sync only half of the scheduled nodes
-		results := make([]trie.SyncResult, len(queue)/2+1)
-		for i, hash := range queue[:len(results)] {
-			data, err := srcDb.TrieDB().Node(hash)
-			if err != nil {
-				data, err = srcDb.ContractCode(common.Hash{}, hash)
+		var nodeProcessd int
+		var codeProcessd int
+		if len(codeElements) > 0 {
+			codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1)
+			for i, element := range codeElements[:len(codeResults)] {
+				data, err := srcDb.ContractCode(common.Hash{}, element.code)
+				if err != nil {
+					t.Fatalf("failed to retrieve contract bytecode for %x", element.code)
+				}
+				codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data}
 			}
-			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x", hash)
+			for _, result := range codeResults {
+				if err := sched.ProcessCode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
 			}
-			results[i] = trie.SyncResult{Hash: hash, Data: data}
+			codeProcessd = len(codeResults)
 		}
-		for _, result := range results {
-			if err := sched.Process(result); err != nil {
-				t.Fatalf("failed to process result %v", err)
+		if len(nodeElements) > 0 {
+			nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1)
+			for i, element := range nodeElements[:len(nodeResults)] {
+				data, err := srcDb.TrieDB().Node(element.hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve contract bytecode for %x", element.code)
+				}
+				nodeResults[i] = trie.NodeSyncResult{Path: element.path, Data: data}
 			}
+			for _, result := range nodeResults {
+				if err := sched.ProcessNode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
+			}
+			nodeProcessd = len(nodeResults)
 		}
 		batch := dstDb.NewBatch()
 		if err := sched.Commit(batch); err != nil {
@@ -278,8 +345,21 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 		}
 		batch.Write()
 
-		nodes, _, codes = sched.Missing(0)
-		queue = append(append(queue[len(results):], nodes...), codes...)
+		paths, nodes, codes = sched.Missing(0)
+		nodeElements = nodeElements[nodeProcessd:]
+		for i := 0; i < len(paths); i++ {
+			nodeElements = append(nodeElements, stateElement{
+				path:     paths[i],
+				hash:     nodes[i],
+				syncPath: trie.NewSyncPath([]byte(paths[i])),
+			})
+		}
+		codeElements = codeElements[codeProcessd:]
+		for i := 0; i < len(codes); i++ {
+			codeElements = append(codeElements, stateElement{
+				code: codes[i],
+			})
+		}
 	}
 	// Cross check that the two states are in sync
 	checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
@@ -299,40 +379,70 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
 	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb, nil)
 
-	queue := make(map[common.Hash]struct{})
-	nodes, _, codes := sched.Missing(count)
-	for _, hash := range append(nodes, codes...) {
-		queue[hash] = struct{}{}
+	nodeQueue := make(map[string]stateElement)
+	codeQueue := make(map[common.Hash]struct{})
+	paths, nodes, codes := sched.Missing(count)
+	for i, path := range paths {
+		nodeQueue[path] = stateElement{
+			path:     path,
+			hash:     nodes[i],
+			syncPath: trie.NewSyncPath([]byte(path)),
+		}
 	}
-	for len(queue) > 0 {
+	for _, hash := range codes {
+		codeQueue[hash] = struct{}{}
+	}
+	for len(nodeQueue)+len(codeQueue) > 0 {
 		// Fetch all the queued nodes in a random order
-		results := make([]trie.SyncResult, 0, len(queue))
-		for hash := range queue {
-			data, err := srcDb.TrieDB().Node(hash)
-			if err != nil {
-				data, err = srcDb.ContractCode(common.Hash{}, hash)
+		if len(codeQueue) > 0 {
+			results := make([]trie.CodeSyncResult, 0, len(codeQueue))
+			for hash := range codeQueue {
+				data, err := srcDb.ContractCode(common.Hash{}, hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve node data for %x", hash)
+				}
+				results = append(results, trie.CodeSyncResult{Hash: hash, Data: data})
 			}
-			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x", hash)
+			for _, result := range results {
+				if err := sched.ProcessCode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
+			}
+		}
+		if len(nodeQueue) > 0 {
+			results := make([]trie.NodeSyncResult, 0, len(nodeQueue))
+			for path, element := range nodeQueue {
+				data, err := srcDb.TrieDB().Node(element.hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path)
+				}
+				results = append(results, trie.NodeSyncResult{Path: path, Data: data})
+			}
+			for _, result := range results {
+				if err := sched.ProcessNode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
 			}
-			results = append(results, trie.SyncResult{Hash: hash, Data: data})
 		}
 		// Feed the retrieved results back and queue new tasks
-		for _, result := range results {
-			if err := sched.Process(result); err != nil {
-				t.Fatalf("failed to process result %v", err)
-			}
-		}
 		batch := dstDb.NewBatch()
 		if err := sched.Commit(batch); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
 
-		queue = make(map[common.Hash]struct{})
-		nodes, _, codes = sched.Missing(count)
-		for _, hash := range append(nodes, codes...) {
-			queue[hash] = struct{}{}
+		nodeQueue = make(map[string]stateElement)
+		codeQueue = make(map[common.Hash]struct{})
+		paths, nodes, codes := sched.Missing(count)
+		for i, path := range paths {
+			nodeQueue[path] = stateElement{
+				path:     path,
+				hash:     nodes[i],
+				syncPath: trie.NewSyncPath([]byte(path)),
+			}
+		}
+		for _, hash := range codes {
+			codeQueue[hash] = struct{}{}
 		}
 	}
 	// Cross check that the two states are in sync
@@ -349,34 +459,62 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb, nil)
 
-	queue := make(map[common.Hash]struct{})
-	nodes, _, codes := sched.Missing(0)
-	for _, hash := range append(nodes, codes...) {
-		queue[hash] = struct{}{}
+	nodeQueue := make(map[string]stateElement)
+	codeQueue := make(map[common.Hash]struct{})
+	paths, nodes, codes := sched.Missing(0)
+	for i, path := range paths {
+		nodeQueue[path] = stateElement{
+			path:     path,
+			hash:     nodes[i],
+			syncPath: trie.NewSyncPath([]byte(path)),
+		}
 	}
-	for len(queue) > 0 {
+	for _, hash := range codes {
+		codeQueue[hash] = struct{}{}
+	}
+	for len(nodeQueue)+len(codeQueue) > 0 {
 		// Sync only half of the scheduled nodes, even those in random order
-		results := make([]trie.SyncResult, 0, len(queue)/2+1)
-		for hash := range queue {
-			delete(queue, hash)
+		if len(codeQueue) > 0 {
+			results := make([]trie.CodeSyncResult, 0, len(codeQueue)/2+1)
+			for hash := range codeQueue {
+				delete(codeQueue, hash)
 
-			data, err := srcDb.TrieDB().Node(hash)
-			if err != nil {
-				data, err = srcDb.ContractCode(common.Hash{}, hash)
-			}
-			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x", hash)
-			}
-			results = append(results, trie.SyncResult{Hash: hash, Data: data})
+				data, err := srcDb.ContractCode(common.Hash{}, hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve node data for %x", hash)
+				}
+				results = append(results, trie.CodeSyncResult{Hash: hash, Data: data})
 
-			if len(results) >= cap(results) {
-				break
+				if len(results) >= cap(results) {
+					break
+				}
+			}
+			for _, result := range results {
+				if err := sched.ProcessCode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
 			}
 		}
-		// Feed the retrieved results back and queue new tasks
-		for _, result := range results {
-			if err := sched.Process(result); err != nil {
-				t.Fatalf("failed to process result %v", err)
+		if len(nodeQueue) > 0 {
+			results := make([]trie.NodeSyncResult, 0, len(nodeQueue)/2+1)
+			for path, element := range nodeQueue {
+				delete(nodeQueue, path)
+
+				data, err := srcDb.TrieDB().Node(element.hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve node data for %x", element.hash)
+				}
+				results = append(results, trie.NodeSyncResult{Path: path, Data: data})
+
+				if len(results) >= cap(results) {
+					break
+				}
+			}
+			// Feed the retrieved results back and queue new tasks
+			for _, result := range results {
+				if err := sched.ProcessNode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
 			}
 		}
 		batch := dstDb.NewBatch()
@@ -384,12 +522,17 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
-		for _, result := range results {
-			delete(queue, result.Hash)
+
+		paths, nodes, codes := sched.Missing(0)
+		for i, path := range paths {
+			nodeQueue[path] = stateElement{
+				path:     path,
+				hash:     nodes[i],
+				syncPath: trie.NewSyncPath([]byte(path)),
+			}
 		}
-		nodes, _, codes = sched.Missing(0)
-		for _, hash := range append(nodes, codes...) {
-			queue[hash] = struct{}{}
+		for _, hash := range codes {
+			codeQueue[hash] = struct{}{}
 		}
 	}
 	// Cross check that the two states are in sync
@@ -416,28 +559,62 @@ func TestIncompleteStateSync(t *testing.T) {
 	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb, nil)
 
-	var added []common.Hash
-
-	nodes, _, codes := sched.Missing(1)
-	queue := append(append([]common.Hash{}, nodes...), codes...)
-
-	for len(queue) > 0 {
-		// Fetch a batch of state nodes
-		results := make([]trie.SyncResult, len(queue))
-		for i, hash := range queue {
-			data, err := srcDb.TrieDB().Node(hash)
-			if err != nil {
-				data, err = srcDb.ContractCode(common.Hash{}, hash)
-			}
-			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x", hash)
-			}
-			results[i] = trie.SyncResult{Hash: hash, Data: data}
+	var (
+		addedCodes []common.Hash
+		addedNodes []common.Hash
+	)
+	nodeQueue := make(map[string]stateElement)
+	codeQueue := make(map[common.Hash]struct{})
+	paths, nodes, codes := sched.Missing(1)
+	for i, path := range paths {
+		nodeQueue[path] = stateElement{
+			path:     path,
+			hash:     nodes[i],
+			syncPath: trie.NewSyncPath([]byte(path)),
 		}
-		// Process each of the state nodes
-		for _, result := range results {
-			if err := sched.Process(result); err != nil {
-				t.Fatalf("failed to process result %v", err)
+	}
+	for _, hash := range codes {
+		codeQueue[hash] = struct{}{}
+	}
+	for len(nodeQueue)+len(codeQueue) > 0 {
+		// Fetch a batch of state nodes
+		if len(codeQueue) > 0 {
+			results := make([]trie.CodeSyncResult, 0, len(codeQueue))
+			for hash := range codeQueue {
+				data, err := srcDb.ContractCode(common.Hash{}, hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve node data for %x", hash)
+				}
+				results = append(results, trie.CodeSyncResult{Hash: hash, Data: data})
+				addedCodes = append(addedCodes, hash)
+			}
+			// Process each of the state nodes
+			for _, result := range results {
+				if err := sched.ProcessCode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
+			}
+		}
+		var nodehashes []common.Hash
+		if len(nodeQueue) > 0 {
+			results := make([]trie.NodeSyncResult, 0, len(nodeQueue))
+			for key, element := range nodeQueue {
+				data, err := srcDb.TrieDB().Node(element.hash)
+				if err != nil {
+					t.Fatalf("failed to retrieve node data for %x", element.hash)
+				}
+				results = append(results, trie.NodeSyncResult{Path: key, Data: data})
+
+				if element.hash != srcRoot {
+					addedNodes = append(addedNodes, element.hash)
+				}
+				nodehashes = append(nodehashes, element.hash)
+			}
+			// Process each of the state nodes
+			for _, result := range results {
+				if err := sched.ProcessNode(result); err != nil {
+					t.Fatalf("failed to process result %v", err)
+				}
 			}
 		}
 		batch := dstDb.NewBatch()
@@ -445,43 +622,44 @@ func TestIncompleteStateSync(t *testing.T) {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
-		for _, result := range results {
-			added = append(added, result.Hash)
-			// Check that all known sub-tries added so far are complete or missing entirely.
-			if _, ok := isCode[result.Hash]; ok {
-				continue
-			}
+
+		for _, root := range nodehashes {
 			// Can't use checkStateConsistency here because subtrie keys may have odd
 			// length and crash in LeafKey.
-			if err := checkTrieConsistency(dstDb, result.Hash); err != nil {
+			if err := checkTrieConsistency(dstDb, root); err != nil {
 				t.Fatalf("state inconsistent: %v", err)
 			}
 		}
 		// Fetch the next batch to retrieve
-		nodes, _, codes = sched.Missing(1)
-		queue = append(append(queue[:0], nodes...), codes...)
+		nodeQueue = make(map[string]stateElement)
+		codeQueue = make(map[common.Hash]struct{})
+		paths, nodes, codes := sched.Missing(1)
+		for i, path := range paths {
+			nodeQueue[path] = stateElement{
+				path:     path,
+				hash:     nodes[i],
+				syncPath: trie.NewSyncPath([]byte(path)),
+			}
+		}
+		for _, hash := range codes {
+			codeQueue[hash] = struct{}{}
+		}
 	}
 	// Sanity check that removing any node from the database is detected
-	for _, node := range added[1:] {
-		var (
-			key     = node.Bytes()
-			_, code = isCode[node]
-			val     []byte
-		)
-		if code {
-			val = rawdb.ReadCode(dstDb, node)
-			rawdb.DeleteCode(dstDb, node)
-		} else {
-			val = rawdb.ReadTrieNode(dstDb, node)
-			rawdb.DeleteTrieNode(dstDb, node)
+	for _, node := range addedCodes {
+		val := rawdb.ReadCode(dstDb, node)
+		rawdb.DeleteCode(dstDb, node)
+		if err := checkStateConsistency(dstDb, srcRoot); err == nil {
+			t.Errorf("trie inconsistency not caught, missing: %x", node)
 		}
-		if err := checkStateConsistency(dstDb, added[0]); err == nil {
-			t.Fatalf("trie inconsistency not caught, missing: %x", key)
-		}
-		if code {
-			rawdb.WriteCode(dstDb, node, val)
-		} else {
-			rawdb.WriteTrieNode(dstDb, node, val)
+		rawdb.WriteCode(dstDb, node, val)
+	}
+	for _, node := range addedNodes {
+		val := rawdb.ReadTrieNode(dstDb, node)
+		rawdb.DeleteTrieNode(dstDb, node)
+		if err := checkStateConsistency(dstDb, srcRoot); err == nil {
+			t.Errorf("trie inconsistency not caught, missing: %v", node.Hex())
 		}
+		rawdb.WriteTrieNode(dstDb, node, val)
 	}
 }

core/state/trie_prefetcher.go

@@ -25,7 +25,7 @@ import (
 )
 
 var (
-	// triePrefetchMetricsPrefix is the prefix under which to publis the metrics.
+	// triePrefetchMetricsPrefix is the prefix under which to publish the metrics.
 	triePrefetchMetricsPrefix = "trie/prefetch/"
 )
 
@@ -35,10 +35,10 @@ var (
 //
 // Note, the prefetcher's API is not thread safe.
 type triePrefetcher struct {
-	db       Database                    // Database to fetch trie nodes through
-	root     common.Hash                 // Root hash of theaccount trie for metrics
-	fetches  map[common.Hash]Trie        // Partially or fully fetcher tries
-	fetchers map[common.Hash]*subfetcher // Subfetchers for each trie
+	db       Database               // Database to fetch trie nodes through
+	root     common.Hash            // Root hash of the account trie for metrics
+	fetches  map[string]Trie        // Partially or fully fetcher tries
+	fetchers map[string]*subfetcher // Subfetchers for each trie
 
 	deliveryMissMeter metrics.Meter
 	accountLoadMeter  metrics.Meter
@@ -51,13 +51,12 @@ type triePrefetcher struct {
 	storageWasteMeter metrics.Meter
 }
 
-// newTriePrefetcher
 func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher {
 	prefix := triePrefetchMetricsPrefix + namespace
 	p := &triePrefetcher{
 		db:       db,
 		root:     root,
-		fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map
+		fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map
 
 		deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil),
 		accountLoadMeter:  metrics.GetOrRegisterMeter(prefix+"/account/load", nil),
@@ -112,7 +111,7 @@ func (p *triePrefetcher) copy() *triePrefetcher {
 	copy := &triePrefetcher{
 		db:      p.db,
 		root:    p.root,
-		fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map
+		fetches: make(map[string]Trie), // Active prefetchers use the fetches map
 
 		deliveryMissMeter: p.deliveryMissMeter,
 		accountLoadMeter:  p.accountLoadMeter,
@@ -132,33 +131,35 @@ func (p *triePrefetcher) copy() *triePrefetcher {
 		return copy
 	}
 	// Otherwise we're copying an active fetcher, retrieve the current states
-	for root, fetcher := range p.fetchers {
-		copy.fetches[root] = fetcher.peek()
+	for id, fetcher := range p.fetchers {
+		copy.fetches[id] = fetcher.peek()
 	}
 	return copy
 }
 
 // prefetch schedules a batch of trie items to prefetch.
-func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) {
+func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) {
 	// If the prefetcher is an inactive one, bail out
 	if p.fetches != nil {
 		return
 	}
 	// Active fetcher, schedule the retrievals
-	fetcher := p.fetchers[root]
+	id := p.trieID(owner, root)
+	fetcher := p.fetchers[id]
 	if fetcher == nil {
-		fetcher = newSubfetcher(p.db, root)
-		p.fetchers[root] = fetcher
+		fetcher = newSubfetcher(p.db, owner, root)
+		p.fetchers[id] = fetcher
 	}
 	fetcher.schedule(keys)
 }
 
 // trie returns the trie matching the root hash, or nil if the prefetcher doesn't
 // have it.
-func (p *triePrefetcher) trie(root common.Hash) Trie {
+func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie {
 	// If the prefetcher is inactive, return from existing deep copies
+	id := p.trieID(owner, root)
 	if p.fetches != nil {
-		trie := p.fetches[root]
+		trie := p.fetches[id]
 		if trie == nil {
 			p.deliveryMissMeter.Mark(1)
 			return nil
@@ -166,7 +167,7 @@ func (p *triePrefetcher) trie(root common.Hash) Trie {
 		return p.db.CopyTrie(trie)
 	}
 	// Otherwise the prefetcher is active, bail if no trie was prefetched for this root
-	fetcher := p.fetchers[root]
+	fetcher := p.fetchers[id]
 	if fetcher == nil {
 		p.deliveryMissMeter.Mark(1)
 		return nil
@@ -185,20 +186,26 @@ func (p *triePrefetcher) trie(root common.Hash) Trie {
 
 // used marks a batch of state items used to allow creating statistics as to
 // how useful or wasteful the prefetcher is.
-func (p *triePrefetcher) used(root common.Hash, used [][]byte) {
-	if fetcher := p.fetchers[root]; fetcher != nil {
+func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) {
+	if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil {
 		fetcher.used = used
 	}
 }
 
+// trieID returns an unique trie identifier consists the trie owner and root hash.
+func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string {
+	return string(append(owner.Bytes(), root.Bytes()...))
+}
+
 // subfetcher is a trie fetcher goroutine responsible for pulling entries for a
 // single trie. It is spawned when a new root is encountered and lives until the
 // main prefetcher is paused and either all requested items are processed or if
 // the trie being worked on is retrieved from the prefetcher.
 type subfetcher struct {
-	db   Database    // Database to load trie nodes through
-	root common.Hash // Root hash of the trie to prefetch
-	trie Trie        // Trie being populated with nodes
+	db    Database    // Database to load trie nodes through
+	owner common.Hash // Owner of the trie, usually account hash
+	root  common.Hash // Root hash of the trie to prefetch
+	trie  Trie        // Trie being populated with nodes
 
 	tasks [][]byte   // Items queued up for retrieval
 	lock  sync.Mutex // Lock protecting the task queue
@@ -215,15 +222,16 @@ type subfetcher struct {
 
 // newSubfetcher creates a goroutine to prefetch state items belonging to a
 // particular root hash.
-func newSubfetcher(db Database, root common.Hash) *subfetcher {
+func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher {
 	sf := &subfetcher{
-		db:   db,
-		root: root,
-		wake: make(chan struct{}, 1),
-		stop: make(chan struct{}),
-		term: make(chan struct{}),
-		copy: make(chan chan Trie),
-		seen: make(map[string]struct{}),
+		db:    db,
+		owner: owner,
+		root:  root,
+		wake:  make(chan struct{}, 1),
+		stop:  make(chan struct{}),
+		term:  make(chan struct{}),
+		copy:  make(chan chan Trie),
+		seen:  make(map[string]struct{}),
 	}
 	go sf.loop()
 	return sf
@@ -279,13 +287,21 @@ func (sf *subfetcher) loop() {
 	defer close(sf.term)
 
 	// Start by opening the trie and stop processing if it fails
-	trie, err := sf.db.OpenTrie(sf.root)
-	if err != nil {
-		log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
-		return
+	if sf.owner == (common.Hash{}) {
+		trie, err := sf.db.OpenTrie(sf.root)
+		if err != nil {
+			log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
+			return
+		}
+		sf.trie = trie
+	} else {
+		trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root)
+		if err != nil {
+			log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
+			return
+		}
+		sf.trie = trie
 	}
-	sf.trie = trie
-
 	// Trie opened successfully, keep prefetching items
 	for {
 		select {

core/state/trie_prefetcher_test.go

@@ -47,20 +47,20 @@ func TestCopyAndClose(t *testing.T) {
 	db := filledStateDB()
 	prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
 	skey := common.HexToHash("aaa")
-	prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
-	prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
+	prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
+	prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
 	time.Sleep(1 * time.Second)
-	a := prefetcher.trie(db.originalRoot)
-	prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
-	b := prefetcher.trie(db.originalRoot)
+	a := prefetcher.trie(common.Hash{}, db.originalRoot)
+	prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
+	b := prefetcher.trie(common.Hash{}, db.originalRoot)
 	cpy := prefetcher.copy()
-	cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
-	cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
-	c := cpy.trie(db.originalRoot)
+	cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
+	cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
+	c := cpy.trie(common.Hash{}, db.originalRoot)
 	prefetcher.close()
 	cpy2 := cpy.copy()
-	cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
-	d := cpy2.trie(db.originalRoot)
+	cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
+	d := cpy2.trie(common.Hash{}, db.originalRoot)
 	cpy.close()
 	cpy2.close()
 	if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() {
@@ -72,10 +72,10 @@ func TestUseAfterClose(t *testing.T) {
 	db := filledStateDB()
 	prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
 	skey := common.HexToHash("aaa")
-	prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
-	a := prefetcher.trie(db.originalRoot)
+	prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
+	a := prefetcher.trie(common.Hash{}, db.originalRoot)
 	prefetcher.close()
-	b := prefetcher.trie(db.originalRoot)
+	b := prefetcher.trie(common.Hash{}, db.originalRoot)
 	if a == nil {
 		t.Fatal("Prefetching before close should not return nil")
 	}
@@ -88,13 +88,13 @@ func TestCopyClose(t *testing.T) {
 	db := filledStateDB()
 	prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
 	skey := common.HexToHash("aaa")
-	prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
+	prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
 	cpy := prefetcher.copy()
-	a := prefetcher.trie(db.originalRoot)
-	b := cpy.trie(db.originalRoot)
+	a := prefetcher.trie(common.Hash{}, db.originalRoot)
+	b := cpy.trie(common.Hash{}, db.originalRoot)
 	prefetcher.close()
-	c := prefetcher.trie(db.originalRoot)
-	d := cpy.trie(db.originalRoot)
+	c := prefetcher.trie(common.Hash{}, db.originalRoot)
+	d := cpy.trie(common.Hash{}, db.originalRoot)
 	if a == nil {
 		t.Fatal("Prefetching before close should not return nil")
 	}

core/state_processor.go

@@ -105,8 +105,8 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 	// Finalize the block, applying any consensus engine specific extras (e.g. block rewards)
 	p.engine.Finalize(p.bc, header, statedb, block.Transactions(), block.Uncles())
 	pluginPostProcessBlock(block)
-	//begin PluGeth code injection
 	blockTracer.PostProcessBlock(block)
+	//end PluGeth code injection
 	return receipts, allLogs, *usedGas, nil
 }
 

core/state_transition.go

@@ -347,7 +347,16 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) {
 	if rules.IsLondon {
 		effectiveTip = cmath.BigMin(st.gasTipCap, new(big.Int).Sub(st.gasFeeCap, st.evm.Context.BaseFee))
 	}
-	st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip))
+
+	if st.evm.Config.NoBaseFee && st.gasFeeCap.Sign() == 0 && st.gasTipCap.Sign() == 0 {
+		// Skip fee payment when NoBaseFee is set and the fee fields
+		// are 0. This avoids a negative effectiveTip being applied to
+		// the coinbase when simulating calls.
+	} else {
+		fee := new(big.Int).SetUint64(st.gasUsed())
+		fee.Mul(fee, effectiveTip)
+		st.state.AddBalance(st.evm.Context.Coinbase, fee)
+	}
 
 	return &ExecutionResult{
 		UsedGas:    st.gasUsed(),

core/tx_pool.go

@@ -1459,7 +1459,7 @@ func (pool *TxPool) truncatePending() {
 	pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending))
 }
 
-// truncateQueue drops the oldes transactions in the queue if the pool is above the global queue limit.
+// truncateQueue drops the oldest transactions in the queue if the pool is above the global queue limit.
 func (pool *TxPool) truncateQueue() {
 	queued := uint64(0)
 	for _, list := range pool.queue {
@@ -1603,10 +1603,6 @@ func (as *accountSet) contains(addr common.Address) bool {
 	return exist
 }
 
-func (as *accountSet) empty() bool {
-	return len(as.accounts) == 0
-}
-
 // containsTx checks if the sender of a given tx is within the set. If the sender
 // cannot be derived, this method returns false.
 func (as *accountSet) containsTx(tx *types.Transaction) bool {

core/tx_pool_test.go

@@ -669,7 +669,6 @@ func TestTransactionPostponing(t *testing.T) {
 	// Add a batch consecutive pending transactions for validation
 	txs := []*types.Transaction{}
 	for i, key := range keys {
-
 		for j := 0; j < 100; j++ {
 			var tx *types.Transaction
 			if (i+j)%2 == 0 {

core/types/block.go

@@ -172,10 +172,6 @@ type Block struct {
 	hash atomic.Value
 	size atomic.Value
 
-	// Td is used by package core to store the total difficulty
-	// of the chain up to and including the block.
-	td *big.Int
-
 	// These fields are used by package eth to track
 	// inter-peer block relay.
 	ReceivedAt   time.Time
@@ -197,7 +193,7 @@ type extblock struct {
 // are ignored and set to values derived from the given txs, uncles
 // and receipts.
 func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher TrieHasher) *Block {
-	b := &Block{header: CopyHeader(header), td: new(big.Int)}
+	b := &Block{header: CopyHeader(header)}
 
 	// TODO: panic if len(txs) != len(receipts)
 	if len(txs) == 0 {

core/types/bloom9_test.go

@@ -92,7 +92,6 @@ func BenchmarkBloom9Lookup(b *testing.B) {
 }
 
 func BenchmarkCreateBloom(b *testing.B) {
-
 	var txs = Transactions{
 		NewContractCreation(1, big.NewInt(1), 1, big.NewInt(1), nil),
 		NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil),

core/types/hashing.go

@@ -31,7 +31,7 @@ var hasherPool = sync.Pool{
 	New: func() interface{} { return sha3.NewLegacyKeccak256() },
 }
 
-// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding.
+// encodeBufferPool holds temporary encoder buffers for DeriveSha and TX encoding.
 var encodeBufferPool = sync.Pool{
 	New: func() interface{} { return new(bytes.Buffer) },
 }

core/types/hashing_test.go

@@ -39,8 +39,7 @@ func TestDeriveSha(t *testing.T) {
 		t.Fatal(err)
 	}
 	for len(txs) < 1000 {
-		tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
-		exp := types.DeriveSha(txs, tr)
+		exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
 		got := types.DeriveSha(txs, trie.NewStackTrie(nil))
 		if !bytes.Equal(got[:], exp[:]) {
 			t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@@ -87,8 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
-			exp = types.DeriveSha(txs, tr)
+			exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
 		}
 	})
 
@@ -109,8 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) {
 	rndSeed := mrand.Int()
 	for i := 0; i < 10; i++ {
 		seed := rndSeed + i
-		tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
-		exp := types.DeriveSha(newDummy(i), tr)
+		exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
 		got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
 		if !bytes.Equal(got[:], exp[:]) {
 			printList(newDummy(seed))
@@ -138,8 +135,7 @@ func TestDerivableList(t *testing.T) {
 		},
 	}
 	for i, tc := range tcs[1:] {
-		tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
-		exp := types.DeriveSha(flatList(tc), tr)
+		exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
 		got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
 		if !bytes.Equal(got[:], exp[:]) {
 			t.Fatalf("case %d: got %x exp %x", i, got, exp)
@@ -201,7 +197,7 @@ func printList(l types.DerivableList) {
 	for i := 0; i < l.Len(); i++ {
 		var buf bytes.Buffer
 		l.EncodeIndex(i, &buf)
-		fmt.Printf("\"0x%x\",\n", buf.Bytes())
+		fmt.Printf("\"%#x\",\n", buf.Bytes())
 	}
 	fmt.Printf("},\n")
 }

core/types/log.go

@@ -98,8 +98,8 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error {
 	return err
 }
 
-// LogForStorage is a wrapper around a Log that flattens and parses the entire content of
-// a log including non-consensus fields.
+// LogForStorage is a wrapper around a Log that handles
+// backward compatibility with prior storage formats.
 type LogForStorage Log
 
 // EncodeRLP implements rlp.Encoder.

core/types/receipt.go

@@ -267,8 +267,8 @@ func (r *Receipt) Size() common.StorageSize {
 	return size
 }
 
-// ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
-// entire content of a receipt, as opposed to only the consensus fields originally.
+// ReceiptForStorage is a wrapper around a Receipt with RLP serialization
+// that omits the Bloom field and deserialization that re-computes it.
 type ReceiptForStorage Receipt
 
 // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt

core/types/transaction.go

@@ -432,6 +432,24 @@ func TxDifference(a, b Transactions) Transactions {
 	return keep
 }
 
+// HashDifference returns a new set which is the difference between a and b.
+func HashDifference(a, b []common.Hash) []common.Hash {
+	keep := make([]common.Hash, 0, len(a))
+
+	remove := make(map[common.Hash]struct{})
+	for _, hash := range b {
+		remove[hash] = struct{}{}
+	}
+
+	for _, hash := range a {
+		if _, ok := remove[hash]; !ok {
+			keep = append(keep, hash)
+		}
+	}
+
+	return keep
+}
+
 // TxByNonce implements the sort interface to allow sorting a list of transactions
 // by their nonces. This is usually only useful for sorting transactions from a
 // single account, otherwise a nonce comparison doesn't make much sense.

core/types/transaction_signing_test.go

@@ -111,7 +111,6 @@ func TestEIP155SigningVitalik(t *testing.T) {
 		if from != addr {
 			t.Errorf("%d: expected %x got %x", i, addr, from)
 		}
-
 	}
 }
 

core/types/transaction_test.go

@@ -114,7 +114,6 @@ func TestEIP2718TransactionSigHash(t *testing.T) {
 
 // This test checks signature operations on access list transactions.
 func TestEIP2930Signer(t *testing.T) {
-
 	var (
 		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		keyAddr = crypto.PubkeyToAddress(key.PublicKey)

core/vm/analysis.go

@@ -76,7 +76,7 @@ func codeBitmapInternal(code, bits bitvec) bitvec {
 	for pc := uint64(0); pc < uint64(len(code)); {
 		op := OpCode(code[pc])
 		pc++
-		if op < PUSH1 || op > PUSH32 {
+		if int8(op) < int8(PUSH1) { // If not PUSH (the int8(op) > int(PUSH32) is always false).
 			continue
 		}
 		numbits := op - PUSH1 + 1

core/vm/contracts.go

@@ -29,8 +29,6 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/bls12381"
 	"github.com/ethereum/go-ethereum/crypto/bn256"
 	"github.com/ethereum/go-ethereum/params"
-
-	//lint:ignore SA1019 Needed for precompile
 	"golang.org/x/crypto/ripemd160"
 )
 

core/vm/instructions_test.go

@@ -46,7 +46,6 @@ var commonParams []*twoOperandParams
 var twoOpMethods map[string]executionFunc
 
 func init() {
-
 	// Params is a list of common edgecases that should be used for some common tests
 	params := []string{
 		"0000000000000000000000000000000000000000000000000000000000000000", // 0
@@ -92,7 +91,6 @@ func init() {
 }
 
 func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) {
-
 	var (
 		env            = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{})
 		stack          = newstack()
@@ -229,32 +227,32 @@ func TestAddMod(t *testing.T) {
 	}
 }
 
-// getResult is a convenience function to generate the expected values
-func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase {
-	var (
-		env         = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{})
-		stack       = newstack()
-		pc          = uint64(0)
-		interpreter = env.interpreter
-	)
-	result := make([]TwoOperandTestcase, len(args))
-	for i, param := range args {
-		x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x))
-		y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y))
-		stack.push(x)
-		stack.push(y)
-		opFn(&pc, interpreter, &ScopeContext{nil, stack, nil})
-		actual := stack.pop()
-		result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)}
-	}
-	return result
-}
-
 // utility function to fill the json-file with testcases
 // Enable this test to generate the 'testcases_xx.json' files
 func TestWriteExpectedValues(t *testing.T) {
 	t.Skip("Enable this test to create json test cases.")
 
+	// getResult is a convenience function to generate the expected values
+	getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase {
+		var (
+			env         = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{})
+			stack       = newstack()
+			pc          = uint64(0)
+			interpreter = env.interpreter
+		)
+		result := make([]TwoOperandTestcase, len(args))
+		for i, param := range args {
+			x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x))
+			y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y))
+			stack.push(x)
+			stack.push(y)
+			opFn(&pc, interpreter, &ScopeContext{nil, stack, nil})
+			actual := stack.pop()
+			result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)}
+		}
+		return result
+	}
+
 	for name, method := range twoOpMethods {
 		data, err := json.Marshal(getResult(commonParams, method))
 		if err != nil {
@@ -641,7 +639,6 @@ func TestCreate2Addreses(t *testing.T) {
 			expected: "0xE33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0",
 		},
 	} {
-
 		origin := common.BytesToAddress(common.FromHex(tt.origin))
 		salt := common.BytesToHash(common.FromHex(tt.salt))
 		code := common.FromHex(tt.code)

core/vm/interpreter.go

@@ -114,7 +114,6 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter {
 // considered a revert-and-consume-all-gas operation except for
 // ErrExecutionReverted which means revert-and-keep-gas-left.
 func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) {
-
 	// Increment the call depth which is restricted to 1024
 	in.evm.depth++
 	defer func() { in.evm.depth-- }()

core/vm/interpreter_test.go

@@ -73,5 +73,4 @@ func TestLoopInterrupt(t *testing.T) {
 			}
 		}
 	}
-
 }

core/vm/jump_table.go

@@ -63,7 +63,7 @@ type JumpTable [256]*operation
 func validate(jt JumpTable) JumpTable {
 	for i, op := range jt {
 		if op == nil {
-			panic(fmt.Sprintf("op 0x%x is not set", i))
+			panic(fmt.Sprintf("op %#x is not set", i))
 		}
 		// The interpreter has an assumption that if the memorySize function is
 		// set, then the dynamicGas function is also set. This is a somewhat
@@ -198,7 +198,6 @@ func newSpuriousDragonInstructionSet() JumpTable {
 	instructionSet := newTangerineWhistleInstructionSet()
 	instructionSet[EXP].dynamicGas = gasExpEIP158
 	return validate(instructionSet)
-
 }
 
 // EIP 150 a.k.a Tangerine Whistle

core/vm/opcodes.go

@@ -392,7 +392,7 @@ var opCodeToString = map[OpCode]string{
 func (op OpCode) String() string {
 	str := opCodeToString[op]
 	if len(str) == 0 {
-		return fmt.Sprintf("opcode 0x%x not defined", int(op))
+		return fmt.Sprintf("opcode %#x not defined", int(op))
 	}
 
 	return str

core/vm/runtime/runtime_test.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
@@ -326,25 +325,6 @@ func TestBlockhash(t *testing.T) {
 	}
 }
 
-type stepCounter struct {
-	inner *logger.JSONLogger
-	steps int
-}
-
-func (s *stepCounter) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) {
-}
-
-func (s *stepCounter) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) {
-}
-
-func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {}
-
-func (s *stepCounter) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) {
-	s.steps++
-	// Enable this for more output
-	//s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err)
-}
-
 // benchmarkNonModifyingCode benchmarks code, but if the code modifies the
 // state, this should not be used, since it does not reset the state between runs.
 func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode string, b *testing.B) {
@@ -399,7 +379,6 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode
 // BenchmarkSimpleLoop test a pretty simple loop which loops until OOG
 // 55 ms
 func BenchmarkSimpleLoop(b *testing.B) {
-
 	staticCallIdentity := []byte{
 		byte(vm.JUMPDEST), //  [ count ]
 		// push args for the call
@@ -518,12 +497,11 @@ func TestEip2929Cases(t *testing.T) {
 	t.Skip("Test only useful for generating documentation")
 	id := 1
 	prettyPrint := func(comment string, code []byte) {
-
 		instrs := make([]string, 0)
 		it := asm.NewInstructionIterator(code)
 		for it.Next() {
 			if it.Arg() != nil && 0 < len(it.Arg()) {
-				instrs = append(instrs, fmt.Sprintf("%v 0x%x", it.Op(), it.Arg()))
+				instrs = append(instrs, fmt.Sprintf("%v %#x", it.Op(), it.Arg()))
 			} else {
 				instrs = append(instrs, fmt.Sprintf("%v", it.Op()))
 			}
@@ -531,7 +509,7 @@ func TestEip2929Cases(t *testing.T) {
 		ops := strings.Join(instrs, ", ")
 		fmt.Printf("### Case %d\n\n", id)
 		id++
-		fmt.Printf("%v\n\nBytecode: \n```\n0x%x\n```\nOperations: \n```\n%v\n```\n\n",
+		fmt.Printf("%v\n\nBytecode: \n```\n%#x\n```\nOperations: \n```\n%v\n```\n\n",
 			comment,
 			code, ops)
 		Execute(code, nil, &Config{