From 9842301376b4328421e8a6163f12766dfa6641f8 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 8 Feb 2023 19:14:34 +0800 Subject: [PATCH] all: remove database commit callback, rework noderesolver (#26637) This change ports some changes from the main PBSS PR: - get rid of callback function in `trie.Database.Commit` which is not required anymore - rework the `nodeResolver` in `trie.Iterator` to make it compatible with multiple state scheme - some other shallow changes in tests and typo-fixes --- cmd/geth/snapshot.go | 4 +- core/blockchain.go | 8 +-- core/blockchain_repair_test.go | 6 +-- core/blockchain_sethead_test.go | 2 +- core/blockchain_snapshot_test.go | 2 +- core/chain_makers.go | 2 +- core/dao_test.go | 8 +-- core/genesis.go | 2 +- core/state/iterator_test.go | 67 ++++++++++++++++++-------- core/state/pruner/pruner.go | 2 +- core/state/snapshot/generate.go | 21 ++++---- core/state/snapshot/generate_test.go | 4 +- core/state/statedb_test.go | 8 +-- core/state/sync_test.go | 2 +- eth/protocols/eth/handler_test.go | 2 +- light/postprocess.go | 4 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- trie/database.go | 11 ++--- trie/encoding_test.go | 6 +-- trie/iterator.go | 26 ++++++---- trie/iterator_test.go | 4 +- trie/stacktrie_test.go | 2 - trie/trie_test.go | 56 ++++++--------------- 23 files changed, 129 insertions(+), 122 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index ae60fb72e..7175bb953 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -165,6 +165,8 @@ block is used. } ) +// Deprecation: this command should be deprecated once the hash-based +// scheme is deprecated. func pruneState(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) defer stack.Close() @@ -433,7 +435,7 @@ func traverseRawState(ctx *cli.Context) error { nodes += 1 node := storageIter.Hash() - // Check the present for non-empty hash node(embedded node doesn't + // Check the presence for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { blob := rawdb.ReadLegacyTrieNode(chaindb, node) diff --git a/core/blockchain.go b/core/blockchain.go index ebb985e9b..c049f8955 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -945,14 +945,14 @@ func (bc *BlockChain) Stop() { recent := bc.GetBlockByNumber(number - offset) log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) - if err := triedb.Commit(recent.Root(), true, nil); err != nil { + if err := triedb.Commit(recent.Root(), true); err != nil { log.Error("Failed to commit recent state trie", "err", err) } } } if snapBase != (common.Hash{}) { log.Info("Writing snapshot state to disk", "root", snapBase) - if err := triedb.Commit(snapBase, true, nil); err != nil { + if err := triedb.Commit(snapBase, true); err != nil { log.Error("Failed to commit recent state trie", "err", err) } } @@ -1343,7 +1343,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { - return bc.triedb.Commit(root, false, nil) + return bc.triedb.Commit(root, false) } // Full but not archive node, do proper garbage collection bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive @@ -1379,7 +1379,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval, "optimum", float64(chosen-bc.lastWrite)/TriesInMemory) } // Flush an entire trie and restart the counters - bc.triedb.Commit(header.Root, true, nil) + bc.triedb.Commit(header.Root, true) bc.lastWrite = chosen bc.gcproc = 0 } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 1b3f1b718..0fdd5ae27 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1803,7 +1803,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { t.Fatalf("Failed to import canonical chain start: %v", err) } if tt.commitBlock > 0 { - chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil) + chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false) if snapshots { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { t.Fatalf("Failed to flatten snapshots: %v", err) @@ -1918,7 +1918,7 @@ func TestIssue23496(t *testing.T) { if _, err := chain.InsertChain(blocks[:1]); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } - chain.stateCache.TrieDB().Commit(blocks[0].Root(), true, nil) + chain.stateCache.TrieDB().Commit(blocks[0].Root(), false) // Insert block B2 and commit the snapshot into disk if _, err := chain.InsertChain(blocks[1:2]); err != nil { @@ -1932,7 +1932,7 @@ func TestIssue23496(t *testing.T) { if _, err := chain.InsertChain(blocks[2:3]); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } - chain.stateCache.TrieDB().Commit(blocks[2].Root(), true, nil) + chain.stateCache.TrieDB().Commit(blocks[2].Root(), false) // Insert the remaining blocks if _, err := chain.InsertChain(blocks[3:]); err != nil { diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 1750cb4e6..fa55c6252 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -2004,7 +2004,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { t.Fatalf("Failed to import canonical chain start: %v", err) } if tt.commitBlock > 0 { - chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil) + chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false) if snapshots { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { t.Fatalf("Failed to flatten snapshots: %v", err) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index b5aa7844b..110d2d1e3 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -99,7 +99,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo startPoint = point if basic.commitBlock > 0 && basic.commitBlock == point { - chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil) + chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false) } if basic.snapshotBlock > 0 && basic.snapshotBlock == point { // Flushing the entire snap tree into the disk, the diff --git a/core/chain_makers.go b/core/chain_makers.go index de63f234a..3518929f8 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -312,7 +312,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } - if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil { + if err := statedb.Database().TrieDB().Commit(root, false); err != nil { panic(fmt.Sprintf("trie write error: %v", err)) } return block, b.receipts diff --git a/core/dao_test.go b/core/dao_test.go index 632eafe4d..4ae86b50f 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } bc.Stop() @@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } bc.Stop() @@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) @@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) diff --git a/core/genesis.go b/core/genesis.go index 62096541f..5ab28001d 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -157,7 +157,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database) error { } // Commit newly generated states into disk if it's not empty. if root != types.EmptyRootHash { - if err := triedb.Commit(root, true, nil); err != nil { + if err := triedb.Commit(root, true); err != nil { return err } } diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 7669ac97a..ab06cb422 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -17,17 +17,17 @@ package state import ( - "bytes" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" ) // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { // Create some arbitrary test state to iterate db, sdb, root, _ := makeTestState() - sdb.TrieDB().Commit(root, false, nil) + sdb.TrieDB().Commit(root, false) state, err := New(root, sdb, nil) if err != nil { @@ -40,29 +40,54 @@ func TestNodeIteratorCoverage(t *testing.T) { hashes[it.Hash] = struct{}{} } } + // Check in-disk nodes + var ( + seenNodes = make(map[common.Hash]struct{}) + seenCodes = make(map[common.Hash]struct{}) + ) + it := db.NewIterator(nil, nil) + for it.Next() { + ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) + if !ok { + continue + } + seenNodes[hash] = struct{}{} + } + it.Release() + + // Check in-disk codes + it = db.NewIterator(nil, nil) + for it.Next() { + ok, hash := rawdb.IsCodeKey(it.Key()) + if !ok { + continue + } + if _, ok := hashes[common.BytesToHash(hash)]; !ok { + t.Errorf("state entry not reported %x", it.Key()) + } + seenCodes[common.BytesToHash(hash)] = struct{}{} + } + it.Release() + // Cross check the iterated hashes and the database/nodepool content for hash := range hashes { - if _, err = sdb.TrieDB().Node(hash); err != nil { - _, err = sdb.ContractCode(common.Hash{}, hash) + _, ok := seenNodes[hash] + if !ok { + _, ok = seenCodes[hash] } - if err != nil { + if !ok { t.Errorf("failed to retrieve reported node %x", hash) } } - for _, hash := range sdb.TrieDB().Nodes() { - if _, ok := hashes[hash]; !ok { - t.Errorf("state entry not reported %x", hash) - } - } - it := db.NewIterator(nil, nil) - for it.Next() { - key := it.Key() - if bytes.HasPrefix(key, []byte("secure-key-")) { - continue - } - if _, ok := hashes[common.BytesToHash(key)]; !ok { - t.Errorf("state entry not reported %x", key) - } - } - it.Release() +} + +// isTrieNode is a helper function which reports if the provided +// database entry belongs to a trie node or not. +func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { + if scheme == rawdb.HashScheme { + if len(key) == common.HashLength { + return true, common.BytesToHash(key) + } + } + return false, common.Hash{} } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index b435ab609..d1ffc4f94 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -363,7 +363,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err } headBlock := rawdb.ReadHeadBlock(db) if headBlock == nil { - return errors.New("Failed to load head block") + return errors.New("failed to load head block") } // Initialize the snapshot tree in recovery mode to handle this special case: // - Users run the `prune-state` command multiple times diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 3ed303cdf..a2be1c24b 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -359,19 +359,22 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi } // We use the snap data to build up a cache which can be used by the // main account trie as a primary lookup when resolving hashes - var snapNodeCache ethdb.Database + var resolver trie.NodeResolver if len(result.keys) > 0 { - snapNodeCache = rawdb.NewMemoryDatabase() - snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie := trie.NewEmpty(snapTrieDb) + mdb := rawdb.NewMemoryDatabase() + tdb := trie.NewDatabase(mdb) + snapTrie := trie.NewEmpty(tdb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, nodes, _ := snapTrie.Commit(false) - if nodes != nil { - snapTrieDb.Update(trie.NewWithNodeSet(nodes)) + root, nodes, err := snapTrie.Commit(false) + if err == nil && nodes != nil { + tdb.Update(trie.NewWithNodeSet(nodes)) + tdb.Commit(root, false) + } + resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte { + return rawdb.ReadTrieNode(mdb, owner, path, hash, tdb.Scheme()) } - snapTrieDb.Commit(root, false, nil) } // Construct the trie for state iteration, reuse the trie // if it's already opened with some nodes resolved. @@ -400,7 +403,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi start = time.Now() internal time.Duration ) - nodeIt.AddResolver(snapNodeCache) + nodeIt.AddResolver(resolver) for iter.Next() { if last != nil && bytes.Compare(iter.Key, last) > 0 { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index bb9e231bc..e79c919c1 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -203,7 +203,7 @@ func (t *testHelper) Commit() common.Hash { t.nodes.Merge(nodes) } t.triedb.Update(t.nodes) - t.triedb.Commit(root, false, nil) + t.triedb.Commit(root, false) return root } @@ -391,7 +391,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 // Delete an account trie leaf and ensure the generator chokes - helper.triedb.Commit(root, false, nil) + helper.triedb.Commit(root, false) helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index a39c83d2d..8aa59e3ee 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -55,7 +55,7 @@ func TestUpdateLeaks(t *testing.T) { } root := state.IntermediateRoot(false) - if err := state.Database().TrieDB().Commit(root, false, nil); err != nil { + if err := state.Database().TrieDB().Commit(root, false); err != nil { t.Errorf("can not commit trie %v to persistent database", root.Hex()) } @@ -106,7 +106,7 @@ func TestIntermediateLeaks(t *testing.T) { if err != nil { t.Fatalf("failed to commit transition state: %v", err) } - if err = transState.Database().TrieDB().Commit(transRoot, false, nil); err != nil { + if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil { t.Errorf("can not commit trie %v to persistent database", transRoot.Hex()) } @@ -114,7 +114,7 @@ func TestIntermediateLeaks(t *testing.T) { if err != nil { t.Fatalf("failed to commit final state: %v", err) } - if err = finalState.Database().TrieDB().Commit(finalRoot, false, nil); err != nil { + if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil { t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex()) } @@ -948,7 +948,7 @@ func TestFlushOrderDataLoss(t *testing.T) { if err := statedb.TrieDB().Cap(1024); err != nil { t.Fatalf("failed to cap trie dirty cache: %v", err) } - if err := statedb.TrieDB().Commit(root, false, nil); err != nil { + if err := statedb.TrieDB().Commit(root, false); err != nil { t.Fatalf("failed to commit state trie: %v", err) } // Reopen the state trie from flushed disk and verify it diff --git a/core/state/sync_test.go b/core/state/sync_test.go index d3b77da9b..84b7bf84e 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -174,7 +174,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // Create a random state to copy _, srcDb, srcRoot, srcAccounts := makeTestState() if commit { - srcDb.TrieDB().Commit(srcRoot, false, nil) + srcDb.TrieDB().Commit(srcRoot, false) } srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB()) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index b4d2574c3..51850c60e 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -111,7 +111,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, panic(err) } for _, block := range bs { - chain.StateCache().TrieDB().Commit(block.Root(), false, nil) + chain.StateCache().TrieDB().Commit(block.Root(), false) } txconfig := txpool.DefaultConfig txconfig.Journal = "" // Don't litter the disk with test journals diff --git a/light/postprocess.go b/light/postprocess.go index 181916deb..61eec3609 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -221,7 +221,7 @@ func (c *ChtIndexerBackend) Commit() error { if err := c.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { return err } - if err := c.triedb.Commit(root, false, nil); err != nil { + if err := c.triedb.Commit(root, false); err != nil { return err } } @@ -467,7 +467,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { if err := b.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { return err } - if err := b.triedb.Commit(root, false, nil); err != nil { + if err := b.triedb.Commit(root, false); err != nil { return err } } diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 6ac6feee9..1d200e9e4 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -190,7 +190,7 @@ func (f *fuzzer) fuzz() int { dbA.Update(trie.NewWithNodeSet(nodes)) } // Flush memdb -> disk (sponge) - dbA.Commit(rootA, false, nil) + dbA.Commit(rootA, false) // Stacktrie requires sorted insertion sort.Sort(vals) diff --git a/trie/database.go b/trie/database.go index 477179b27..74247d59c 100644 --- a/trie/database.go +++ b/trie/database.go @@ -632,7 +632,7 @@ func (db *Database) Cap(limit common.StorageSize) error { // // Note, this method is a non-synchronized mutator. It is unsafe to call this // concurrently with other mutators. -func (db *Database) Commit(node common.Hash, report bool, callback func(common.Hash)) error { +func (db *Database) Commit(node common.Hash, report bool) error { // Create a database batch to flush persistent data out. It is important that // outside code doesn't see an inconsistent state (referenced data removed from // memory cache during commit but not yet in persistent storage). This is ensured @@ -650,7 +650,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H nodes, storage := len(db.dirties), db.dirtiesSize uncacher := &cleaner{db} - if err := db.commit(node, batch, uncacher, callback); err != nil { + if err := db.commit(node, batch, uncacher); err != nil { log.Error("Failed to commit trie from trie database", "err", err) return err } @@ -687,7 +687,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H } // commit is the private locked version of Commit. -func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleaner, callback func(common.Hash)) error { +func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleaner) error { // If the node does not exist, it's a previously committed node node, ok := db.dirties[hash] if !ok { @@ -696,7 +696,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane var err error node.forChilds(func(child common.Hash) { if err == nil { - err = db.commit(child, batch, uncacher, callback) + err = db.commit(child, batch, uncacher) } }) if err != nil { @@ -704,9 +704,6 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane } // If we've reached an optimal batch size, commit and start over rawdb.WriteLegacyTrieNode(batch, hash, node.rlp()) - if callback != nil { - callback(hash) - } if batch.ValueSize() >= ethdb.IdealBatchSize { if err := batch.Write(); err != nil { return err diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 16393313f..e8fe4f3c6 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -78,17 +78,17 @@ func TestHexKeybytes(t *testing.T) { } func TestHexToCompactInPlace(t *testing.T) { - for i, keyS := range []string{ + for i, key := range []string{ "00", "060a040c0f000a090b040803010801010900080d090a0a0d0903000b10", "10", } { - hexBytes, _ := hex.DecodeString(keyS) + hexBytes, _ := hex.DecodeString(key) exp := hexToCompact(hexBytes) sz := hexToCompactInPlace(hexBytes) got := hexBytes[:sz] if !bytes.Equal(exp, got) { - t.Fatalf("test %d: encoding err\ninp %v\ngot %x\nexp %x\n", i, keyS, got, exp) + t.Fatalf("test %d: encoding err\ninp %v\ngot %x\nexp %x\n", i, key, got, exp) } } } diff --git a/trie/iterator.go b/trie/iterator.go index b13651fc0..aa621cd54 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -22,9 +22,15 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" ) +// NodeResolver is used for looking up trie nodes before reaching into the real +// persistent layer. This is not mandatory, rather is an optimization for cases +// where trie nodes can be recovered from some external mechanism without reading +// from disk. In those cases, this resolver allows short circuiting accesses and +// returning them from memory. +type NodeResolver func(owner common.Hash, path []byte, hash common.Hash) []byte + // Iterator is a key-value trie iterator that traverses a Trie. type Iterator struct { nodeIt NodeIterator @@ -107,8 +113,8 @@ type NodeIterator interface { // to the value after calling Next. LeafProof() [][]byte - // AddResolver sets an intermediate database to use for looking up trie nodes - // before reaching into the real persistent layer. + // AddResolver sets a node resolver to use for looking up trie nodes before + // reaching into the real persistent layer. // // This is not required for normal operation, rather is an optimization for // cases where trie nodes can be recovered from some external mechanism without @@ -118,7 +124,7 @@ type NodeIterator interface { // Before adding a similar mechanism to any other place in Geth, consider // making trie.Database an interface and wrapping at that level. It's a huge // refactor, but it could be worth it if another occurrence arises. - AddResolver(ethdb.KeyValueReader) + AddResolver(NodeResolver) } // nodeIteratorState represents the iteration state at one particular node of the @@ -137,7 +143,7 @@ type nodeIterator struct { path []byte // Path to the current node err error // Failure set in case of an internal error in the iterator - resolver ethdb.KeyValueReader // Optional intermediate resolver above the disk layer + resolver NodeResolver // optional node resolver for avoiding disk hits } // errIteratorEnd is stored in nodeIterator.err when iteration is done. @@ -165,7 +171,7 @@ func newNodeIterator(trie *Trie, start []byte) NodeIterator { return it } -func (it *nodeIterator) AddResolver(resolver ethdb.KeyValueReader) { +func (it *nodeIterator) AddResolver(resolver NodeResolver) { it.resolver = resolver } @@ -369,7 +375,7 @@ func (it *nodeIterator) peekSeek(seekKey []byte) (*nodeIteratorState, *int, []by func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { if it.resolver != nil { - if blob, err := it.resolver.Get(hash); err == nil && len(blob) > 0 { + if blob := it.resolver(it.trie.owner, path, common.BytesToHash(hash)); len(blob) > 0 { if resolved, err := decodeNode(hash, blob); err == nil { return resolved, nil } @@ -385,7 +391,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { if it.resolver != nil { - if blob, err := it.resolver.Get(hash); err == nil && len(blob) > 0 { + if blob := it.resolver(it.trie.owner, path, common.BytesToHash(hash)); len(blob) > 0 { return blob, nil } } @@ -589,7 +595,7 @@ func (it *differenceIterator) NodeBlob() []byte { return it.b.NodeBlob() } -func (it *differenceIterator) AddResolver(resolver ethdb.KeyValueReader) { +func (it *differenceIterator) AddResolver(resolver NodeResolver) { panic("not implemented") } @@ -704,7 +710,7 @@ func (it *unionIterator) NodeBlob() []byte { return (*it.items)[0].NodeBlob() } -func (it *unionIterator) AddResolver(resolver ethdb.KeyValueReader) { +func (it *unionIterator) AddResolver(resolver NodeResolver) { panic("not implemented") } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 2664dab2d..1fb6a97ea 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -337,7 +337,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { _, nodes, _ := tr.Commit(false) triedb.Update(NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(tr.Hash(), true, nil) + triedb.Commit(tr.Hash(), false) } wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil) @@ -429,7 +429,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { root, nodes, _ := ctr.Commit(false) triedb.Update(NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, true, nil) + triedb.Commit(root, false) } barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e") var ( diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 215c97cfc..3e6cc8cd5 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -255,7 +255,6 @@ func TestValLength56(t *testing.T) { func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - kvs := []struct { K string V string @@ -284,7 +283,6 @@ func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - kvs := []struct { K string V string diff --git a/trie/trie_test.go b/trie/trie_test.go index a7577040b..aa9db5063 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -86,7 +86,7 @@ func testMissingNode(t *testing.T, memonly bool) { root, nodes, _ := trie.Commit(false) triedb.Update(NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, true, nil) + triedb.Commit(root, false) } trie, _ = New(TrieID(root), triedb) @@ -791,29 +791,23 @@ func (b *spongeBatch) Reset() {} func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil } // TestCommitSequence tests that the trie.Commit operation writes the elements of the trie -// in the expected order, and calls the callbacks in the expected order. +// in the expected order. // The test data was based on the 'master' code, and is basically random. It can be used // to check whether changes to the trie modifies the write order or data in any way. func TestCommitSequence(t *testing.T) { for i, tc := range []struct { - count int - expWriteSeqHash []byte - expCallbackSeqHash []byte + count int + expWriteSeqHash []byte }{ - {20, common.FromHex("873c78df73d60e59d4a2bcf3716e8bfe14554549fea2fc147cb54129382a8066"), - common.FromHex("ff00f91ac05df53b82d7f178d77ada54fd0dca64526f537034a5dbe41b17df2a")}, - {200, common.FromHex("ba03d891bb15408c940eea5ee3d54d419595102648d02774a0268d892add9c8e"), - common.FromHex("f3cd509064c8d319bbdd1c68f511850a902ad275e6ed5bea11547e23d492a926")}, - {2000, common.FromHex("f7a184f20df01c94f09537401d11e68d97ad0c00115233107f51b9c287ce60c7"), - common.FromHex("ff795ea898ba1e4cfed4a33b4cf5535a347a02cf931f88d88719faf810f9a1c9")}, + {20, common.FromHex("873c78df73d60e59d4a2bcf3716e8bfe14554549fea2fc147cb54129382a8066")}, + {200, common.FromHex("ba03d891bb15408c940eea5ee3d54d419595102648d02774a0268d892add9c8e")}, + {2000, common.FromHex("f7a184f20df01c94f09537401d11e68d97ad0c00115233107f51b9c287ce60c7")}, } { addresses, accounts := makeAccounts(tc.count) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(rawdb.NewDatabase(s)) trie := NewEmpty(db) - // Another sponge is used to check the callback-sequence - callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements for i := 0; i < tc.count; i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -822,16 +816,10 @@ func TestCommitSequence(t *testing.T) { root, nodes, _ := trie.Commit(false) db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) - db.Commit(root, false, func(c common.Hash) { - // And spongify the callback-order - callbackSponge.Write(c[:]) - }) + db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { t.Errorf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) } - if got, exp := callbackSponge.Sum(nil), tc.expCallbackSeqHash; !bytes.Equal(got, exp) { - t.Errorf("test %d, call back sequence wrong:\ngot: %x exp %x\n", i, got, exp) - } } } @@ -839,24 +827,18 @@ func TestCommitSequence(t *testing.T) { // but uses random blobs instead of 'accounts' func TestCommitSequenceRandomBlobs(t *testing.T) { for i, tc := range []struct { - count int - expWriteSeqHash []byte - expCallbackSeqHash []byte + count int + expWriteSeqHash []byte }{ - {20, common.FromHex("8e4a01548551d139fa9e833ebc4e66fc1ba40a4b9b7259d80db32cff7b64ebbc"), - common.FromHex("450238d73bc36dc6cc6f926987e5428535e64be403877c4560e238a52749ba24")}, - {200, common.FromHex("6869b4e7b95f3097a19ddb30ff735f922b915314047e041614df06958fc50554"), - common.FromHex("0ace0b03d6cb8c0b82f6289ef5b1a1838306b455a62dafc63cada8e2924f2550")}, - {2000, common.FromHex("444200e6f4e2df49f77752f629a96ccf7445d4698c164f962bbd85a0526ef424"), - common.FromHex("117d30dafaa62a1eed498c3dfd70982b377ba2b46dd3e725ed6120c80829e518")}, + {20, common.FromHex("8e4a01548551d139fa9e833ebc4e66fc1ba40a4b9b7259d80db32cff7b64ebbc")}, + {200, common.FromHex("6869b4e7b95f3097a19ddb30ff735f922b915314047e041614df06958fc50554")}, + {2000, common.FromHex("444200e6f4e2df49f77752f629a96ccf7445d4698c164f962bbd85a0526ef424")}, } { prng := rand.New(rand.NewSource(int64(i))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(rawdb.NewDatabase(s)) trie := NewEmpty(db) - // Another sponge is used to check the callback-sequence - callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements for i := 0; i < tc.count; i++ { key := make([]byte, 32) @@ -875,16 +857,10 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { root, nodes, _ := trie.Commit(false) db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) - db.Commit(root, false, func(c common.Hash) { - // And spongify the callback-order - callbackSponge.Write(c[:]) - }) + db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { t.Fatalf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) } - if got, exp := callbackSponge.Sum(nil), tc.expCallbackSeqHash; !bytes.Equal(got, exp) { - t.Fatalf("test %d, call back sequence wrong:\ngot: %x exp %x\n", i, got, exp) - } } } @@ -920,7 +896,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) db.Update(NewWithNodeSet(nodes)) - db.Commit(root, false, nil) + db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() if err != nil { @@ -968,7 +944,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) db.Update(NewWithNodeSet(nodes)) - db.Commit(root, false, nil) + db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() if err != nil {