cerc-io/go-ethereum
18
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

Patch for concurrent iterator & others (onto v1.11.6) #386

Closed
roysc wants to merge 1565 commits from v1.11.6-statediff-v5 into master
pull from: v1.11.6-statediff-v5
merge into: cerc-io:master
Conversation 0 Commits 1565 Files Changed 1471 +578 -548
40 changed files with 578 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 22defa5af7 - Show all commits

2
cmd/geth/dbcmd.go
View File

@ -519,7 +519,7 @@ func dbDumpTrie(ctx *cli.Context) error {
return err return err
} }
} }
theTrie, err := trie.New(stRoot, trie.NewDatabase(db)) theTrie, err := trie.New(common.Hash{}, stRoot, trie.NewDatabase(db))
if err != nil { if err != nil {
return err return err
} }

8
cmd/geth/snapshot.go
View File

@ -278,7 +278,7 @@ func traverseState(ctx *cli.Context) error {
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
} }
triedb := trie.NewDatabase(chaindb) triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(root, triedb) t, err := trie.NewSecure(common.Hash{}, root, triedb)
if err != nil { if err != nil {
log.Error("Failed to open trie", "root", root, "err", err) log.Error("Failed to open trie", "root", root, "err", err)
return err return err
@ -299,7 +299,7 @@ func traverseState(ctx *cli.Context) error {
return err return err
} }
if acc.Root != emptyRoot { if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(acc.Root, triedb) storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb)
if err != nil { if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "err", err) log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
return err return err
@ -367,7 +367,7 @@ func traverseRawState(ctx *cli.Context) error {
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
} }
triedb := trie.NewDatabase(chaindb) triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(root, triedb) t, err := trie.NewSecure(common.Hash{}, root, triedb)
if err != nil { if err != nil {
log.Error("Failed to open trie", "root", root, "err", err) log.Error("Failed to open trie", "root", root, "err", err)
return err return err
@ -413,7 +413,7 @@ func traverseRawState(ctx *cli.Context) error {
return errors.New("invalid account") return errors.New("invalid account")
} }
if acc.Root != emptyRoot { if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(acc.Root, triedb) storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb)
if err != nil { if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "err", err) log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
return errors.New("missing storage trie") return errors.New("missing storage trie")

2
core/blockchain.go
View File

@ -674,7 +674,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
if block == nil { if block == nil {
return fmt.Errorf("non existent block [%x..]", hash[:4]) return fmt.Errorf("non existent block [%x..]", hash[:4])
} }
if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil { if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil {
return err return err
} }

4
core/state/database.go
View File

@ -133,7 +133,7 @@ type cachingDB struct {
// OpenTrie opens the main account trie at a specific root hash. // OpenTrie opens the main account trie at a specific root hash.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(root, db.db) tr, err := trie.NewSecure(common.Hash{}, root, db.db)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -142,7 +142,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
// OpenStorageTrie opens the storage trie of an account. // OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(root, db.db) tr, err := trie.NewSecure(addrHash, root, db.db)
if err != nil { if err != nil {
return nil, err return nil, err
} }

4
core/state/pruner/pruner.go
View File

@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil { if genesis == nil {
return errors.New("missing genesis block") return errors.New("missing genesis block")
} }
t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db)) t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db))
if err != nil { if err != nil {
return err return err
} }
@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
return err return err
} }
if acc.Root != emptyRoot { if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db)) storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db))
if err != nil { if err != nil {
return err return err
} }

8
core/state/snapshot/conversion.go
View File

@ -43,7 +43,7 @@ type trieKV struct {
type ( type (
// trieGeneratorFn is the interface of trie generation which can // trieGeneratorFn is the interface of trie generation which can
// be implemented by different trie algorithm. // be implemented by different trie algorithm.
trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash))
// leafCallbackFn is the callback invoked at the leaves of the trie, // leafCallbackFn is the callback invoked at the leaves of the trie,
// returns the subtrie root with the specified subtrie identifier. // returns the subtrie root with the specified subtrie identifier.
@ -253,7 +253,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
generatorFn(db, in, out) generatorFn(db, account, in, out)
}() }()
// Spin up a go-routine for progress logging // Spin up a go-routine for progress logging
if report && stats != nil { if report && stats != nil {
@ -360,8 +360,8 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
return stop(nil) return stop(nil)
} }
func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) {
t := trie.NewStackTrie(db) t := trie.NewStackTrieWithOwner(db, owner)
for leaf := range in { for leaf := range in {
t.TryUpdate(leaf.key[:], leaf.value) t.TryUpdate(leaf.key[:], leaf.value)
} }

18
core/state/snapshot/generate.go
View File

@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error)
// //
// The proof result will be returned if the range proving is finished, otherwise // The proof result will be returned if the range proving is finished, otherwise
// the error will be returned to abort the entire procedure. // the error will be returned to abort the entire procedure.
func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
var ( var (
keys [][]byte keys [][]byte
vals [][]byte vals [][]byte
@ -234,7 +234,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix
// The snap state is exhausted, pass the entire key/val set for verification // The snap state is exhausted, pass the entire key/val set for verification
if origin == nil && !diskMore { if origin == nil && !diskMore {
stackTr := trie.NewStackTrie(nil) stackTr := trie.NewStackTrieWithOwner(nil, owner)
for i, key := range keys { for i, key := range keys {
stackTr.TryUpdate(key, vals[i]) stackTr.TryUpdate(key, vals[i])
} }
@ -248,7 +248,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix
return &proofResult{keys: keys, vals: vals}, nil return &proofResult{keys: keys, vals: vals}, nil
} }
// Snap state is chunked, generate edge proofs for verification. // Snap state is chunked, generate edge proofs for verification.
tr, err := trie.New(root, dl.triedb) tr, err := trie.New(owner, root, dl.triedb)
if err != nil { if err != nil {
ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return nil, errMissingTrie return nil, errMissingTrie
@ -313,9 +313,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
// generateRange generates the state segment with particular prefix. Generation can // generateRange generates the state segment with particular prefix. Generation can
// either verify the correctness of existing state through range-proof and skip // either verify the correctness of existing state through range-proof and skip
// generation, or iterate trie to regenerate state on demand. // generation, or iterate trie to regenerate state on demand.
func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
// Use range prover to check the validity of the flat state in the range // Use range prover to check the validity of the flat state in the range
result, err := dl.proveRange(ctx, root, prefix, kind, origin, max, valueConvertFn) result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn)
if err != nil { if err != nil {
return false, nil, err return false, nil, err
} }
@ -363,7 +363,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref
if len(result.keys) > 0 { if len(result.keys) > 0 {
snapNodeCache = memorydb.New() snapNodeCache = memorydb.New()
snapTrieDb := trie.NewDatabase(snapNodeCache) snapTrieDb := trie.NewDatabase(snapNodeCache)
snapTrie, _ := trie.New(common.Hash{}, snapTrieDb) snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb)
for i, key := range result.keys { for i, key := range result.keys {
snapTrie.Update(key, result.vals[i]) snapTrie.Update(key, result.vals[i])
} }
@ -374,7 +374,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref
// if it's already opened with some nodes resolved. // if it's already opened with some nodes resolved.
tr := result.tr tr := result.tr
if tr == nil { if tr == nil {
tr, err = trie.New(root, dl.triedb) tr, err = trie.New(owner, root, dl.triedb)
if err != nil { if err != nil {
ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return false, nil, errMissingTrie return false, nil, errMissingTrie
@ -537,7 +537,7 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash,
// Loop for re-generating the missing storage slots. // Loop for re-generating the missing storage slots.
var origin = common.CopyBytes(storeMarker) var origin = common.CopyBytes(storeMarker)
for { for {
exhausted, last, err := dl.generateRange(ctx, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
if err != nil { if err != nil {
return err // The procedure it aborted, either by external signal or internal error. return err // The procedure it aborted, either by external signal or internal error.
} }
@ -637,7 +637,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
} }
origin := common.CopyBytes(accMarker) origin := common.CopyBytes(accMarker)
for { for {
exhausted, last, err := dl.generateRange(ctx, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
if err != nil { if err != nil {
return err // The procedure it aborted, either by external signal or internal error. return err // The procedure it aborted, either by external signal or internal error.
} }

480
core/state/snapshot/generate_test.go
View File

@ -26,61 +26,12 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"golang.org/x/crypto/sha3" "golang.org/x/crypto/sha3"
) )
// Tests that snapshot generation from an empty database.
func TestGeneration(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
triedb.Commit(root, false, nil)
if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
t.Fatalf("have %#x want %#x", have, want)
}
snap := generateSnapshot(diskdb, triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats)
snap.genAbort <- stop
<-stop
}
func hashData(input []byte) common.Hash { func hashData(input []byte) common.Hash {
var hasher = sha3.NewLegacyKeccak256() var hasher = sha3.NewLegacyKeccak256()
var hash common.Hash var hash common.Hash
@ -90,48 +41,25 @@ func hashData(input []byte) common.Hash {
return hash return hash
} }
// Tests that snapshot generation with existent flat state. // Tests that snapshot generation from an empty database.
func TestGenerateExistentState(t *testing.T) { func TestGeneration(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
var ( var helper = newHelper()
diskdb = memorydb.New() stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
accTrie, _ := trie.NewSecure(common.Hash{}, triedb) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
val, _ := rlp.EncodeToBytes(acc) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val)
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3"))
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
val, _ = rlp.EncodeToBytes(acc) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
diskdb.Put(hashData([]byte("acc-2")).Bytes(), val)
rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val)
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} root, snap := helper.CommitAndGenerate()
val, _ = rlp.EncodeToBytes(acc) if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 t.Fatalf("have %#x want %#x", have, want)
rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val) }
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3"))
root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -140,6 +68,43 @@ func TestGenerateExistentState(t *testing.T) {
t.Errorf("Snapshot generation failed") t.Errorf("Snapshot generation failed")
} }
checkSnapRoot(t, snap, root) checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats)
snap.genAbort <- stop
<-stop
}
// Tests that snapshot generation with existent flat state.
func TestGenerateExistentState(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down // Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats) stop := make(chan *generatorStats)
snap.genAbort <- stop snap.genAbort <- stop
@ -163,7 +128,6 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
} }
return hash, nil return hash, nil
}, newGenerateStats(), true) }, newGenerateStats(), true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -176,15 +140,15 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
} }
type testHelper struct { type testHelper struct {
diskdb *memorydb.Database diskdb ethdb.Database
triedb *trie.Database triedb *trie.Database
accTrie *trie.SecureTrie accTrie *trie.SecureTrie
} }
func newHelper() *testHelper { func newHelper() *testHelper {
diskdb := memorydb.New() diskdb := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(diskdb) triedb := trie.NewDatabase(diskdb)
accTrie, _ := trie.NewSecure(common.Hash{}, triedb) accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
return &testHelper{ return &testHelper{
diskdb: diskdb, diskdb: diskdb,
triedb: triedb, triedb: triedb,
@ -215,18 +179,28 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string)
} }
} }
func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte {
stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb) stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb)
for i, k := range keys { for i, k := range keys {
stTrie.Update([]byte(k), []byte(vals[i])) stTrie.Update([]byte(k), []byte(vals[i]))
} }
root, _, _ := stTrie.Commit(nil) var root common.Hash
if !commit {
root = stTrie.Hash()
} else {
root, _, _ = stTrie.Commit(nil)
}
return root.Bytes() return root.Bytes()
} }
func (t *testHelper) Generate() (common.Hash, *diskLayer) { func (t *testHelper) Commit() common.Hash {
root, _, _ := t.accTrie.Commit(nil) root, _, _ := t.accTrie.Commit(nil)
t.triedb.Commit(root, false, nil) t.triedb.Commit(root, false, nil)
return root
}
func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
root := t.Commit()
snap := generateSnapshot(t.diskdb, t.triedb, 16, root) snap := generateSnapshot(t.diskdb, t.triedb, 16, root)
return root, snap return root, snap
} }
@ -249,26 +223,29 @@ func (t *testHelper) Generate() (common.Hash, *diskLayer) {
// - extra slots in the end // - extra slots in the end
func TestGenerateExistentStateWithWrongStorage(t *testing.T) { func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
helper := newHelper() helper := newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Account one, empty root but non-empty database // Account one, empty root but non-empty database
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Account two, non empty root but empty database // Account two, non empty root but empty database
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
// Miss slots // Miss slots
{ {
// Account three, non empty root but misses slots in the beginning // Account three, non empty root but misses slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"})
// Account four, non empty root but misses slots in the middle // Account four, non empty root but misses slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"})
// Account five, non empty root but misses slots in the end // Account five, non empty root but misses slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"})
} }
@ -276,18 +253,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// Wrong storage slots // Wrong storage slots
{ {
// Account six, non empty root but wrong slots in the beginning // Account six, non empty root but wrong slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"})
// Account seven, non empty root but wrong slots in the middle // Account seven, non empty root but wrong slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"})
// Account eight, non empty root but wrong slots in the end // Account eight, non empty root but wrong slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"})
// Account 9, non empty root but rotated slots // Account 9, non empty root but rotated slots
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"})
} }
@ -295,19 +276,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// Extra storage slots // Extra storage slots
{ {
// Account 10, non empty root but extra slots in the beginning // Account 10, non empty root but extra slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"})
// Account 11, non empty root but extra slots in the middle // Account 11, non empty root but extra slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"})
// Account 12, non empty root but extra slots in the end // Account 12, non empty root but extra slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"})
} }
root, snap := helper.Generate() root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0
select { select {
@ -331,7 +315,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// - extra accounts // - extra accounts
func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper := newHelper() helper := newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
// Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6]
// Extra accounts [acc-0, acc-5, acc-7] // Extra accounts [acc-0, acc-5, acc-7]
@ -359,7 +348,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end
} }
root, snap := helper.Generate() root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8
select { select {
@ -383,29 +372,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// without any storage slots to keep the test smaller. // without any storage slots to keep the test smaller.
var ( helper := newHelper()
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
tr, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
val, _ = rlp.EncodeToBytes(acc) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
val, _ = rlp.EncodeToBytes(acc)
tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
tr.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
// Delete an account trie leaf and ensure the generator chokes // Delete an account trie leaf and ensure the generator chokes
triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil) helper.triedb.Commit(root, false, nil)
diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978")) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -427,45 +406,30 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
var ( helper := newHelper()
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
accTrie, _ := trie.NewSecure(common.Hash{}, triedb) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
val, _ := rlp.EncodeToBytes(acc) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} root, _, _ := helper.accTrie.Commit(nil)
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
// We can only corrupt the disk database, so flush the tries out // We can only corrupt the disk database, so flush the tries out
triedb.Reference( helper.triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), common.BytesToHash(stRoot),
common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
) )
triedb.Reference( helper.triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), common.BytesToHash(stRoot),
common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
) )
triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) helper.triedb.Commit(root, false, nil)
// Delete a storage trie root and ensure the generator chokes // Delete a storage trie root and ensure the generator chokes
diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes()) helper.diskdb.Delete(stRoot)
snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -486,45 +450,31 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
var ( helper := newHelper()
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
accTrie, _ := trie.NewSecure(common.Hash{}, triedb) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
val, _ := rlp.EncodeToBytes(acc) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} root, _, _ := helper.accTrie.Commit(nil)
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
// We can only corrupt the disk database, so flush the tries out // We can only corrupt the disk database, so flush the tries out
triedb.Reference( helper.triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), common.BytesToHash(stRoot),
common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
) )
triedb.Reference( helper.triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), common.BytesToHash(stRoot),
common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
) )
triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) helper.triedb.Commit(root, false, nil)
// Delete a storage trie leaf and ensure the generator chokes // Delete a storage trie leaf and ensure the generator chokes
diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -539,56 +489,51 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
<-stop <-stop
} }
func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie {
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
for i := 0; i < n; i++ {
k := fmt.Sprintf("key-%d", i)
v := fmt.Sprintf("val-%d", i)
stTrie.Update([]byte(k), []byte(v))
}
stTrie.Commit(nil)
return stTrie
}
// Tests that snapshot generation when an extra account with storage exists in the snap state. // Tests that snapshot generation when an extra account with storage exists in the snap state.
func TestGenerateWithExtraAccounts(t *testing.T) { func TestGenerateWithExtraAccounts(t *testing.T) {
var ( helper := newHelper()
diskdb = memorydb.New() {
triedb = trie.NewDatabase(diskdb) // Account one in the trie
stTrie = getStorageTrie(5, triedb) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
) []string{"key-1", "key-2", "key-3", "key-4", "key-5"},
accTrie, _ := trie.NewSecure(common.Hash{}, triedb) []string{"val-1", "val-2", "val-3", "val-4", "val-5"},
{ // Account one in the trie true,
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} )
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
// Identical in the snap // Identical in the snap
key := hashData([]byte("acc-1")) key := hashData([]byte("acc-1"))
rawdb.WriteAccountSnapshot(diskdb, key, val) rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5"))
} }
{ // Account two exists only in the snapshot {
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} // Account two exists only in the snapshot
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")),
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true,
)
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte("acc-2")) key := hashData([]byte("acc-2"))
rawdb.WriteAccountSnapshot(diskdb, key, val) rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3"))
} }
root, _, _ := accTrie.Commit(nil) root := helper.Commit()
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
// To verify the test: If we now inspect the snap db, there should exist extraneous storage items // To verify the test: If we now inspect the snap db, there should exist extraneous storage items
if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
t.Fatalf("expected snap storage to exist") t.Fatalf("expected snap storage to exist")
} }
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
snap := generateSnapshot(diskdb, triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -597,12 +542,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
t.Errorf("Snapshot generation failed") t.Errorf("Snapshot generation failed")
} }
checkSnapRoot(t, snap, root) checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down // Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats) stop := make(chan *generatorStats)
snap.genAbort <- stop snap.genAbort <- stop
<-stop <-stop
// If we now inspect the snap db, there should exist no extraneous storage items // If we now inspect the snap db, there should exist no extraneous storage items
if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
t.Fatalf("expected slot to be removed, got %v", string(data)) t.Fatalf("expected slot to be removed, got %v", string(data))
} }
} }
@ -616,37 +562,36 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
if false { if false {
enableLogging() enableLogging()
} }
var ( helper := newHelper()
diskdb = memorydb.New() {
triedb = trie.NewDatabase(diskdb) // Account one in the trie
stTrie = getStorageTrie(3, triedb) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
) []string{"key-1", "key-2", "key-3"},
accTrie, _ := trie.NewSecure(common.Hash{}, triedb) []string{"val-1", "val-2", "val-3"},
{ // Account one in the trie true,
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} )
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
// Identical in the snap // Identical in the snap
key := hashData([]byte("acc-1")) key := hashData([]byte("acc-1"))
rawdb.WriteAccountSnapshot(diskdb, key, val) rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
} }
{ // 100 accounts exist only in snapshot {
// 100 accounts exist only in snapshot
for i := 0; i < 1000; i++ { for i := 0; i < 1000; i++ {
//acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} //acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte(fmt.Sprintf("acc-%d", i))) key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
rawdb.WriteAccountSnapshot(diskdb, key, val) rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
} }
} }
root, _, _ := accTrie.Commit(nil) root, snap := helper.CommitAndGenerate()
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -675,31 +620,22 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
if false { if false {
enableLogging() enableLogging()
} }
var ( helper := newHelper()
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
accTrie, _ := trie.New(common.Hash{}, triedb)
{ {
acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
accTrie.Update(common.HexToHash("0x03").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
accTrie.Update(common.HexToHash("0x07").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val)
} }
root, snap := helper.CommitAndGenerate()
root, _, _ := accTrie.Commit(nil)
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -721,29 +657,20 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
if false { if false {
enableLogging() enableLogging()
} }
var ( helper := newHelper()
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
accTrie, _ := trie.New(common.Hash{}, triedb)
{ {
acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
accTrie.Update(common.HexToHash("0x03").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
junk := make([]byte, 100) junk := make([]byte, 100)
copy(junk, []byte{0xde, 0xad}) copy(junk, []byte{0xde, 0xad})
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk)
} }
root, snap := helper.CommitAndGenerate()
root, _, _ := accTrie.Commit(nil)
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -757,7 +684,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
snap.genAbort <- stop snap.genAbort <- stop
<-stop <-stop
// If we now inspect the snap db, there should exist no extraneous storage items // If we now inspect the snap db, there should exist no extraneous storage items
if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
t.Fatalf("expected slot to be removed, got %v", string(data)) t.Fatalf("expected slot to be removed, got %v", string(data))
} }
} }
@ -767,13 +694,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
accountCheckRange = 10 accountCheckRange = 10
storageCheckRange = 20 storageCheckRange = 20
helper := newHelper() helper := newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Add 1K accounts to the trie // Add 1K accounts to the trie
for i := 0; i < 400; i++ { for i := 0; i < 400; i++ {
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount(fmt.Sprintf("acc-%d", i), helper.addTrieAccount(fmt.Sprintf("acc-%d", i),
&Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
} }
root, snap := helper.Generate() root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
select { select {
@ -802,12 +729,12 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
helper := newHelper() helper := newHelper()
stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
stRoot := helper.makeStorageTrie(stKeys, stVals)
// We add 8 accounts, each one is missing exactly one of the storage slots. This means // We add 8 accounts, each one is missing exactly one of the storage slots. This means
// we don't have to order the keys and figure out exactly which hash-key winds up // we don't have to order the keys and figure out exactly which hash-key winds up
// on the sensitive spots at the boundaries // on the sensitive spots at the boundaries
for i := 0; i < 8; i++ { for i := 0; i < 8; i++ {
accKey := fmt.Sprintf("acc-%d", i) accKey := fmt.Sprintf("acc-%d", i)
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true)
helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()})
var moddedKeys []string var moddedKeys []string
var moddedVals []string var moddedVals []string
@ -819,8 +746,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
} }
helper.addSnapStorage(accKey, moddedKeys, moddedVals) helper.addSnapStorage(accKey, moddedKeys, moddedVals)
} }
root, snap := helper.CommitAndGenerate()
root, snap := helper.Generate()
t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff
select { select {
@ -899,7 +825,7 @@ func populateDangling(disk ethdb.KeyValueStore) {
// This test will populate some dangling storages to see if they can be cleaned up. // This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper() var helper = newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
@ -910,7 +836,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
populateDangling(helper.diskdb) populateDangling(helper.diskdb)
root, snap := helper.Generate() root, snap := helper.CommitAndGenerate()
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded
@ -932,7 +858,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
// This test will populate some dangling storages to see if they can be cleaned up. // This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper() var helper = newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
@ -940,7 +866,7 @@ func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
populateDangling(helper.diskdb) populateDangling(helper.diskdb)
root, snap := helper.Generate() root, snap := helper.CommitAndGenerate()
select { select {
case <-snap.genPending: case <-snap.genPending:
// Snapshot generation succeeded // Snapshot generation succeeded

6
core/state/state_object.go
View File

@ -154,7 +154,7 @@ func (s *stateObject) getTrie(db Database) Trie {
if s.data.Root != emptyRoot && s.db.prefetcher != nil { if s.data.Root != emptyRoot && s.db.prefetcher != nil {
// When the miner is creating the pending state, there is no // When the miner is creating the pending state, there is no
// prefetcher // prefetcher
s.trie = s.db.prefetcher.trie(s.data.Root) s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
} }
if s.trie == nil { if s.trie == nil {
var err error var err error
@ -295,7 +295,7 @@ func (s *stateObject) finalise(prefetch bool) {
} }
} }
if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot { if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot {
s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch) s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch)
} }
if len(s.dirtyStorage) > 0 { if len(s.dirtyStorage) > 0 {
s.dirtyStorage = make(Storage) s.dirtyStorage = make(Storage)
@ -352,7 +352,7 @@ func (s *stateObject) updateTrie(db Database) Trie {
usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure
} }
if s.db.prefetcher != nil { if s.db.prefetcher != nil {
s.db.prefetcher.used(s.data.Root, usedStorage) s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage)
} }
if len(s.pendingStorage) > 0 { if len(s.pendingStorage) > 0 {
s.pendingStorage = make(Storage) s.pendingStorage = make(Storage)

21
core/state/statedb.go
View File

@ -62,11 +62,14 @@ func (n *proofList) Delete(key []byte) error {
// * Contracts // * Contracts
// * Accounts // * Accounts
type StateDB struct { type StateDB struct {
db Database db Database
prefetcher *triePrefetcher prefetcher *triePrefetcher
originalRoot common.Hash // The pre-state root, before any changes were made trie Trie
trie Trie hasher crypto.KeccakState
hasher crypto.KeccakState
// originalRoot is the pre-state root, before any changes were made.
// It will be updated when the Commit is called.
originalRoot common.Hash
snaps *snapshot.Tree snaps *snapshot.Tree
snap snapshot.Snapshot snap snapshot.Snapshot
@ -648,6 +651,7 @@ func (s *StateDB) Copy() *StateDB {
state := &StateDB{ state := &StateDB{
db: s.db, db: s.db,
trie: s.db.CopyTrie(s.trie), trie: s.db.CopyTrie(s.trie),
originalRoot: s.originalRoot,
stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)),
stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)),
stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)),
@ -810,7 +814,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) {
addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure
} }
if s.prefetcher != nil && len(addressesToPrefetch) > 0 { if s.prefetcher != nil && len(addressesToPrefetch) > 0 {
s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch) s.prefetcher.prefetch(common.Hash{}, s.originalRoot, addressesToPrefetch)
} }
// Invalidate journal because reverting across transactions is not allowed. // Invalidate journal because reverting across transactions is not allowed.
s.clearJournalAndRefund() s.clearJournalAndRefund()
@ -851,7 +855,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
// _untouched_. We can check with the prefetcher, if it can give us a trie // _untouched_. We can check with the prefetcher, if it can give us a trie
// which has the same root, but also has some content loaded into it. // which has the same root, but also has some content loaded into it.
if prefetcher != nil { if prefetcher != nil {
if trie := prefetcher.trie(s.originalRoot); trie != nil { if trie := prefetcher.trie(common.Hash{}, s.originalRoot); trie != nil {
s.trie = trie s.trie = trie
} }
} }
@ -867,7 +871,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure
} }
if prefetcher != nil { if prefetcher != nil {
prefetcher.used(s.originalRoot, usedAddrs) prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs)
} }
if len(s.stateObjectsPending) > 0 { if len(s.stateObjectsPending) > 0 {
s.stateObjectsPending = make(map[common.Address]struct{}) s.stateObjectsPending = make(map[common.Address]struct{})
@ -980,6 +984,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
} }
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
} }
s.originalRoot = root
return root, err return root, err
} }

6
core/state/sync_test.go
View File

@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
if v, _ := db.Get(root[:]); v == nil { if v, _ := db.Get(root[:]); v == nil {
return nil // Consider a non existent state consistent. return nil // Consider a non existent state consistent.
} }
trie, err := trie.New(root, trie.NewDatabase(db)) trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db))
if err != nil { if err != nil {
return err return err
} }
@ -166,7 +166,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
if commit { if commit {
srcDb.TrieDB().Commit(srcRoot, false, nil) srcDb.TrieDB().Commit(srcRoot, false, nil)
} }
srcTrie, _ := trie.New(srcRoot, srcDb.TrieDB()) srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB())
// Create a destination state and sync with the scheduler // Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase() dstDb := rawdb.NewMemoryDatabase()
@ -207,7 +207,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil { if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil {
t.Fatalf("failed to decode account on path %x: %v", path, err) t.Fatalf("failed to decode account on path %x: %v", path, err)
} }
stTrie, err := trie.New(acc.Root, srcDb.TrieDB()) stTrie, err := trie.New(common.BytesToHash(path[0]), acc.Root, srcDb.TrieDB())
if err != nil { if err != nil {
t.Fatalf("failed to retriev storage trie for path %x: %v", path, err) t.Fatalf("failed to retriev storage trie for path %x: %v", path, err)
} }

88
core/state/trie_prefetcher.go
View File

@ -25,7 +25,7 @@ import (
) )
var ( var (
// triePrefetchMetricsPrefix is the prefix under which to publis the metrics. // triePrefetchMetricsPrefix is the prefix under which to publish the metrics.
triePrefetchMetricsPrefix = "trie/prefetch/" triePrefetchMetricsPrefix = "trie/prefetch/"
) )
@ -35,10 +35,10 @@ var (
// //
// Note, the prefetcher's API is not thread safe. // Note, the prefetcher's API is not thread safe.
type triePrefetcher struct { type triePrefetcher struct {
db Database // Database to fetch trie nodes through db Database // Database to fetch trie nodes through
root common.Hash // Root hash of theaccount trie for metrics root common.Hash // Root hash of the account trie for metrics
fetches map[common.Hash]Trie // Partially or fully fetcher tries fetches map[string]Trie // Partially or fully fetcher tries
fetchers map[common.Hash]*subfetcher // Subfetchers for each trie fetchers map[string]*subfetcher // Subfetchers for each trie
deliveryMissMeter metrics.Meter deliveryMissMeter metrics.Meter
accountLoadMeter metrics.Meter accountLoadMeter metrics.Meter
@ -51,13 +51,12 @@ type triePrefetcher struct {
storageWasteMeter metrics.Meter storageWasteMeter metrics.Meter
} }
// newTriePrefetcher
func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher {
prefix := triePrefetchMetricsPrefix + namespace prefix := triePrefetchMetricsPrefix + namespace
p := &triePrefetcher{ p := &triePrefetcher{
db: db, db: db,
root: root, root: root,
fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map
deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil),
accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil),
@ -112,7 +111,7 @@ func (p *triePrefetcher) copy() *triePrefetcher {
copy := &triePrefetcher{ copy := &triePrefetcher{
db: p.db, db: p.db,
root: p.root, root: p.root,
fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map fetches: make(map[string]Trie), // Active prefetchers use the fetches map
deliveryMissMeter: p.deliveryMissMeter, deliveryMissMeter: p.deliveryMissMeter,
accountLoadMeter: p.accountLoadMeter, accountLoadMeter: p.accountLoadMeter,
@ -132,33 +131,35 @@ func (p *triePrefetcher) copy() *triePrefetcher {
return copy return copy
} }
// Otherwise we're copying an active fetcher, retrieve the current states // Otherwise we're copying an active fetcher, retrieve the current states
for root, fetcher := range p.fetchers { for id, fetcher := range p.fetchers {
copy.fetches[root] = fetcher.peek() copy.fetches[id] = fetcher.peek()
} }
return copy return copy
} }
// prefetch schedules a batch of trie items to prefetch. // prefetch schedules a batch of trie items to prefetch.
func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) { func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) {
// If the prefetcher is an inactive one, bail out // If the prefetcher is an inactive one, bail out
if p.fetches != nil { if p.fetches != nil {
return return
} }
// Active fetcher, schedule the retrievals // Active fetcher, schedule the retrievals
fetcher := p.fetchers[root] id := p.trieID(owner, root)
fetcher := p.fetchers[id]
if fetcher == nil { if fetcher == nil {
fetcher = newSubfetcher(p.db, root) fetcher = newSubfetcher(p.db, owner, root)
p.fetchers[root] = fetcher p.fetchers[id] = fetcher
} }
fetcher.schedule(keys) fetcher.schedule(keys)
} }
// trie returns the trie matching the root hash, or nil if the prefetcher doesn't // trie returns the trie matching the root hash, or nil if the prefetcher doesn't
// have it. // have it.
func (p *triePrefetcher) trie(root common.Hash) Trie { func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie {
// If the prefetcher is inactive, return from existing deep copies // If the prefetcher is inactive, return from existing deep copies
id := p.trieID(owner, root)
if p.fetches != nil { if p.fetches != nil {
trie := p.fetches[root] trie := p.fetches[id]
if trie == nil { if trie == nil {
p.deliveryMissMeter.Mark(1) p.deliveryMissMeter.Mark(1)
return nil return nil
@ -166,7 +167,7 @@ func (p *triePrefetcher) trie(root common.Hash) Trie {
return p.db.CopyTrie(trie) return p.db.CopyTrie(trie)
} }
// Otherwise the prefetcher is active, bail if no trie was prefetched for this root // Otherwise the prefetcher is active, bail if no trie was prefetched for this root
fetcher := p.fetchers[root] fetcher := p.fetchers[id]
if fetcher == nil { if fetcher == nil {
p.deliveryMissMeter.Mark(1) p.deliveryMissMeter.Mark(1)
return nil return nil
@ -185,20 +186,26 @@ func (p *triePrefetcher) trie(root common.Hash) Trie {
// used marks a batch of state items used to allow creating statistics as to // used marks a batch of state items used to allow creating statistics as to
// how useful or wasteful the prefetcher is. // how useful or wasteful the prefetcher is.
func (p *triePrefetcher) used(root common.Hash, used [][]byte) { func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) {
if fetcher := p.fetchers[root]; fetcher != nil { if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil {
fetcher.used = used fetcher.used = used
} }
} }
// trieID returns an unique trie identifier consists the trie owner and root hash.
func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string {
return string(append(owner.Bytes(), root.Bytes()...))
}
// subfetcher is a trie fetcher goroutine responsible for pulling entries for a // subfetcher is a trie fetcher goroutine responsible for pulling entries for a
// single trie. It is spawned when a new root is encountered and lives until the // single trie. It is spawned when a new root is encountered and lives until the
// main prefetcher is paused and either all requested items are processed or if // main prefetcher is paused and either all requested items are processed or if
// the trie being worked on is retrieved from the prefetcher. // the trie being worked on is retrieved from the prefetcher.
type subfetcher struct { type subfetcher struct {
db Database // Database to load trie nodes through db Database // Database to load trie nodes through
root common.Hash // Root hash of the trie to prefetch owner common.Hash // Owner of the trie, usually account hash
trie Trie // Trie being populated with nodes root common.Hash // Root hash of the trie to prefetch
trie Trie // Trie being populated with nodes
tasks [][]byte // Items queued up for retrieval tasks [][]byte // Items queued up for retrieval
lock sync.Mutex // Lock protecting the task queue lock sync.Mutex // Lock protecting the task queue
@ -215,15 +222,16 @@ type subfetcher struct {
// newSubfetcher creates a goroutine to prefetch state items belonging to a // newSubfetcher creates a goroutine to prefetch state items belonging to a
// particular root hash. // particular root hash.
func newSubfetcher(db Database, root common.Hash) *subfetcher { func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher {
sf := &subfetcher{ sf := &subfetcher{
db: db, db: db,
root: root, owner: owner,
wake: make(chan struct{}, 1), root: root,
stop: make(chan struct{}), wake: make(chan struct{}, 1),
term: make(chan struct{}), stop: make(chan struct{}),
copy: make(chan chan Trie), term: make(chan struct{}),
seen: make(map[string]struct{}), copy: make(chan chan Trie),
seen: make(map[string]struct{}),
} }
go sf.loop() go sf.loop()
return sf return sf
@ -279,13 +287,21 @@ func (sf *subfetcher) loop() {
defer close(sf.term) defer close(sf.term)
// Start by opening the trie and stop processing if it fails // Start by opening the trie and stop processing if it fails
trie, err := sf.db.OpenTrie(sf.root) if sf.owner == (common.Hash{}) {
if err != nil { trie, err := sf.db.OpenTrie(sf.root)
log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) if err != nil {
return log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
return
}
sf.trie = trie
} else {
trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root)
if err != nil {
log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
return
}
sf.trie = trie
} }
sf.trie = trie
// Trie opened successfully, keep prefetching items // Trie opened successfully, keep prefetching items
for { for {
select { select {

36
core/state/trie_prefetcher_test.go
View File

@ -47,20 +47,20 @@ func TestCopyAndClose(t *testing.T) {
db := filledStateDB() db := filledStateDB()
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
skey := common.HexToHash("aaa") skey := common.HexToHash("aaa")
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
a := prefetcher.trie(db.originalRoot) a := prefetcher.trie(common.Hash{}, db.originalRoot)
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
b := prefetcher.trie(db.originalRoot) b := prefetcher.trie(common.Hash{}, db.originalRoot)
cpy := prefetcher.copy() cpy := prefetcher.copy()
cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
c := cpy.trie(db.originalRoot) c := cpy.trie(common.Hash{}, db.originalRoot)
prefetcher.close() prefetcher.close()
cpy2 := cpy.copy() cpy2 := cpy.copy()
cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
d := cpy2.trie(db.originalRoot) d := cpy2.trie(common.Hash{}, db.originalRoot)
cpy.close() cpy.close()
cpy2.close() cpy2.close()
if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() {
@ -72,10 +72,10 @@ func TestUseAfterClose(t *testing.T) {
db := filledStateDB() db := filledStateDB()
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
skey := common.HexToHash("aaa") skey := common.HexToHash("aaa")
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
a := prefetcher.trie(db.originalRoot) a := prefetcher.trie(common.Hash{}, db.originalRoot)
prefetcher.close() prefetcher.close()
b := prefetcher.trie(db.originalRoot) b := prefetcher.trie(common.Hash{}, db.originalRoot)
if a == nil { if a == nil {
t.Fatal("Prefetching before close should not return nil") t.Fatal("Prefetching before close should not return nil")
} }
@ -88,13 +88,13 @@ func TestCopyClose(t *testing.T) {
db := filledStateDB() db := filledStateDB()
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
skey := common.HexToHash("aaa") skey := common.HexToHash("aaa")
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
cpy := prefetcher.copy() cpy := prefetcher.copy()
a := prefetcher.trie(db.originalRoot) a := prefetcher.trie(common.Hash{}, db.originalRoot)
b := cpy.trie(db.originalRoot) b := cpy.trie(common.Hash{}, db.originalRoot)
prefetcher.close() prefetcher.close()
c := prefetcher.trie(db.originalRoot) c := prefetcher.trie(common.Hash{}, db.originalRoot)
d := cpy.trie(db.originalRoot) d := cpy.trie(common.Hash{}, db.originalRoot)
if a == nil { if a == nil {
t.Fatal("Prefetching before close should not return nil") t.Fatal("Prefetching before close should not return nil")
} }

12
core/types/hashing_test.go
View File

@ -39,8 +39,7 @@ func TestDeriveSha(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
for len(txs) < 1000 { for len(txs) < 1000 {
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp := types.DeriveSha(txs, tr)
got := types.DeriveSha(txs, trie.NewStackTrie(nil)) got := types.DeriveSha(txs, trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) { if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@ -87,8 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) {
b.ResetTimer() b.ResetTimer()
b.ReportAllocs() b.ReportAllocs()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp = types.DeriveSha(txs, tr)
} }
}) })
@ -109,8 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) {
rndSeed := mrand.Int() rndSeed := mrand.Int()
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
seed := rndSeed + i seed := rndSeed + i
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp := types.DeriveSha(newDummy(i), tr)
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) { if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed)) printList(newDummy(seed))
@ -138,8 +135,7 @@ func TestDerivableList(t *testing.T) {
}, },
} }
for i, tc := range tcs[1:] { for i, tc := range tcs[1:] {
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp := types.DeriveSha(flatList(tc), tr)
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) { if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp) t.Fatalf("case %d: got %x exp %x", i, got, exp)

4
eth/api.go
View File

@ -529,11 +529,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
} }
triedb := api.eth.BlockChain().StateCache().TrieDB() triedb := api.eth.BlockChain().StateCache().TrieDB()
oldTrie, err := trie.NewSecure(startBlock.Root(), triedb) oldTrie, err := trie.NewSecure(common.Hash{}, startBlock.Root(), triedb)
if err != nil { if err != nil {
return nil, err return nil, err
} }
newTrie, err := trie.NewSecure(endBlock.Root(), triedb) newTrie, err := trie.NewSecure(common.Hash{}, endBlock.Root(), triedb)
if err != nil { if err != nil {
return nil, err return nil, err
} }

10
eth/protocols/snap/handler.go
View File

@ -285,7 +285,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac
req.Bytes = softResponseLimit req.Bytes = softResponseLimit
} }
// Retrieve the requested state and bail out if non existent // Retrieve the requested state and bail out if non existent
tr, err := trie.New(req.Root, chain.StateCache().TrieDB()) tr, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB())
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
@ -415,7 +415,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
if origin != (common.Hash{}) || (abort && len(storage) > 0) { if origin != (common.Hash{}) || (abort && len(storage) > 0) {
// Request started at a non-zero hash or was capped prematurely, add // Request started at a non-zero hash or was capped prematurely, add
// the endpoint Merkle proofs // the endpoint Merkle proofs
accTrie, err := trie.New(req.Root, chain.StateCache().TrieDB()) accTrie, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB())
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
@ -423,7 +423,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil {
return nil, nil return nil, nil
} }
stTrie, err := trie.New(acc.Root, chain.StateCache().TrieDB()) stTrie, err := trie.New(account, acc.Root, chain.StateCache().TrieDB())
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
@ -489,7 +489,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
// Make sure we have the state associated with the request // Make sure we have the state associated with the request
triedb := chain.StateCache().TrieDB() triedb := chain.StateCache().TrieDB()
accTrie, err := trie.NewSecure(req.Root, triedb) accTrie, err := trie.NewSecure(common.Hash{}, req.Root, triedb)
if err != nil { if err != nil {
// We don't have the requested state available, bail out // We don't have the requested state available, bail out
return nil, nil return nil, nil
@ -531,7 +531,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
if err != nil || account == nil { if err != nil || account == nil {
break break
} }
stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) stTrie, err := trie.NewSecure(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb)
loads++ // always account database reads, even for failures loads++ // always account database reads, even for failures
if err != nil { if err != nil {
break break

10
eth/protocols/snap/sync.go
View File

@ -712,7 +712,7 @@ func (s *Syncer) loadSyncStatus() {
} }
task.genTrie = trie.NewStackTrie(task.genBatch) task.genTrie = trie.NewStackTrie(task.genBatch)
for _, subtasks := range task.SubTasks { for accountHash, subtasks := range task.SubTasks {
for _, subtask := range subtasks { for _, subtask := range subtasks {
subtask.genBatch = ethdb.HookedBatch{ subtask.genBatch = ethdb.HookedBatch{
Batch: s.db.NewBatch(), Batch: s.db.NewBatch(),
@ -720,7 +720,7 @@ func (s *Syncer) loadSyncStatus() {
s.storageBytes += common.StorageSize(len(key) + len(value)) s.storageBytes += common.StorageSize(len(key) + len(value))
}, },
} }
subtask.genTrie = trie.NewStackTrie(subtask.genBatch) subtask.genTrie = trie.NewStackTrieWithOwner(subtask.genBatch, accountHash)
} }
} }
} }
@ -1971,7 +1971,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
Last: r.End(), Last: r.End(),
root: acc.Root, root: acc.Root,
genBatch: batch, genBatch: batch,
genTrie: trie.NewStackTrie(batch), genTrie: trie.NewStackTrieWithOwner(batch, account),
}) })
for r.Next() { for r.Next() {
batch := ethdb.HookedBatch{ batch := ethdb.HookedBatch{
@ -1985,7 +1985,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
Last: r.End(), Last: r.End(),
root: acc.Root, root: acc.Root,
genBatch: batch, genBatch: batch,
genTrie: trie.NewStackTrie(batch), genTrie: trie.NewStackTrieWithOwner(batch, account),
}) })
} }
for _, task := range tasks { for _, task := range tasks {
@ -2030,7 +2030,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
slots += len(res.hashes[i]) slots += len(res.hashes[i])
if i < len(res.hashes)-1 || res.subTask == nil { if i < len(res.hashes)-1 || res.subTask == nil {
tr := trie.NewStackTrie(batch) tr := trie.NewStackTrieWithOwner(batch, account)
for j := 0; j < len(res.hashes[i]); j++ { for j := 0; j < len(res.hashes[i]); j++ {
tr.Update(res.hashes[i][j][:], res.slots[i][j]) tr.Update(res.hashes[i][j][:], res.slots[i][j])
} }

50
eth/protocols/snap/sync_test.go
View File

@ -1349,7 +1349,7 @@ func getCodeByHash(hash common.Hash) []byte {
// makeAccountTrieNoStorage spits out a trie, along with the leafs // makeAccountTrieNoStorage spits out a trie, along with the leafs
func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) {
db := trie.NewDatabase(rawdb.NewMemoryDatabase()) db := trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie, _ := trie.New(common.Hash{}, db) accTrie := trie.NewEmpty(db)
var entries entrySlice var entries entrySlice
for i := uint64(1); i <= uint64(n); i++ { for i := uint64(1); i <= uint64(n); i++ {
value, _ := rlp.EncodeToBytes(&types.StateAccount{ value, _ := rlp.EncodeToBytes(&types.StateAccount{
@ -1376,8 +1376,8 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
entries entrySlice entries entrySlice
boundaries []common.Hash boundaries []common.Hash
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase())
trie, _ = trie.New(common.Hash{}, db) trie = trie.NewEmpty(db)
) )
// Initialize boundaries // Initialize boundaries
var next common.Hash var next common.Hash
@ -1429,7 +1429,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
var ( var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie, _ = trie.New(common.Hash{}, db) accTrie = trie.NewEmpty(db)
entries entrySlice entries entrySlice
storageTries = make(map[common.Hash]*trie.Trie) storageTries = make(map[common.Hash]*trie.Trie)
storageEntries = make(map[common.Hash]entrySlice) storageEntries = make(map[common.Hash]entrySlice)
@ -1442,7 +1442,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
codehash = getCodeHash(i) codehash = getCodeHash(i)
} }
// Create a storage trie // Create a storage trie
stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) stTrie, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db)
stRoot := stTrie.Hash() stRoot := stTrie.Hash()
stTrie.Commit(nil) stTrie.Commit(nil)
value, _ := rlp.EncodeToBytes(&types.StateAccount{ value, _ := rlp.EncodeToBytes(&types.StateAccount{
@ -1468,23 +1468,11 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
var ( var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie, _ = trie.New(common.Hash{}, db) accTrie = trie.NewEmpty(db)
entries entrySlice entries entrySlice
storageTries = make(map[common.Hash]*trie.Trie) storageTries = make(map[common.Hash]*trie.Trie)
storageEntries = make(map[common.Hash]entrySlice) storageEntries = make(map[common.Hash]entrySlice)
) )
// Make a storage trie which we reuse for the whole lot
var (
stTrie *trie.Trie
stEntries entrySlice
)
if boundary {
stTrie, stEntries = makeBoundaryStorageTrie(slots, db)
} else {
stTrie, stEntries = makeStorageTrieWithSeed(uint64(slots), 0, db)
}
stRoot := stTrie.Hash()
// Create n accounts in the trie // Create n accounts in the trie
for i := uint64(1); i <= uint64(accounts); i++ { for i := uint64(1); i <= uint64(accounts); i++ {
key := key32(i) key := key32(i)
@ -1492,6 +1480,19 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
if code { if code {
codehash = getCodeHash(i) codehash = getCodeHash(i)
} }
// Make a storage trie
var (
stTrie *trie.Trie
stEntries entrySlice
)
if boundary {
stTrie, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db)
} else {
stTrie, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db)
}
stRoot := stTrie.Hash()
stTrie.Commit(nil)
value, _ := rlp.EncodeToBytes(&types.StateAccount{ value, _ := rlp.EncodeToBytes(&types.StateAccount{
Nonce: i, Nonce: i,
Balance: big.NewInt(int64(i)), Balance: big.NewInt(int64(i)),
@ -1506,7 +1507,6 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
storageEntries[common.BytesToHash(key)] = stEntries storageEntries[common.BytesToHash(key)] = stEntries
} }
sort.Sort(entries) sort.Sort(entries)
stTrie.Commit(nil)
accTrie.Commit(nil) accTrie.Commit(nil)
return accTrie, entries, storageTries, storageEntries return accTrie, entries, storageTries, storageEntries
} }
@ -1514,8 +1514,8 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
// makeStorageTrieWithSeed fills a storage trie with n items, returning the // makeStorageTrieWithSeed fills a storage trie with n items, returning the
// not-yet-committed trie and the sorted entries. The seeds can be used to ensure // not-yet-committed trie and the sorted entries. The seeds can be used to ensure
// that tries are unique. // that tries are unique.
func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) {
trie, _ := trie.New(common.Hash{}, db) trie, _ := trie.New(owner, common.Hash{}, db)
var entries entrySlice var entries entrySlice
for i := uint64(1); i <= n; i++ { for i := uint64(1); i <= n; i++ {
// store 'x' at slot 'x' // store 'x' at slot 'x'
@ -1537,11 +1537,11 @@ func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, ent
// makeBoundaryStorageTrie constructs a storage trie. Instead of filling // makeBoundaryStorageTrie constructs a storage trie. Instead of filling
// storage slots normally, this function will fill a few slots which have // storage slots normally, this function will fill a few slots which have
// boundary hash. // boundary hash.
func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie.Trie, entrySlice) {
var ( var (
entries entrySlice entries entrySlice
boundaries []common.Hash boundaries []common.Hash
trie, _ = trie.New(common.Hash{}, db) trie, _ = trie.New(owner, common.Hash{}, db)
) )
// Initialize boundaries // Initialize boundaries
var next common.Hash var next common.Hash
@ -1588,7 +1588,7 @@ func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice)
func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
t.Helper() t.Helper()
triedb := trie.NewDatabase(db) triedb := trie.NewDatabase(db)
accTrie, err := trie.New(root, triedb) accTrie, err := trie.New(common.Hash{}, root, triedb)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -1606,7 +1606,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
} }
accounts++ accounts++
if acc.Root != emptyRoot { if acc.Root != emptyRoot {
storeTrie, err := trie.NewSecure(acc.Root, triedb) storeTrie, err := trie.NewSecure(common.BytesToHash(accIt.Key), acc.Root, triedb)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

2
les/downloader/downloader_test.go
View File

@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block {
func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error {
// For now only check that the state trie is correct // For now only check that the state trie is correct
if block := dl.GetBlockByHash(hash); block != nil { if block := dl.GetBlockByHash(hash); block != nil {
_, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb)) _, err := trie.NewSecure(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb))
return err return err
} }
return fmt.Errorf("non existent block: %x", hash[:4]) return fmt.Errorf("non existent block: %x", hash[:4])

8
les/handler_test.go
View File

@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) {
accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}}
for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ {
header := bc.GetHeaderByNumber(i) header := bc.GetHeaderByNumber(i)
trie, _ := trie.New(header.Root, trie.NewDatabase(server.db)) trie, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db))
for _, acc := range accounts { for _, acc := range accounts {
req := ProofReq{ req := ProofReq{
@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) {
var expected []rlp.RawValue var expected []rlp.RawValue
if wantOK { if wantOK {
proofsV2 := light.NewNodeSet() proofsV2 := light.NewNodeSet()
t, _ := trie.New(header.Root, trie.NewDatabase(server.db)) t, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db))
t.Prove(account, 0, proofsV2) t.Prove(account, 0, proofsV2)
expected = proofsV2.NodeList() expected = proofsV2.NodeList()
} }
@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) {
AuxData: [][]byte{rlp}, AuxData: [][]byte{rlp},
} }
root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix)))
trie.Prove(key, 0, &proofsV2.Proofs) trie.Prove(key, 0, &proofsV2.Proofs)
// Assemble the requests for the different protocols // Assemble the requests for the different protocols
requestsV2 := []HelperTrieReq{{ requestsV2 := []HelperTrieReq{{
@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) {
var proofs HelperTrieResps var proofs HelperTrieResps
root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix)))
trie.Prove(key, 0, &proofs.Proofs) trie.Prove(key, 0, &proofs.Proofs)
// Send the proof request and verify the response // Send the proof request and verify the response

4
les/server_handler.go
View File

@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool {
// getAccount retrieves an account from the state based on root. // getAccount retrieves an account from the state based on root.
func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) {
trie, err := trie.New(root, triedb) trie, err := trie.New(common.Hash{}, root, triedb)
if err != nil { if err != nil {
return types.StateAccount{}, err return types.StateAccount{}, err
} }
@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie {
if root == (common.Hash{}) { if root == (common.Hash{}) {
return nil return nil
} }
trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix)))
return trie return trie
} }

2
light/odr_test.go
View File

@ -82,7 +82,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number) req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number)
} }
case *TrieRequest: case *TrieRequest:
t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) t, _ := trie.New(common.BytesToHash(req.Id.AccKey), req.Id.Root, trie.NewDatabase(odr.sdb))
nodes := NewNodeSet() nodes := NewNodeSet()
t.Prove(req.Key, 0, nodes) t.Prove(req.Key, 0, nodes)
req.Proof = nodes req.Proof = nodes

13
light/postprocess.go
View File

@ -187,12 +187,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti
root = GetChtRoot(c.diskdb, section-1, lastSectionHead) root = GetChtRoot(c.diskdb, section-1, lastSectionHead)
} }
var err error var err error
c.trie, err = trie.New(root, c.triedb) c.trie, err = trie.New(common.Hash{}, root, c.triedb)
if err != nil && c.odr != nil { if err != nil && c.odr != nil {
err = c.fetchMissingNodes(ctx, section, root) err = c.fetchMissingNodes(ctx, section, root)
if err == nil { if err == nil {
c.trie, err = trie.New(root, c.triedb) c.trie, err = trie.New(common.Hash{}, root, c.triedb)
} }
} }
c.section = section c.section = section
@ -253,9 +253,8 @@ func (c *ChtIndexerBackend) Commit() error {
return nil return nil
} }
// PruneSections implements core.ChainIndexerBackend which deletes all // Prune implements core.ChainIndexerBackend which deletes all chain data
// chain data(except hash<->number mappings) older than the specified // (except hash<->number mappings) older than the specified threshold.
// threshold.
func (c *ChtIndexerBackend) Prune(threshold uint64) error { func (c *ChtIndexerBackend) Prune(threshold uint64) error {
// Short circuit if the light pruning is disabled. // Short circuit if the light pruning is disabled.
if c.disablePruning { if c.disablePruning {
@ -404,11 +403,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las
root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead)
} }
var err error var err error
b.trie, err = trie.New(root, b.triedb) b.trie, err = trie.New(common.Hash{}, root, b.triedb)
if err != nil && b.odr != nil { if err != nil && b.odr != nil {
err = b.fetchMissingNodes(ctx, section, root) err = b.fetchMissingNodes(ctx, section, root)
if err == nil { if err == nil {
b.trie, err = trie.New(root, b.triedb) b.trie, err = trie.New(common.Hash{}, root, b.triedb)
} }
} }
b.section = section b.section = section

12
light/trie.go
View File

@ -169,7 +169,11 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
for { for {
var err error var err error
if t.trie == nil { if t.trie == nil {
t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) var owner common.Hash
if len(t.id.AccKey) > 0 {
owner = common.BytesToHash(t.id.AccKey)
}
t.trie, err = trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database()))
} }
if err == nil { if err == nil {
err = fn() err = fn()
@ -195,7 +199,11 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator {
// Open the actual non-ODR trie if that hasn't happened yet. // Open the actual non-ODR trie if that hasn't happened yet.
if t.trie == nil { if t.trie == nil {
it.do(func() error { it.do(func() error {
t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) var owner common.Hash
if len(t.id.AccKey) > 0 {
owner = common.BytesToHash(t.id.AccKey)
}
t, err := trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database()))
if err == nil { if err == nil {
it.t.trie = t it.t.trie = t
} }

4
tests/fuzzers/les/les-fuzzer.go
View File

@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) {
} }
func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) {
chtTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
bloomTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < testChainLen; i++ { for i := 0; i < testChainLen; i++ {
// The element in CHT is <big-endian block number> -> <block hash> // The element in CHT is <big-endian block number> -> <block hash>
key := make([]byte, 8) key := make([]byte, 8)

2
tests/fuzzers/rangeproof/rangeproof-fuzzer.go
View File

@ -62,7 +62,7 @@ func (f *fuzzer) readInt() uint64 {
} }
func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {
trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv) vals := make(map[string]*kv)
size := f.readInt() size := f.readInt()
// Fill it with some fluff // Fill it with some fluff

3
tests/fuzzers/stacktrie/trie_fuzzer.go
View File

@ -25,7 +25,6 @@ import (
"io" "io"
"sort" "sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"golang.org/x/crypto/sha3" "golang.org/x/crypto/sha3"
@ -144,7 +143,7 @@ func (f *fuzzer) fuzz() int {
var ( var (
spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbA = trie.NewDatabase(spongeA) dbA = trie.NewDatabase(spongeA)
trieA, _ = trie.New(common.Hash{}, dbA) trieA = trie.NewEmpty(dbA)
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
trieB = trie.NewStackTrie(spongeB) trieB = trie.NewStackTrie(spongeB)
vals kvs vals kvs

6
tests/fuzzers/trie/trie-fuzzer.go
View File

@ -144,7 +144,7 @@ func runRandTest(rt randTest) error {
triedb := trie.NewDatabase(memorydb.New()) triedb := trie.NewDatabase(memorydb.New())
tr, _ := trie.New(common.Hash{}, triedb) tr := trie.NewEmpty(triedb)
values := make(map[string]string) // tracks content of the trie values := make(map[string]string) // tracks content of the trie
for i, step := range rt { for i, step := range rt {
@ -170,13 +170,13 @@ func runRandTest(rt randTest) error {
if err != nil { if err != nil {
return err return err
} }
newtr, err := trie.New(hash, triedb) newtr, err := trie.New(common.Hash{}, hash, triedb)
if err != nil { if err != nil {
return err return err
} }
tr = newtr tr = newtr
case opItercheckhash: case opItercheckhash:
checktr, _ := trie.New(common.Hash{}, triedb) checktr := trie.NewEmpty(triedb)
it := trie.NewIterator(tr.NodeIterator(nil)) it := trie.NewIterator(tr.NodeIterator(nil))
for it.Next() { for it.Next() {
checktr.Update(it.Key, it.Value) checktr.Update(it.Key, it.Value)

13
trie/errors.go
View File

@ -26,10 +26,21 @@ import (
// in the case where a trie node is not present in the local database. It contains // in the case where a trie node is not present in the local database. It contains
// information necessary for retrieving the missing node. // information necessary for retrieving the missing node.
type MissingNodeError struct { type MissingNodeError struct {
Owner common.Hash // owner of the trie if it's 2-layered trie
NodeHash common.Hash // hash of the missing node NodeHash common.Hash // hash of the missing node
Path []byte // hex-encoded path to the missing node Path []byte // hex-encoded path to the missing node
err error // concrete error for missing trie node
}
// Unwrap returns the concrete error for missing trie node which
// allows us for further analysis outside.
func (err *MissingNodeError) Unwrap() error {
return err.err
} }
func (err *MissingNodeError) Error() string { func (err *MissingNodeError) Error() string {
return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path) if err.Owner == (common.Hash{}) {
return fmt.Sprintf("missing trie node %x (path %x) %v", err.NodeHash, err.Path, err.err)
}
return fmt.Sprintf("missing trie node %x (owner %x) (path %x) %v", err.NodeHash, err.Owner, err.Path, err.err)
} }

18
trie/iterator_test.go
View File

@ -297,7 +297,7 @@ func TestUnionIterator(t *testing.T) {
} }
func TestIteratorNoDups(t *testing.T) { func TestIteratorNoDups(t *testing.T) {
tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for _, val := range testdata1 { for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v)) tr.Update([]byte(val.k), []byte(val.v))
} }
@ -312,7 +312,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
diskdb := memorydb.New() diskdb := memorydb.New()
triedb := NewDatabase(diskdb) triedb := NewDatabase(diskdb)
tr, _ := New(common.Hash{}, triedb) tr := NewEmpty(triedb)
for _, val := range testdata1 { for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v)) tr.Update([]byte(val.k), []byte(val.v))
} }
@ -337,7 +337,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
} }
for i := 0; i < 20; i++ { for i := 0; i < 20; i++ {
// Create trie that will load all nodes from DB. // Create trie that will load all nodes from DB.
tr, _ := New(tr.Hash(), triedb) tr, _ := New(common.Hash{}, tr.Hash(), triedb)
// Remove a random node from the database. It can't be the root node // Remove a random node from the database. It can't be the root node
// because that one is already loaded. // because that one is already loaded.
@ -403,7 +403,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
diskdb := memorydb.New() diskdb := memorydb.New()
triedb := NewDatabase(diskdb) triedb := NewDatabase(diskdb)
ctr, _ := New(common.Hash{}, triedb) ctr := NewEmpty(triedb)
for _, val := range testdata1 { for _, val := range testdata1 {
ctr.Update([]byte(val.k), []byte(val.v)) ctr.Update([]byte(val.k), []byte(val.v))
} }
@ -425,7 +425,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
} }
// Create a new iterator that seeks to "bars". Seeking can't proceed because // Create a new iterator that seeks to "bars". Seeking can't proceed because
// the node is missing. // the node is missing.
tr, _ := New(root, triedb) tr, _ := New(common.Hash{}, root, triedb)
it := tr.NodeIterator([]byte("bars")) it := tr.NodeIterator([]byte("bars"))
missing, ok := it.Error().(*MissingNodeError) missing, ok := it.Error().(*MissingNodeError)
if !ok { if !ok {
@ -513,7 +513,7 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) {
// Create an empty trie // Create an empty trie
logDb := &loggingDb{0, memorydb.New()} logDb := &loggingDb{0, memorydb.New()}
triedb := NewDatabase(logDb) triedb := NewDatabase(logDb)
trie, _ := NewSecure(common.Hash{}, triedb) trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb)
// Fill it with some arbitrary data // Fill it with some arbitrary data
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
@ -546,9 +546,9 @@ func TestNodeIteratorLargeTrie(t *testing.T) {
func TestIteratorNodeBlob(t *testing.T) { func TestIteratorNodeBlob(t *testing.T) {
var ( var (
db = memorydb.New() db = memorydb.New()
triedb = NewDatabase(db) triedb = NewDatabase(db)
trie, _ = New(common.Hash{}, triedb) trie = NewEmpty(triedb)
) )
vals := []struct{ k, v string }{ vals := []struct{ k, v string }{
{"do", "verb"}, {"do", "verb"},

20
trie/proof_test.go
View File

@ -80,7 +80,7 @@ func TestProof(t *testing.T) {
} }
func TestOneElementProof(t *testing.T) { func TestOneElementProof(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "k", "v") updateString(trie, "k", "v")
for i, prover := range makeProvers(trie) { for i, prover := range makeProvers(trie) {
proof := prover([]byte("k")) proof := prover([]byte("k"))
@ -131,7 +131,7 @@ func TestBadProof(t *testing.T) {
// Tests that missing keys can also be proven. The test explicitly uses a single // Tests that missing keys can also be proven. The test explicitly uses a single
// entry trie and checks for missing keys both before and after the single entry. // entry trie and checks for missing keys both before and after the single entry.
func TestMissingKeyProof(t *testing.T) { func TestMissingKeyProof(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "k", "v") updateString(trie, "k", "v")
for i, key := range []string{"a", "j", "l", "z"} { for i, key := range []string{"a", "j", "l", "z"} {
@ -387,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) {
} }
// Test the mini trie with only a single element. // Test the mini trie with only a single element.
tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
entry := &kv{randBytes(32), randBytes(20), false} entry := &kv{randBytes(32), randBytes(20), false}
tinyTrie.Update(entry.k, entry.v) tinyTrie.Update(entry.k, entry.v)
@ -459,7 +459,7 @@ func TestAllElementsProof(t *testing.T) {
// TestSingleSideRangeProof tests the range starts from zero. // TestSingleSideRangeProof tests the range starts from zero.
func TestSingleSideRangeProof(t *testing.T) { func TestSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ { for i := 0; i < 64; i++ {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice var entries entrySlice
for i := 0; i < 4096; i++ { for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false} value := &kv{randBytes(32), randBytes(20), false}
@ -494,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) {
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
func TestReverseSingleSideRangeProof(t *testing.T) { func TestReverseSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ { for i := 0; i < 64; i++ {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice var entries entrySlice
for i := 0; i < 4096; i++ { for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false} value := &kv{randBytes(32), randBytes(20), false}
@ -601,7 +601,7 @@ func TestBadRangeProof(t *testing.T) {
// TestGappedRangeProof focuses on the small trie with embedded nodes. // TestGappedRangeProof focuses on the small trie with embedded nodes.
// If the gapped node is embedded in the trie, it should be detected too. // If the gapped node is embedded in the trie, it should be detected too.
func TestGappedRangeProof(t *testing.T) { func TestGappedRangeProof(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries []*kv // Sorted entries var entries []*kv // Sorted entries
for i := byte(0); i < 10; i++ { for i := byte(0); i < 10; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -675,7 +675,7 @@ func TestSameSideProofs(t *testing.T) {
} }
func TestHasRightElement(t *testing.T) { func TestHasRightElement(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice var entries entrySlice
for i := 0; i < 4096; i++ { for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false} value := &kv{randBytes(32), randBytes(20), false}
@ -1028,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) {
} }
func randomTrie(n int) (*Trie, map[string]*kv) { func randomTrie(n int) (*Trie, map[string]*kv) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv) vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ { for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -1053,7 +1053,7 @@ func randBytes(n int) []byte {
} }
func nonRandomTrie(n int) (*Trie, map[string]*kv) { func nonRandomTrie(n int) (*Trie, map[string]*kv) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv) vals := make(map[string]*kv)
max := uint64(0xffffffffffffffff) max := uint64(0xffffffffffffffff)
for i := uint64(0); i < uint64(n); i++ { for i := uint64(0); i < uint64(n); i++ {
@ -1078,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) {
common.Hex2Bytes("02"), common.Hex2Bytes("02"),
common.Hex2Bytes("03"), common.Hex2Bytes("03"),
} }
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i, key := range keys { for i, key := range keys {
trie.Update(key, vals[i]) trie.Update(key, vals[i])
} }

4
trie/secure_trie.go
View File

@ -53,11 +53,11 @@ type SecureTrie struct {
// Loaded nodes are kept around until their 'cache generation' expires. // Loaded nodes are kept around until their 'cache generation' expires.
// A new cache generation is created by each call to Commit. // A new cache generation is created by each call to Commit.
// cachelimit sets the number of past cache generations to keep. // cachelimit sets the number of past cache generations to keep.
func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) { func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) {
if db == nil { if db == nil {
panic("trie.NewSecure called without a database") panic("trie.NewSecure called without a database")
} }
trie, err := New(root, db) trie, err := New(owner, root, db)
if err != nil { if err != nil {
return nil, err return nil, err
} }

4
trie/secure_trie_test.go
View File

@ -28,7 +28,7 @@ import (
) )
func newEmptySecure() *SecureTrie { func newEmptySecure() *SecureTrie {
trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New())) trie, _ := NewSecure(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New()))
return trie return trie
} }
@ -36,7 +36,7 @@ func newEmptySecure() *SecureTrie {
func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
// Create an empty trie // Create an empty trie
triedb := NewDatabase(memorydb.New()) triedb := NewDatabase(memorydb.New())
trie, _ := NewSecure(common.Hash{}, triedb) trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb)
// Fill it with some arbitrary data // Fill it with some arbitrary data
content := make(map[string][]byte) content := make(map[string][]byte)

47
trie/stacktrie.go
View File

@ -38,9 +38,10 @@ var stPool = sync.Pool{
}, },
} }
func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie { func stackTrieFromPool(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie {
st := stPool.Get().(*StackTrie) st := stPool.Get().(*StackTrie)
st.db = db st.db = db
st.owner = owner
return st return st
} }
@ -53,6 +54,7 @@ func returnToPool(st *StackTrie) {
// in order. Once it determines that a subtree will no longer be inserted // in order. Once it determines that a subtree will no longer be inserted
// into, it will hash it and free up the memory it uses. // into, it will hash it and free up the memory it uses.
type StackTrie struct { type StackTrie struct {
owner common.Hash // the owner of the trie
nodeType uint8 // node type (as in branch, ext, leaf) nodeType uint8 // node type (as in branch, ext, leaf)
val []byte // value contained by this node if it's a leaf val []byte // value contained by this node if it's a leaf
key []byte // key chunk covered by this (leaf|ext) node key []byte // key chunk covered by this (leaf|ext) node
@ -68,6 +70,16 @@ func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie {
} }
} }
// NewStackTrieWithOwner allocates and initializes an empty trie, but with
// the additional owner field.
func NewStackTrieWithOwner(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie {
return &StackTrie{
owner: owner,
nodeType: emptyNode,
db: db,
}
}
// NewFromBinary initialises a serialized stacktrie with the given db. // NewFromBinary initialises a serialized stacktrie with the given db.
func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) { func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) {
var st StackTrie var st StackTrie
@ -88,10 +100,12 @@ func (st *StackTrie) MarshalBinary() (data []byte, err error) {
w = bufio.NewWriter(&b) w = bufio.NewWriter(&b)
) )
if err := gob.NewEncoder(w).Encode(struct { if err := gob.NewEncoder(w).Encode(struct {
Nodetype uint8 Owner common.Hash
NodeType uint8
Val []byte Val []byte
Key []byte Key []byte
}{ }{
st.owner,
st.nodeType, st.nodeType,
st.val, st.val,
st.key, st.key,
@ -122,12 +136,14 @@ func (st *StackTrie) UnmarshalBinary(data []byte) error {
func (st *StackTrie) unmarshalBinary(r io.Reader) error { func (st *StackTrie) unmarshalBinary(r io.Reader) error {
var dec struct { var dec struct {
Nodetype uint8 Owner common.Hash
NodeType uint8
Val []byte Val []byte
Key []byte Key []byte
} }
gob.NewDecoder(r).Decode(&dec) gob.NewDecoder(r).Decode(&dec)
st.nodeType = dec.Nodetype st.owner = dec.Owner
st.nodeType = dec.NodeType
st.val = dec.Val st.val = dec.Val
st.key = dec.Key st.key = dec.Key
@ -154,16 +170,16 @@ func (st *StackTrie) setDb(db ethdb.KeyValueWriter) {
} }
} }
func newLeaf(key, val []byte, db ethdb.KeyValueWriter) *StackTrie { func newLeaf(owner common.Hash, key, val []byte, db ethdb.KeyValueWriter) *StackTrie {
st := stackTrieFromPool(db) st := stackTrieFromPool(db, owner)
st.nodeType = leafNode st.nodeType = leafNode
st.key = append(st.key, key...) st.key = append(st.key, key...)
st.val = val st.val = val
return st return st
} }
func newExt(key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { func newExt(owner common.Hash, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie {
st := stackTrieFromPool(db) st := stackTrieFromPool(db, owner)
st.nodeType = extNode st.nodeType = extNode
st.key = append(st.key, key...) st.key = append(st.key, key...)
st.children[0] = child st.children[0] = child
@ -196,6 +212,7 @@ func (st *StackTrie) Update(key, value []byte) {
} }
func (st *StackTrie) Reset() { func (st *StackTrie) Reset() {
st.owner = common.Hash{}
st.db = nil st.db = nil
st.key = st.key[:0] st.key = st.key[:0]
st.val = nil st.val = nil
@ -236,7 +253,7 @@ func (st *StackTrie) insert(key, value []byte) {
// Add new child // Add new child
if st.children[idx] == nil { if st.children[idx] == nil {
st.children[idx] = newLeaf(key[1:], value, st.db) st.children[idx] = newLeaf(st.owner, key[1:], value, st.db)
} else { } else {
st.children[idx].insert(key[1:], value) st.children[idx].insert(key[1:], value)
} }
@ -262,7 +279,7 @@ func (st *StackTrie) insert(key, value []byte) {
// node directly. // node directly.
var n *StackTrie var n *StackTrie
if diffidx < len(st.key)-1 { if diffidx < len(st.key)-1 {
n = newExt(st.key[diffidx+1:], st.children[0], st.db) n = newExt(st.owner, st.key[diffidx+1:], st.children[0], st.db)
} else { } else {
// Break on the last byte, no need to insert // Break on the last byte, no need to insert
// an extension node: reuse the current node // an extension node: reuse the current node
@ -282,12 +299,12 @@ func (st *StackTrie) insert(key, value []byte) {
// the common prefix is at least one byte // the common prefix is at least one byte
// long, insert a new intermediate branch // long, insert a new intermediate branch
// node. // node.
st.children[0] = stackTrieFromPool(st.db) st.children[0] = stackTrieFromPool(st.db, st.owner)
st.children[0].nodeType = branchNode st.children[0].nodeType = branchNode
p = st.children[0] p = st.children[0]
} }
// Create a leaf for the inserted part // Create a leaf for the inserted part
o := newLeaf(key[diffidx+1:], value, st.db) o := newLeaf(st.owner, key[diffidx+1:], value, st.db)
// Insert both child leaves where they belong: // Insert both child leaves where they belong:
origIdx := st.key[diffidx] origIdx := st.key[diffidx]
@ -323,7 +340,7 @@ func (st *StackTrie) insert(key, value []byte) {
// Convert current node into an ext, // Convert current node into an ext,
// and insert a child branch node. // and insert a child branch node.
st.nodeType = extNode st.nodeType = extNode
st.children[0] = NewStackTrie(st.db) st.children[0] = NewStackTrieWithOwner(st.db, st.owner)
st.children[0].nodeType = branchNode st.children[0].nodeType = branchNode
p = st.children[0] p = st.children[0]
} }
@ -332,11 +349,11 @@ func (st *StackTrie) insert(key, value []byte) {
// value and another containing the new value. The child leaf // value and another containing the new value. The child leaf
// is hashed directly in order to free up some memory. // is hashed directly in order to free up some memory.
origIdx := st.key[diffidx] origIdx := st.key[diffidx]
p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db) p.children[origIdx] = newLeaf(st.owner, st.key[diffidx+1:], st.val, st.db)
p.children[origIdx].hash() p.children[origIdx].hash()
newIdx := key[diffidx] newIdx := key[diffidx]
p.children[newIdx] = newLeaf(key[diffidx+1:], value, st.db) p.children[newIdx] = newLeaf(st.owner, key[diffidx+1:], value, st.db)
// Finally, cut off the key part that has been passed // Finally, cut off the key part that has been passed
// over to the children. // over to the children.

14
trie/stacktrie_test.go
View File

@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) {
func TestSizeBug(t *testing.T) { func TestSizeBug(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) nt := NewEmpty(NewDatabase(memorydb.New()))
leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) {
func TestEmptyBug(t *testing.T) { func TestEmptyBug(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) nt := NewEmpty(NewDatabase(memorydb.New()))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) {
func TestValLength56(t *testing.T) { func TestValLength56(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) nt := NewEmpty(NewDatabase(memorydb.New()))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -254,7 +254,8 @@ func TestValLength56(t *testing.T) {
// which causes a lot of node-within-node. This case was found via fuzzing. // which causes a lot of node-within-node. This case was found via fuzzing.
func TestUpdateSmallNodes(t *testing.T) { func TestUpdateSmallNodes(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) nt := NewEmpty(NewDatabase(memorydb.New()))
kvs := []struct { kvs := []struct {
K string K string
V string V string
@ -282,7 +283,8 @@ func TestUpdateSmallNodes(t *testing.T) {
func TestUpdateVariableKeys(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) {
t.SkipNow() t.SkipNow()
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) nt := NewEmpty(NewDatabase(memorydb.New()))
kvs := []struct { kvs := []struct {
K string K string
V string V string
@ -352,7 +354,7 @@ func TestStacktrieNotModifyValues(t *testing.T) {
func TestStacktrieSerialization(t *testing.T) { func TestStacktrieSerialization(t *testing.T) {
var ( var (
st = NewStackTrie(nil) st = NewStackTrie(nil)
nt, _ = New(common.Hash{}, NewDatabase(memorydb.New())) nt = NewEmpty(NewDatabase(memorydb.New()))
keyB = big.NewInt(1) keyB = big.NewInt(1)
keyDelta = big.NewInt(1) keyDelta = big.NewInt(1)
vals [][]byte vals [][]byte

10
trie/sync_test.go
View File

@ -29,7 +29,7 @@ import (
func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) {
// Create an empty trie // Create an empty trie
triedb := NewDatabase(memorydb.New()) triedb := NewDatabase(memorydb.New())
trie, _ := NewSecure(common.Hash{}, triedb) trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb)
// Fill it with some arbitrary data // Fill it with some arbitrary data
content := make(map[string][]byte) content := make(map[string][]byte)
@ -60,7 +60,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) {
// content map. // content map.
func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) {
// Check root availability and trie contents // Check root availability and trie contents
trie, err := NewSecure(common.BytesToHash(root), db) trie, err := NewSecure(common.Hash{}, common.BytesToHash(root), db)
if err != nil { if err != nil {
t.Fatalf("failed to create trie at %x: %v", root, err) t.Fatalf("failed to create trie at %x: %v", root, err)
} }
@ -77,7 +77,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri
// checkTrieConsistency checks that all nodes in a trie are indeed present. // checkTrieConsistency checks that all nodes in a trie are indeed present.
func checkTrieConsistency(db *Database, root common.Hash) error { func checkTrieConsistency(db *Database, root common.Hash) error {
// Create and iterate a trie rooted in a subnode // Create and iterate a trie rooted in a subnode
trie, err := NewSecure(root, db) trie, err := NewSecure(common.Hash{}, root, db)
if err != nil { if err != nil {
return nil // Consider a non existent state consistent return nil // Consider a non existent state consistent
} }
@ -91,8 +91,8 @@ func checkTrieConsistency(db *Database, root common.Hash) error {
func TestEmptySync(t *testing.T) { func TestEmptySync(t *testing.T) {
dbA := NewDatabase(memorydb.New()) dbA := NewDatabase(memorydb.New())
dbB := NewDatabase(memorydb.New()) dbB := NewDatabase(memorydb.New())
emptyA, _ := New(common.Hash{}, dbA) emptyA := NewEmpty(dbA)
emptyB, _ := New(emptyRoot, dbB) emptyB, _ := New(common.Hash{}, emptyRoot, dbB)
for i, trie := range []*Trie{emptyA, emptyB} { for i, trie := range []*Trie{emptyA, emptyB} {
sync := NewSync(trie.Hash(), memorydb.New(), nil) sync := NewSync(trie.Hash(), memorydb.New(), nil)

68
trie/trie.go
View File

@ -61,8 +61,9 @@ type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent commo
// //
// Trie is not safe for concurrent use. // Trie is not safe for concurrent use.
type Trie struct { type Trie struct {
db *Database db *Database
root node root node
owner common.Hash
// Keep track of the number leaves which have been inserted since the last // Keep track of the number leaves which have been inserted since the last
// hashing operation. This number will not directly map to the number of // hashing operation. This number will not directly map to the number of
@ -84,33 +85,27 @@ func (t *Trie) Copy() *Trie {
return &Trie{ return &Trie{
db: t.db, db: t.db,
root: t.root, root: t.root,
owner: t.owner,
unhashed: t.unhashed, unhashed: t.unhashed,
tracer: t.tracer.copy(), tracer: t.tracer.copy(),
} }
} }
// New creates a trie with an existing root node from db. // New creates a trie with an existing root node from db and an assigned
// owner for storage proximity.
// //
// If root is the zero hash or the sha3 hash of an empty string, the // If root is the zero hash or the sha3 hash of an empty string, the
// trie is initially empty and does not require a database. Otherwise, // trie is initially empty and does not require a database. Otherwise,
// New will panic if db is nil and returns a MissingNodeError if root does // New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand. // not exist in the database. Accessing the trie loads nodes from db on demand.
func New(root common.Hash, db *Database) (*Trie, error) { func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
if db == nil { return newTrie(owner, root, db)
panic("trie.New called without a database") }
}
trie := &Trie{ // NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
db: db, func NewEmpty(db *Database) *Trie {
//tracer: newTracer(), tr, _ := newTrie(common.Hash{}, common.Hash{}, db)
} return tr
if root != (common.Hash{}) && root != emptyRoot {
rootnode, err := trie.resolveHash(root[:], nil)
if err != nil {
return nil, err
}
trie.root = rootnode
}
return trie, nil
} }
// newWithRootNode initializes the trie with the given root node. // newWithRootNode initializes the trie with the given root node.
@ -123,6 +118,26 @@ func newWithRootNode(root node) *Trie {
} }
} }
// newTrie is the internal function used to construct the trie with given parameters.
func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
if db == nil {
panic("trie.New called without a database")
}
trie := &Trie{
db: db,
owner: owner,
//tracer: newTracer(),
}
if root != (common.Hash{}) && root != emptyRoot {
rootnode, err := trie.resolveHash(root[:], nil)
if err != nil {
return nil, err
}
trie.root = rootnode
}
return trie, nil
}
// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
// the key after the given start key. // the key after the given start key.
func (t *Trie) NodeIterator(start []byte) NodeIterator { func (t *Trie) NodeIterator(start []byte) NodeIterator {
@ -562,7 +577,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
if node := t.db.node(hash); node != nil { if node := t.db.node(hash); node != nil {
return node, nil return node, nil
} }
return nil, &MissingNodeError{NodeHash: hash, Path: prefix} return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix}
} }
func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
@ -571,7 +586,7 @@ func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
if len(blob) != 0 { if len(blob) != 0 {
return blob, nil return blob, nil
} }
return nil, &MissingNodeError{NodeHash: hash, Path: prefix} return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix}
} }
// Hash returns the root hash of the trie. It does not write to the // Hash returns the root hash of the trie. It does not write to the
@ -602,7 +617,10 @@ func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
// Do a quick check if we really need to commit, before we spin // Do a quick check if we really need to commit, before we spin
// up goroutines. This can happen e.g. if we load a trie for reading storage // up goroutines. This can happen e.g. if we load a trie for reading storage
// values, but don't write to it. // values, but don't write to it.
if _, dirty := t.root.cache(); !dirty { if hashedNode, dirty := t.root.cache(); !dirty {
// Replace the root node with the origin hash in order to
// ensure all resolved nodes are dropped after the commit.
t.root = hashedNode
return rootHash, 0, nil return rootHash, 0, nil
} }
var wg sync.WaitGroup var wg sync.WaitGroup
@ -647,6 +665,12 @@ func (t *Trie) hashRoot() (node, node, error) {
// Reset drops the referenced root node and cleans all internal state. // Reset drops the referenced root node and cleans all internal state.
func (t *Trie) Reset() { func (t *Trie) Reset() {
t.root = nil t.root = nil
t.owner = common.Hash{}
t.unhashed = 0 t.unhashed = 0
t.tracer.reset() t.tracer.reset()
} }
// Owner returns the associated trie owner.
func (t *Trie) Owner() common.Hash {
return t.owner
}

58
trie/trie_test.go
View File

@ -48,12 +48,12 @@ func init() {
// Used for testing // Used for testing
func newEmpty() *Trie { func newEmpty() *Trie {
trie, _ := New(common.Hash{}, NewDatabase(memorydb.New())) trie := NewEmpty(NewDatabase(memorydb.New()))
return trie return trie
} }
func TestEmptyTrie(t *testing.T) { func TestEmptyTrie(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
res := trie.Hash() res := trie.Hash()
exp := emptyRoot exp := emptyRoot
if res != exp { if res != exp {
@ -62,7 +62,7 @@ func TestEmptyTrie(t *testing.T) {
} }
func TestNull(t *testing.T) { func TestNull(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
key := make([]byte, 32) key := make([]byte, 32)
value := []byte("test") value := []byte("test")
trie.Update(key, value) trie.Update(key, value)
@ -72,7 +72,7 @@ func TestNull(t *testing.T) {
} }
func TestMissingRoot(t *testing.T) { func TestMissingRoot(t *testing.T) {
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) trie, err := New(common.Hash{}, common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New()))
if trie != nil { if trie != nil {
t.Error("New returned non-nil trie for invalid root") t.Error("New returned non-nil trie for invalid root")
} }
@ -88,7 +88,7 @@ func testMissingNode(t *testing.T, memonly bool) {
diskdb := memorydb.New() diskdb := memorydb.New()
triedb := NewDatabase(diskdb) triedb := NewDatabase(diskdb)
trie, _ := New(common.Hash{}, triedb) trie := NewEmpty(triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _, _ := trie.Commit(nil) root, _, _ := trie.Commit(nil)
@ -96,27 +96,27 @@ func testMissingNode(t *testing.T, memonly bool) {
triedb.Commit(root, true, nil) triedb.Commit(root, true, nil)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
_, err := trie.TryGet([]byte("120000")) _, err := trie.TryGet([]byte("120000"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("120099")) _, err = trie.TryGet([]byte("120099"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("123456")) _, err = trie.TryGet([]byte("123456"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryDelete([]byte("123456")) err = trie.TryDelete([]byte("123456"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
@ -129,27 +129,27 @@ func testMissingNode(t *testing.T, memonly bool) {
diskdb.Delete(hash[:]) diskdb.Delete(hash[:])
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("120000")) _, err = trie.TryGet([]byte("120000"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("120099")) _, err = trie.TryGet([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("123456")) _, err = trie.TryGet([]byte("123456"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryDelete([]byte("123456")) err = trie.TryDelete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
@ -277,7 +277,7 @@ func TestReplication(t *testing.T) {
} }
// create a new trie on top of the database and check that lookups work. // create a new trie on top of the database and check that lookups work.
trie2, err := New(exp, trie.db) trie2, err := New(common.Hash{}, exp, trie.db)
if err != nil { if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err) t.Fatalf("can't recreate trie at %x: %v", exp, err)
} }
@ -410,10 +410,10 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
func runRandTest(rt randTest) bool { func runRandTest(rt randTest) bool {
var ( var (
triedb = NewDatabase(memorydb.New()) triedb = NewDatabase(memorydb.New())
tr, _ = New(common.Hash{}, triedb) tr = NewEmpty(triedb)
values = make(map[string]string) // tracks content of the trie values = make(map[string]string) // tracks content of the trie
origTrie, _ = New(common.Hash{}, triedb) origTrie = NewEmpty(triedb)
) )
tr.tracer = newTracer() tr.tracer = newTracer()
@ -445,7 +445,7 @@ func runRandTest(rt randTest) bool {
rt[i].err = err rt[i].err = err
return false return false
} }
newtr, err := New(hash, triedb) newtr, err := New(common.Hash{}, hash, triedb)
if err != nil { if err != nil {
rt[i].err = err rt[i].err = err
return false return false
@ -455,7 +455,7 @@ func runRandTest(rt randTest) bool {
origTrie = tr.Copy() origTrie = tr.Copy()
case opItercheckhash: case opItercheckhash:
checktr, _ := New(common.Hash{}, triedb) checktr := NewEmpty(triedb)
it := NewIterator(tr.NodeIterator(nil)) it := NewIterator(tr.NodeIterator(nil))
for it.Next() { for it.Next() {
checktr.Update(it.Key, it.Value) checktr.Update(it.Key, it.Value)
@ -542,10 +542,10 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000 const benchElemCount = 20000
func benchGet(b *testing.B, commit bool) { func benchGet(b *testing.B, commit bool) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
if commit { if commit {
tmpdb := tempDB(b) tmpdb := tempDB(b)
trie, _ = New(common.Hash{}, tmpdb) trie = NewEmpty(tmpdb)
} }
k := make([]byte, 32) k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ { for i := 0; i < benchElemCount; i++ {
@ -664,7 +664,7 @@ func TestTinyTrie(t *testing.T) {
if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
t.Errorf("3: got %x, exp %x", root, exp) t.Errorf("3: got %x, exp %x", root, exp)
} }
checktr, _ := New(common.Hash{}, trie.db) checktr := NewEmpty(trie.db)
it := NewIterator(trie.NodeIterator(nil)) it := NewIterator(trie.NodeIterator(nil))
for it.Next() { for it.Next() {
checktr.Update(it.Key, it.Value) checktr.Update(it.Key, it.Value)
@ -790,7 +790,7 @@ func TestCommitSequence(t *testing.T) {
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(s) db := NewDatabase(s)
trie, _ := New(common.Hash{}, db) trie := NewEmpty(db)
// Another sponge is used to check the callback-sequence // Another sponge is used to check the callback-sequence
callbackSponge := sha3.NewLegacyKeccak256() callbackSponge := sha3.NewLegacyKeccak256()
// Fill the trie with elements // Fill the trie with elements
@ -832,7 +832,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) {
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(s) db := NewDatabase(s)
trie, _ := New(common.Hash{}, db) trie := NewEmpty(db)
// Another sponge is used to check the callback-sequence // Another sponge is used to check the callback-sequence
callbackSponge := sha3.NewLegacyKeccak256() callbackSponge := sha3.NewLegacyKeccak256()
// Fill the trie with elements // Fill the trie with elements
@ -871,7 +871,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(s) db := NewDatabase(s)
trie, _ := New(common.Hash{}, db) trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits // Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(stackTrieSponge) stTrie := NewStackTrie(stackTrieSponge)
@ -927,7 +927,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
func TestCommitSequenceSmallRoot(t *testing.T) { func TestCommitSequenceSmallRoot(t *testing.T) {
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(s) db := NewDatabase(s)
trie, _ := New(common.Hash{}, db) trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits // Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(stackTrieSponge) stTrie := NewStackTrie(stackTrieSponge)

7
trie/util_test.go
View File

@ -19,14 +19,12 @@ package trie
import ( import (
"testing" "testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
) )
// Tests if the trie diffs are tracked correctly. // Tests if the trie diffs are tracked correctly.
func TestTrieTracer(t *testing.T) { func TestTrieTracer(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase()) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie, _ := New(common.Hash{}, db)
trie.tracer = newTracer() trie.tracer = newTracer()
// Insert a batch of entries, all the nodes should be marked as inserted // Insert a batch of entries, all the nodes should be marked as inserted
@ -93,8 +91,7 @@ func TestTrieTracer(t *testing.T) {
} }
func TestTrieTracerNoop(t *testing.T) { func TestTrieTracerNoop(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase()) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie, _ := New(common.Hash{}, db)
trie.tracer = newTracer() trie.tracer = newTracer()
// Insert a batch of entries, all the nodes should be marked as inserted // Insert a batch of entries, all the nodes should be marked as inserted

38
trie/utils.go
View File

@ -29,24 +29,39 @@ package trie
// This tool can track all of them no matter the node is embedded in its // This tool can track all of them no matter the node is embedded in its
// parent or not, but valueNode is never tracked. // parent or not, but valueNode is never tracked.
// //
// Besides, it's also used for recording the original value of the nodes
// when they are resolved from the disk. The pre-value of the nodes will
// be used to construct reverse-diffs in the future.
//
// Note tracer is not thread-safe, callers should be responsible for handling // Note tracer is not thread-safe, callers should be responsible for handling
// the concurrency issues by themselves. // the concurrency issues by themselves.
type tracer struct { type tracer struct {
insert map[string]struct{} insert map[string]struct{}
delete map[string]struct{} delete map[string]struct{}
origin map[string][]byte
} }
// newTracer initializes trie node diff tracer. // newTracer initializes the tracer for capturing trie changes.
func newTracer() *tracer { func newTracer() *tracer {
return &tracer{ return &tracer{
insert: make(map[string]struct{}), insert: make(map[string]struct{}),
delete: make(map[string]struct{}), delete: make(map[string]struct{}),
origin: make(map[string][]byte),
} }
} }
// onInsert tracks the newly inserted trie node. If it's already // onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally.
// in the deletion set(resurrected node), then just wipe it from // Don't change the value outside of function since it's not deep-copied.
// the deletion set as it's untouched. func (t *tracer) onRead(key []byte, val []byte) {
// Tracer isn't used right now, remove this check later.
if t == nil {
return
}
t.origin[string(key)] = val
}
// onInsert tracks the newly inserted trie node. If it's already in the deletion set
// (resurrected node), then just wipe it from the deletion set as the "untouched".
func (t *tracer) onInsert(key []byte) { func (t *tracer) onInsert(key []byte) {
// Tracer isn't used right now, remove this check later. // Tracer isn't used right now, remove this check later.
if t == nil { if t == nil {
@ -100,6 +115,15 @@ func (t *tracer) deleteList() [][]byte {
return ret return ret
} }
// getPrev returns the cached original value of the specified node.
func (t *tracer) getPrev(key []byte) []byte {
// Don't panic on uninitialized tracer, it's possible in testing.
if t == nil {
return nil
}
return t.origin[string(key)]
}
// reset clears the content tracked by tracer. // reset clears the content tracked by tracer.
func (t *tracer) reset() { func (t *tracer) reset() {
// Tracer isn't used right now, remove this check later. // Tracer isn't used right now, remove this check later.
@ -108,6 +132,7 @@ func (t *tracer) reset() {
} }
t.insert = make(map[string]struct{}) t.insert = make(map[string]struct{})
t.delete = make(map[string]struct{}) t.delete = make(map[string]struct{})
t.origin = make(map[string][]byte)
} }
// copy returns a deep copied tracer instance. // copy returns a deep copied tracer instance.
@ -119,6 +144,7 @@ func (t *tracer) copy() *tracer {
var ( var (
insert = make(map[string]struct{}) insert = make(map[string]struct{})
delete = make(map[string]struct{}) delete = make(map[string]struct{})
origin = make(map[string][]byte)
) )
for key := range t.insert { for key := range t.insert {
insert[key] = struct{}{} insert[key] = struct{}{}
@ -126,8 +152,12 @@ func (t *tracer) copy() *tracer {
for key := range t.delete { for key := range t.delete {
delete[key] = struct{}{} delete[key] = struct{}{}
} }
for key, val := range t.origin {
origin[key] = val
}
return &tracer{ return &tracer{
insert: insert, insert: insert,
delete: delete, delete: delete,
origin: origin,
} }
} }