cerc-io/go-ethereum
18
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

Patch for concurrent iterator & others (onto v1.11.6) #386

Closed
roysc wants to merge 1565 commits from v1.11.6-statediff-v5 into master
pull from: v1.11.6-statediff-v5
merge into: cerc-io:master
Conversation 0 Commits 1565 Files Changed 1471 +578 -548
40 changed files with 578 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 22defa5af7 - Show all commits

2
cmd/geth/dbcmd.go
View File

@ -519,7 +519,7 @@ func dbDumpTrie(ctx *cli.Context) error {
return err
}
}
theTrie, err := trie.New(stRoot, trie.NewDatabase(db))
theTrie, err := trie.New(common.Hash{}, stRoot, trie.NewDatabase(db))
if err != nil {
return err
}

8
cmd/geth/snapshot.go
View File

@ -278,7 +278,7 @@ func traverseState(ctx *cli.Context) error {
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(root, triedb)
t, err := trie.NewSecure(common.Hash{}, root, triedb)
if err != nil {
log.Error("Failed to open trie", "root", root, "err", err)
return err
@ -299,7 +299,7 @@ func traverseState(ctx *cli.Context) error {
return err
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(acc.Root, triedb)
storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb)
if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
return err
@ -367,7 +367,7 @@ func traverseRawState(ctx *cli.Context) error {
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(root, triedb)
t, err := trie.NewSecure(common.Hash{}, root, triedb)
if err != nil {
log.Error("Failed to open trie", "root", root, "err", err)
return err
@ -413,7 +413,7 @@ func traverseRawState(ctx *cli.Context) error {
return errors.New("invalid account")
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(acc.Root, triedb)
storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb)
if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
return errors.New("missing storage trie")

2
core/blockchain.go
View File

@ -674,7 +674,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x..]", hash[:4])
}
if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil {
if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil {
return err
}

4
core/state/database.go
View File

@ -133,7 +133,7 @@ type cachingDB struct {
// OpenTrie opens the main account trie at a specific root hash.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(root, db.db)
tr, err := trie.NewSecure(common.Hash{}, root, db.db)
if err != nil {
return nil, err
}
@ -142,7 +142,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
// OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(root, db.db)
tr, err := trie.NewSecure(addrHash, root, db.db)
if err != nil {
return nil, err
}

4
core/state/pruner/pruner.go
View File

@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil {
return errors.New("missing genesis block")
}
t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db))
t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db))
if err != nil {
return err
}
@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
return err
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db))
storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db))
if err != nil {
return err
}

8
core/state/snapshot/conversion.go
View File

@ -43,7 +43,7 @@ type trieKV struct {
type (
// trieGeneratorFn is the interface of trie generation which can
// be implemented by different trie algorithm.
trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash))
trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash))
// leafCallbackFn is the callback invoked at the leaves of the trie,
// returns the subtrie root with the specified subtrie identifier.
@ -253,7 +253,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
wg.Add(1)
go func() {
defer wg.Done()
generatorFn(db, in, out)
generatorFn(db, account, in, out)
}()
// Spin up a go-routine for progress logging
if report && stats != nil {
@ -360,8 +360,8 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
return stop(nil)
}
func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) {
t := trie.NewStackTrie(db)
func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) {
t := trie.NewStackTrieWithOwner(db, owner)
for leaf := range in {
t.TryUpdate(leaf.key[:], leaf.value)
}

18
core/state/snapshot/generate.go
View File

@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error)
//
// The proof result will be returned if the range proving is finished, otherwise
// the error will be returned to abort the entire procedure.
func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
var (
keys [][]byte
vals [][]byte
@ -234,7 +234,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix
// The snap state is exhausted, pass the entire key/val set for verification
if origin == nil && !diskMore {
stackTr := trie.NewStackTrie(nil)
stackTr := trie.NewStackTrieWithOwner(nil, owner)
for i, key := range keys {
stackTr.TryUpdate(key, vals[i])
}
@ -248,7 +248,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix
return &proofResult{keys: keys, vals: vals}, nil
}
// Snap state is chunked, generate edge proofs for verification.
tr, err := trie.New(root, dl.triedb)
tr, err := trie.New(owner, root, dl.triedb)
if err != nil {
ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return nil, errMissingTrie
@ -313,9 +313,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
// generateRange generates the state segment with particular prefix. Generation can
// either verify the correctness of existing state through range-proof and skip
// generation, or iterate trie to regenerate state on demand.
func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
// Use range prover to check the validity of the flat state in the range
result, err := dl.proveRange(ctx, root, prefix, kind, origin, max, valueConvertFn)
result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn)
if err != nil {
return false, nil, err
}
@ -363,7 +363,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref
if len(result.keys) > 0 {
snapNodeCache = memorydb.New()
snapTrieDb := trie.NewDatabase(snapNodeCache)
snapTrie, _ := trie.New(common.Hash{}, snapTrieDb)
snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb)
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])
}
@ -374,7 +374,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref
// if it's already opened with some nodes resolved.
tr := result.tr
if tr == nil {
tr, err = trie.New(root, dl.triedb)
tr, err = trie.New(owner, root, dl.triedb)
if err != nil {
ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return false, nil, errMissingTrie
@ -537,7 +537,7 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash,
// Loop for re-generating the missing storage slots.
var origin = common.CopyBytes(storeMarker)
for {
exhausted, last, err := dl.generateRange(ctx, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
@ -637,7 +637,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
}
origin := common.CopyBytes(accMarker)
for {
exhausted, last, err := dl.generateRange(ctx, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}

476
core/state/snapshot/generate_test.go
View File

@ -26,61 +26,12 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"golang.org/x/crypto/sha3"
)
// Tests that snapshot generation from an empty database.
func TestGeneration(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
triedb.Commit(root, false, nil)
if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
t.Fatalf("have %#x want %#x", have, want)
}
snap := generateSnapshot(diskdb, triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats)
snap.genAbort <- stop
<-stop
}
func hashData(input []byte) common.Hash {
var hasher = sha3.NewLegacyKeccak256()
var hash common.Hash
@ -90,48 +41,25 @@ func hashData(input []byte) common.Hash {
return hash
}
// Tests that snapshot generation with existent flat state.
func TestGenerateExistentState(t *testing.T) {
// Tests that snapshot generation from an empty database.
func TestGeneration(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val)
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3"))
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
diskdb.Put(hashData([]byte("acc-2")).Bytes(), val)
rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val)
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3"))
root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
root, snap := helper.CommitAndGenerate()
if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
t.Fatalf("have %#x want %#x", have, want)
}
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -140,6 +68,43 @@ func TestGenerateExistentState(t *testing.T) {
t.Errorf("Snapshot generation failed")
}
checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats)
snap.genAbort <- stop
<-stop
}
// Tests that snapshot generation with existent flat state.
func TestGenerateExistentState(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats)
snap.genAbort <- stop
@ -163,7 +128,6 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
}
return hash, nil
}, newGenerateStats(), true)
if err != nil {
t.Fatal(err)
}
@ -176,15 +140,15 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
}
type testHelper struct {
diskdb *memorydb.Database
diskdb ethdb.Database
triedb *trie.Database
accTrie *trie.SecureTrie
}
func newHelper() *testHelper {
diskdb := memorydb.New()
diskdb := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(diskdb)
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
return &testHelper{
diskdb: diskdb,
triedb: triedb,
@ -215,18 +179,28 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string)
}
}
func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte {
stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb)
func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte {
stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb)
for i, k := range keys {
stTrie.Update([]byte(k), []byte(vals[i]))
}
root, _, _ := stTrie.Commit(nil)
var root common.Hash
if !commit {
root = stTrie.Hash()
} else {
root, _, _ = stTrie.Commit(nil)
}
return root.Bytes()
}
func (t *testHelper) Generate() (common.Hash, *diskLayer) {
func (t *testHelper) Commit() common.Hash {
root, _, _ := t.accTrie.Commit(nil)
t.triedb.Commit(root, false, nil)
return root
}
func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
root := t.Commit()
snap := generateSnapshot(t.diskdb, t.triedb, 16, root)
return root, snap
}
@ -249,26 +223,29 @@ func (t *testHelper) Generate() (common.Hash, *diskLayer) {
// - extra slots in the end
func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
helper := newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Account one, empty root but non-empty database
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Account two, non empty root but empty database
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
// Miss slots
{
// Account three, non empty root but misses slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"})
// Account four, non empty root but misses slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"})
// Account five, non empty root but misses slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"})
}
@ -276,18 +253,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// Wrong storage slots
{
// Account six, non empty root but wrong slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"})
// Account seven, non empty root but wrong slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"})
// Account eight, non empty root but wrong slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"})
// Account 9, non empty root but rotated slots
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"})
}
@ -295,19 +276,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// Extra storage slots
{
// Account 10, non empty root but extra slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"})
// Account 11, non empty root but extra slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"})
// Account 12, non empty root but extra slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"})
}
root, snap := helper.Generate()
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0
select {
@ -331,7 +315,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// - extra accounts
func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper := newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
// Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6]
// Extra accounts [acc-0, acc-5, acc-7]
@ -359,7 +348,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end
}
root, snap := helper.Generate()
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8
select {
@ -383,29 +372,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// without any storage slots to keep the test smaller.
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
tr, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
helper := newHelper()
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
tr.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
// Delete an account trie leaf and ensure the generator chokes
triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil)
diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
helper.triedb.Commit(root, false, nil)
helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"))
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -427,45 +406,30 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper := newHelper()
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root, _, _ := helper.accTrie.Commit(nil)
// We can only corrupt the disk database, so flush the tries out
triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
)
triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
)
triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil)
helper.triedb.Commit(root, false, nil)
// Delete a storage trie root and ensure the generator chokes
diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes())
helper.diskdb.Delete(stRoot)
snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"))
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -486,45 +450,31 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper := newHelper()
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
root, _, _ := helper.accTrie.Commit(nil)
// We can only corrupt the disk database, so flush the tries out
triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
)
triedb.Reference(
common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
)
triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil)
helper.triedb.Commit(root, false, nil)
// Delete a storage trie leaf and ensure the generator chokes
diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"))
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -539,56 +489,51 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
<-stop
}
func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie {
stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
for i := 0; i < n; i++ {
k := fmt.Sprintf("key-%d", i)
v := fmt.Sprintf("val-%d", i)
stTrie.Update([]byte(k), []byte(v))
}
stTrie.Commit(nil)
return stTrie
}
// Tests that snapshot generation when an extra account with storage exists in the snap state.
func TestGenerateWithExtraAccounts(t *testing.T) {
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
stTrie = getStorageTrie(5, triedb)
helper := newHelper()
{
// Account one in the trie
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true,
)
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
{ // Account one in the trie
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
// Identical in the snap
key := hashData([]byte("acc-1"))
rawdb.WriteAccountSnapshot(diskdb, key, val)
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5"))
rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3"))
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4"))
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5"))
}
{ // Account two exists only in the snapshot
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
{
// Account two exists only in the snapshot
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")),
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true,
)
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte("acc-2"))
rawdb.WriteAccountSnapshot(diskdb, key, val)
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3"))
rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1"))
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2"))
rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3"))
}
root, _, _ := accTrie.Commit(nil)
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
root := helper.Commit()
// To verify the test: If we now inspect the snap db, there should exist extraneous storage items
if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
t.Fatalf("expected snap storage to exist")
}
snap := generateSnapshot(diskdb, triedb, 16, root)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -597,12 +542,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
t.Errorf("Snapshot generation failed")
}
checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
stop := make(chan *generatorStats)
snap.genAbort <- stop
<-stop
// If we now inspect the snap db, there should exist no extraneous storage items
if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
t.Fatalf("expected slot to be removed, got %v", string(data))
}
}
@ -616,37 +562,36 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
if false {
enableLogging()
}
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
stTrie = getStorageTrie(3, triedb)
helper := newHelper()
{
// Account one in the trie
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
[]string{"key-1", "key-2", "key-3"},
[]string{"val-1", "val-2", "val-3"},
true,
)
accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
{ // Account one in the trie
acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
// Identical in the snap
key := hashData([]byte("acc-1"))
rawdb.WriteAccountSnapshot(diskdb, key, val)
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
}
{ // 100 accounts exist only in snapshot
{
// 100 accounts exist only in snapshot
for i := 0; i < 1000; i++ {
//acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
rawdb.WriteAccountSnapshot(diskdb, key, val)
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
}
}
root, _, _ := accTrie.Commit(nil)
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -675,31 +620,22 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
if false {
enableLogging()
}
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
accTrie, _ := trie.New(common.Hash{}, triedb)
helper := newHelper()
{
acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update(common.HexToHash("0x03").Bytes(), val)
accTrie.Update(common.HexToHash("0x07").Bytes(), val)
helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val)
}
root, _, _ := accTrie.Commit(nil)
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -721,29 +657,20 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
if false {
enableLogging()
}
var (
diskdb = memorydb.New()
triedb = trie.NewDatabase(diskdb)
)
accTrie, _ := trie.New(common.Hash{}, triedb)
helper := newHelper()
{
acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
accTrie.Update(common.HexToHash("0x03").Bytes(), val)
helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
junk := make([]byte, 100)
copy(junk, []byte{0xde, 0xad})
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk)
rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk)
}
root, _, _ := accTrie.Commit(nil)
t.Logf("root: %x", root)
triedb.Commit(root, false, nil)
snap := generateSnapshot(diskdb, triedb, 16, root)
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -757,7 +684,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
snap.genAbort <- stop
<-stop
// If we now inspect the snap db, there should exist no extraneous storage items
if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
t.Fatalf("expected slot to be removed, got %v", string(data))
}
}
@ -767,13 +694,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
accountCheckRange = 10
storageCheckRange = 20
helper := newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Add 1K accounts to the trie
for i := 0; i < 400; i++ {
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount(fmt.Sprintf("acc-%d", i),
&Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
}
root, snap := helper.Generate()
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
select {
@ -802,12 +729,12 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
helper := newHelper()
stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
stRoot := helper.makeStorageTrie(stKeys, stVals)
// We add 8 accounts, each one is missing exactly one of the storage slots. This means
// we don't have to order the keys and figure out exactly which hash-key winds up
// on the sensitive spots at the boundaries
for i := 0; i < 8; i++ {
accKey := fmt.Sprintf("acc-%d", i)
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true)
helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()})
var moddedKeys []string
var moddedVals []string
@ -819,8 +746,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
}
helper.addSnapStorage(accKey, moddedKeys, moddedVals)
}
root, snap := helper.Generate()
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff
select {
@ -899,7 +825,7 @@ func populateDangling(disk ethdb.KeyValueStore) {
// This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
@ -910,7 +836,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
populateDangling(helper.diskdb)
root, snap := helper.Generate()
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded
@ -932,7 +858,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
// This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper()
stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
@ -940,7 +866,7 @@ func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
populateDangling(helper.diskdb)
root, snap := helper.Generate()
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
// Snapshot generation succeeded

6
core/state/state_object.go
View File

@ -154,7 +154,7 @@ func (s *stateObject) getTrie(db Database) Trie {
if s.data.Root != emptyRoot && s.db.prefetcher != nil {
// When the miner is creating the pending state, there is no
// prefetcher
s.trie = s.db.prefetcher.trie(s.data.Root)
s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
}
if s.trie == nil {
var err error
@ -295,7 +295,7 @@ func (s *stateObject) finalise(prefetch bool) {
}
}
if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot {
s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch)
s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch)
}
if len(s.dirtyStorage) > 0 {
s.dirtyStorage = make(Storage)
@ -352,7 +352,7 @@ func (s *stateObject) updateTrie(db Database) Trie {
usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure
}
if s.db.prefetcher != nil {
s.db.prefetcher.used(s.data.Root, usedStorage)
s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage)
}
if len(s.pendingStorage) > 0 {
s.pendingStorage = make(Storage)

13
core/state/statedb.go
View File

@ -64,10 +64,13 @@ func (n *proofList) Delete(key []byte) error {
type StateDB struct {
db Database
prefetcher *triePrefetcher
originalRoot common.Hash // The pre-state root, before any changes were made
trie Trie
hasher crypto.KeccakState
// originalRoot is the pre-state root, before any changes were made.
// It will be updated when the Commit is called.
originalRoot common.Hash
snaps *snapshot.Tree
snap snapshot.Snapshot
snapDestructs map[common.Hash]struct{}
@ -648,6 +651,7 @@ func (s *StateDB) Copy() *StateDB {
state := &StateDB{
db: s.db,
trie: s.db.CopyTrie(s.trie),
originalRoot: s.originalRoot,
stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)),
stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)),
stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)),
@ -810,7 +814,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) {
addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure
}
if s.prefetcher != nil && len(addressesToPrefetch) > 0 {
s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch)
s.prefetcher.prefetch(common.Hash{}, s.originalRoot, addressesToPrefetch)
}
// Invalidate journal because reverting across transactions is not allowed.
s.clearJournalAndRefund()
@ -851,7 +855,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
// _untouched_. We can check with the prefetcher, if it can give us a trie
// which has the same root, but also has some content loaded into it.
if prefetcher != nil {
if trie := prefetcher.trie(s.originalRoot); trie != nil {
if trie := prefetcher.trie(common.Hash{}, s.originalRoot); trie != nil {
s.trie = trie
}
}
@ -867,7 +871,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure
}
if prefetcher != nil {
prefetcher.used(s.originalRoot, usedAddrs)
prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs)
}
if len(s.stateObjectsPending) > 0 {
s.stateObjectsPending = make(map[common.Address]struct{})
@ -980,6 +984,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
}
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
}
s.originalRoot = root
return root, err
}

6
core/state/sync_test.go
View File

@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
if v, _ := db.Get(root[:]); v == nil {
return nil // Consider a non existent state consistent.
}
trie, err := trie.New(root, trie.NewDatabase(db))
trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db))
if err != nil {
return err
}
@ -166,7 +166,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
if commit {
srcDb.TrieDB().Commit(srcRoot, false, nil)
}
srcTrie, _ := trie.New(srcRoot, srcDb.TrieDB())
srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB())
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
@ -207,7 +207,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil {
t.Fatalf("failed to decode account on path %x: %v", path, err)
}
stTrie, err := trie.New(acc.Root, srcDb.TrieDB())
stTrie, err := trie.New(common.BytesToHash(path[0]), acc.Root, srcDb.TrieDB())
if err != nil {
t.Fatalf("failed to retriev storage trie for path %x: %v", path, err)
}

56
core/state/trie_prefetcher.go
View File

@ -25,7 +25,7 @@ import (
)
var (
// triePrefetchMetricsPrefix is the prefix under which to publis the metrics.
// triePrefetchMetricsPrefix is the prefix under which to publish the metrics.
triePrefetchMetricsPrefix = "trie/prefetch/"
)
@ -36,9 +36,9 @@ var (
// Note, the prefetcher's API is not thread safe.
type triePrefetcher struct {
db Database // Database to fetch trie nodes through
root common.Hash // Root hash of theaccount trie for metrics
fetches map[common.Hash]Trie // Partially or fully fetcher tries
fetchers map[common.Hash]*subfetcher // Subfetchers for each trie
root common.Hash // Root hash of the account trie for metrics
fetches map[string]Trie // Partially or fully fetcher tries
fetchers map[string]*subfetcher // Subfetchers for each trie
deliveryMissMeter metrics.Meter
accountLoadMeter metrics.Meter
@ -51,13 +51,12 @@ type triePrefetcher struct {
storageWasteMeter metrics.Meter
}
// newTriePrefetcher
func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher {
prefix := triePrefetchMetricsPrefix + namespace
p := &triePrefetcher{
db: db,
root: root,
fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map
fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map
deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil),
accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil),
@ -112,7 +111,7 @@ func (p *triePrefetcher) copy() *triePrefetcher {
copy := &triePrefetcher{
db: p.db,
root: p.root,
fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map
fetches: make(map[string]Trie), // Active prefetchers use the fetches map
deliveryMissMeter: p.deliveryMissMeter,
accountLoadMeter: p.accountLoadMeter,
@ -132,33 +131,35 @@ func (p *triePrefetcher) copy() *triePrefetcher {
return copy
}
// Otherwise we're copying an active fetcher, retrieve the current states
for root, fetcher := range p.fetchers {
copy.fetches[root] = fetcher.peek()
for id, fetcher := range p.fetchers {
copy.fetches[id] = fetcher.peek()
}
return copy
}
// prefetch schedules a batch of trie items to prefetch.
func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) {
func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) {
// If the prefetcher is an inactive one, bail out
if p.fetches != nil {
return
}
// Active fetcher, schedule the retrievals
fetcher := p.fetchers[root]
id := p.trieID(owner, root)
fetcher := p.fetchers[id]
if fetcher == nil {
fetcher = newSubfetcher(p.db, root)
p.fetchers[root] = fetcher
fetcher = newSubfetcher(p.db, owner, root)
p.fetchers[id] = fetcher
}
fetcher.schedule(keys)
}
// trie returns the trie matching the root hash, or nil if the prefetcher doesn't
// have it.
func (p *triePrefetcher) trie(root common.Hash) Trie {
func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie {
// If the prefetcher is inactive, return from existing deep copies
id := p.trieID(owner, root)
if p.fetches != nil {
trie := p.fetches[root]
trie := p.fetches[id]
if trie == nil {
p.deliveryMissMeter.Mark(1)
return nil
@ -166,7 +167,7 @@ func (p *triePrefetcher) trie(root common.Hash) Trie {
return p.db.CopyTrie(trie)
}
// Otherwise the prefetcher is active, bail if no trie was prefetched for this root
fetcher := p.fetchers[root]
fetcher := p.fetchers[id]
if fetcher == nil {
p.deliveryMissMeter.Mark(1)
return nil
@ -185,18 +186,24 @@ func (p *triePrefetcher) trie(root common.Hash) Trie {
// used marks a batch of state items used to allow creating statistics as to
// how useful or wasteful the prefetcher is.
func (p *triePrefetcher) used(root common.Hash, used [][]byte) {
if fetcher := p.fetchers[root]; fetcher != nil {
func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) {
if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil {
fetcher.used = used
}
}
// trieID returns an unique trie identifier consists the trie owner and root hash.
func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string {
return string(append(owner.Bytes(), root.Bytes()...))
}
// subfetcher is a trie fetcher goroutine responsible for pulling entries for a
// single trie. It is spawned when a new root is encountered and lives until the
// main prefetcher is paused and either all requested items are processed or if
// the trie being worked on is retrieved from the prefetcher.
type subfetcher struct {
db Database // Database to load trie nodes through
owner common.Hash // Owner of the trie, usually account hash
root common.Hash // Root hash of the trie to prefetch
trie Trie // Trie being populated with nodes
@ -215,9 +222,10 @@ type subfetcher struct {
// newSubfetcher creates a goroutine to prefetch state items belonging to a
// particular root hash.
func newSubfetcher(db Database, root common.Hash) *subfetcher {
func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher {
sf := &subfetcher{
db: db,
owner: owner,
root: root,
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
@ -279,13 +287,21 @@ func (sf *subfetcher) loop() {
defer close(sf.term)
// Start by opening the trie and stop processing if it fails
if sf.owner == (common.Hash{}) {
trie, err := sf.db.OpenTrie(sf.root)
if err != nil {
log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
return
}
sf.trie = trie
} else {
trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root)
if err != nil {
log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
return
}
sf.trie = trie
}
// Trie opened successfully, keep prefetching items
for {
select {

36
core/state/trie_prefetcher_test.go
View File

@ -47,20 +47,20 @@ func TestCopyAndClose(t *testing.T) {
db := filledStateDB()
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
skey := common.HexToHash("aaa")
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
time.Sleep(1 * time.Second)
a := prefetcher.trie(db.originalRoot)
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
b := prefetcher.trie(db.originalRoot)
a := prefetcher.trie(common.Hash{}, db.originalRoot)
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
b := prefetcher.trie(common.Hash{}, db.originalRoot)
cpy := prefetcher.copy()
cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
c := cpy.trie(db.originalRoot)
cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
c := cpy.trie(common.Hash{}, db.originalRoot)
prefetcher.close()
cpy2 := cpy.copy()
cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
d := cpy2.trie(db.originalRoot)
cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
d := cpy2.trie(common.Hash{}, db.originalRoot)
cpy.close()
cpy2.close()
if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() {
@ -72,10 +72,10 @@ func TestUseAfterClose(t *testing.T) {
db := filledStateDB()
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
skey := common.HexToHash("aaa")
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
a := prefetcher.trie(db.originalRoot)
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
a := prefetcher.trie(common.Hash{}, db.originalRoot)
prefetcher.close()
b := prefetcher.trie(db.originalRoot)
b := prefetcher.trie(common.Hash{}, db.originalRoot)
if a == nil {
t.Fatal("Prefetching before close should not return nil")
}
@ -88,13 +88,13 @@ func TestCopyClose(t *testing.T) {
db := filledStateDB()
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "")
skey := common.HexToHash("aaa")
prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()})
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()})
cpy := prefetcher.copy()
a := prefetcher.trie(db.originalRoot)
b := cpy.trie(db.originalRoot)
a := prefetcher.trie(common.Hash{}, db.originalRoot)
b := cpy.trie(common.Hash{}, db.originalRoot)
prefetcher.close()
c := prefetcher.trie(db.originalRoot)
d := cpy.trie(db.originalRoot)
c := prefetcher.trie(common.Hash{}, db.originalRoot)
d := cpy.trie(common.Hash{}, db.originalRoot)
if a == nil {
t.Fatal("Prefetching before close should not return nil")
}

12
core/types/hashing_test.go
View File

@ -39,8 +39,7 @@ func TestDeriveSha(t *testing.T) {
t.Fatal(err)
}
for len(txs) < 1000 {
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp := types.DeriveSha(txs, tr)
exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
got := types.DeriveSha(txs, trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@ -87,8 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp = types.DeriveSha(txs, tr)
exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
}
})
@ -109,8 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) {
rndSeed := mrand.Int()
for i := 0; i < 10; i++ {
seed := rndSeed + i
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp := types.DeriveSha(newDummy(i), tr)
exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed))
@ -138,8 +135,7 @@ func TestDerivableList(t *testing.T) {
},
}
for i, tc := range tcs[1:] {
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp := types.DeriveSha(flatList(tc), tr)
exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp)

4
eth/api.go
View File

@ -529,11 +529,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
}
triedb := api.eth.BlockChain().StateCache().TrieDB()
oldTrie, err := trie.NewSecure(startBlock.Root(), triedb)
oldTrie, err := trie.NewSecure(common.Hash{}, startBlock.Root(), triedb)
if err != nil {
return nil, err
}
newTrie, err := trie.NewSecure(endBlock.Root(), triedb)
newTrie, err := trie.NewSecure(common.Hash{}, endBlock.Root(), triedb)
if err != nil {
return nil, err
}

10
eth/protocols/snap/handler.go
View File

@ -285,7 +285,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac
req.Bytes = softResponseLimit
}
// Retrieve the requested state and bail out if non existent
tr, err := trie.New(req.Root, chain.StateCache().TrieDB())
tr, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB())
if err != nil {
return nil, nil
}
@ -415,7 +415,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
if origin != (common.Hash{}) || (abort && len(storage) > 0) {
// Request started at a non-zero hash or was capped prematurely, add
// the endpoint Merkle proofs
accTrie, err := trie.New(req.Root, chain.StateCache().TrieDB())
accTrie, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB())
if err != nil {
return nil, nil
}
@ -423,7 +423,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil {
return nil, nil
}
stTrie, err := trie.New(acc.Root, chain.StateCache().TrieDB())
stTrie, err := trie.New(account, acc.Root, chain.StateCache().TrieDB())
if err != nil {
return nil, nil
}
@ -489,7 +489,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
// Make sure we have the state associated with the request
triedb := chain.StateCache().TrieDB()
accTrie, err := trie.NewSecure(req.Root, triedb)
accTrie, err := trie.NewSecure(common.Hash{}, req.Root, triedb)
if err != nil {
// We don't have the requested state available, bail out
return nil, nil
@ -531,7 +531,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
if err != nil || account == nil {
break
}
stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb)
stTrie, err := trie.NewSecure(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb)
loads++ // always account database reads, even for failures
if err != nil {
break

10
eth/protocols/snap/sync.go
View File

@ -712,7 +712,7 @@ func (s *Syncer) loadSyncStatus() {
}
task.genTrie = trie.NewStackTrie(task.genBatch)
for _, subtasks := range task.SubTasks {
for accountHash, subtasks := range task.SubTasks {
for _, subtask := range subtasks {
subtask.genBatch = ethdb.HookedBatch{
Batch: s.db.NewBatch(),
@ -720,7 +720,7 @@ func (s *Syncer) loadSyncStatus() {
s.storageBytes += common.StorageSize(len(key) + len(value))
},
}
subtask.genTrie = trie.NewStackTrie(subtask.genBatch)
subtask.genTrie = trie.NewStackTrieWithOwner(subtask.genBatch, accountHash)
}
}
}
@ -1971,7 +1971,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
Last: r.End(),
root: acc.Root,
genBatch: batch,
genTrie: trie.NewStackTrie(batch),
genTrie: trie.NewStackTrieWithOwner(batch, account),
})
for r.Next() {
batch := ethdb.HookedBatch{
@ -1985,7 +1985,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
Last: r.End(),
root: acc.Root,
genBatch: batch,
genTrie: trie.NewStackTrie(batch),
genTrie: trie.NewStackTrieWithOwner(batch, account),
})
}
for _, task := range tasks {
@ -2030,7 +2030,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
slots += len(res.hashes[i])
if i < len(res.hashes)-1 || res.subTask == nil {
tr := trie.NewStackTrie(batch)
tr := trie.NewStackTrieWithOwner(batch, account)
for j := 0; j < len(res.hashes[i]); j++ {
tr.Update(res.hashes[i][j][:], res.slots[i][j])
}

48
eth/protocols/snap/sync_test.go
View File

@ -1349,7 +1349,7 @@ func getCodeByHash(hash common.Hash) []byte {
// makeAccountTrieNoStorage spits out a trie, along with the leafs
func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) {
db := trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie, _ := trie.New(common.Hash{}, db)
accTrie := trie.NewEmpty(db)
var entries entrySlice
for i := uint64(1); i <= uint64(n); i++ {
value, _ := rlp.EncodeToBytes(&types.StateAccount{
@ -1377,7 +1377,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
boundaries []common.Hash
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
trie, _ = trie.New(common.Hash{}, db)
trie = trie.NewEmpty(db)
)
// Initialize boundaries
var next common.Hash
@ -1429,7 +1429,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie, _ = trie.New(common.Hash{}, db)
accTrie = trie.NewEmpty(db)
entries entrySlice
storageTries = make(map[common.Hash]*trie.Trie)
storageEntries = make(map[common.Hash]entrySlice)
@ -1442,7 +1442,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
codehash = getCodeHash(i)
}
// Create a storage trie
stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db)
stTrie, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db)
stRoot := stTrie.Hash()
stTrie.Commit(nil)
value, _ := rlp.EncodeToBytes(&types.StateAccount{
@ -1468,23 +1468,11 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie, _ = trie.New(common.Hash{}, db)
accTrie = trie.NewEmpty(db)
entries entrySlice
storageTries = make(map[common.Hash]*trie.Trie)
storageEntries = make(map[common.Hash]entrySlice)
)
// Make a storage trie which we reuse for the whole lot
var (
stTrie *trie.Trie
stEntries entrySlice
)
if boundary {
stTrie, stEntries = makeBoundaryStorageTrie(slots, db)
} else {
stTrie, stEntries = makeStorageTrieWithSeed(uint64(slots), 0, db)
}
stRoot := stTrie.Hash()
// Create n accounts in the trie
for i := uint64(1); i <= uint64(accounts); i++ {
key := key32(i)
@ -1492,6 +1480,19 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
if code {
codehash = getCodeHash(i)
}
// Make a storage trie
var (
stTrie *trie.Trie
stEntries entrySlice
)
if boundary {
stTrie, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db)
} else {
stTrie, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db)
}
stRoot := stTrie.Hash()
stTrie.Commit(nil)
value, _ := rlp.EncodeToBytes(&types.StateAccount{
Nonce: i,
Balance: big.NewInt(int64(i)),
@ -1506,7 +1507,6 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
storageEntries[common.BytesToHash(key)] = stEntries
}
sort.Sort(entries)
stTrie.Commit(nil)
accTrie.Commit(nil)
return accTrie, entries, storageTries, storageEntries
}
@ -1514,8 +1514,8 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
// makeStorageTrieWithSeed fills a storage trie with n items, returning the
// not-yet-committed trie and the sorted entries. The seeds can be used to ensure
// that tries are unique.
func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) {
trie, _ := trie.New(common.Hash{}, db)
func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) {
trie, _ := trie.New(owner, common.Hash{}, db)
var entries entrySlice
for i := uint64(1); i <= n; i++ {
// store 'x' at slot 'x'
@ -1537,11 +1537,11 @@ func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, ent
// makeBoundaryStorageTrie constructs a storage trie. Instead of filling
// storage slots normally, this function will fill a few slots which have
// boundary hash.
func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) {
func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie.Trie, entrySlice) {
var (
entries entrySlice
boundaries []common.Hash
trie, _ = trie.New(common.Hash{}, db)
trie, _ = trie.New(owner, common.Hash{}, db)
)
// Initialize boundaries
var next common.Hash
@ -1588,7 +1588,7 @@ func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice)
func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
t.Helper()
triedb := trie.NewDatabase(db)
accTrie, err := trie.New(root, triedb)
accTrie, err := trie.New(common.Hash{}, root, triedb)
if err != nil {
t.Fatal(err)
}
@ -1606,7 +1606,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
}
accounts++
if acc.Root != emptyRoot {
storeTrie, err := trie.NewSecure(acc.Root, triedb)
storeTrie, err := trie.NewSecure(common.BytesToHash(accIt.Key), acc.Root, triedb)
if err != nil {
t.Fatal(err)
}

2
les/downloader/downloader_test.go
View File

@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block {
func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error {
// For now only check that the state trie is correct
if block := dl.GetBlockByHash(hash); block != nil {
_, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb))
_, err := trie.NewSecure(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb))
return err
}
return fmt.Errorf("non existent block: %x", hash[:4])

8
les/handler_test.go
View File

@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) {
accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}}
for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ {
header := bc.GetHeaderByNumber(i)
trie, _ := trie.New(header.Root, trie.NewDatabase(server.db))
trie, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db))
for _, acc := range accounts {
req := ProofReq{
@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) {
var expected []rlp.RawValue
if wantOK {
proofsV2 := light.NewNodeSet()
t, _ := trie.New(header.Root, trie.NewDatabase(server.db))
t, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db))
t.Prove(account, 0, proofsV2)
expected = proofsV2.NodeList()
}
@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) {
AuxData: [][]byte{rlp},
}
root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix)))
trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix)))
trie.Prove(key, 0, &proofsV2.Proofs)
// Assemble the requests for the different protocols
requestsV2 := []HelperTrieReq{{
@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) {
var proofs HelperTrieResps
root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix)))
trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix)))
trie.Prove(key, 0, &proofs.Proofs)
// Send the proof request and verify the response

4
les/server_handler.go
View File

@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool {
// getAccount retrieves an account from the state based on root.
func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) {
trie, err := trie.New(root, triedb)
trie, err := trie.New(common.Hash{}, root, triedb)
if err != nil {
return types.StateAccount{}, err
}
@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie {
if root == (common.Hash{}) {
return nil
}
trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix)))
trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix)))
return trie
}

2
light/odr_test.go
View File

@ -82,7 +82,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number)
}
case *TrieRequest:
t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb))
t, _ := trie.New(common.BytesToHash(req.Id.AccKey), req.Id.Root, trie.NewDatabase(odr.sdb))
nodes := NewNodeSet()
t.Prove(req.Key, 0, nodes)
req.Proof = nodes

13
light/postprocess.go
View File

@ -187,12 +187,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti
root = GetChtRoot(c.diskdb, section-1, lastSectionHead)
}
var err error
c.trie, err = trie.New(root, c.triedb)
c.trie, err = trie.New(common.Hash{}, root, c.triedb)
if err != nil && c.odr != nil {
err = c.fetchMissingNodes(ctx, section, root)
if err == nil {
c.trie, err = trie.New(root, c.triedb)
c.trie, err = trie.New(common.Hash{}, root, c.triedb)
}
}
c.section = section
@ -253,9 +253,8 @@ func (c *ChtIndexerBackend) Commit() error {
return nil
}
// PruneSections implements core.ChainIndexerBackend which deletes all
// chain data(except hash<->number mappings) older than the specified
// threshold.
// Prune implements core.ChainIndexerBackend which deletes all chain data
// (except hash<->number mappings) older than the specified threshold.
func (c *ChtIndexerBackend) Prune(threshold uint64) error {
// Short circuit if the light pruning is disabled.
if c.disablePruning {
@ -404,11 +403,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las
root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead)
}
var err error
b.trie, err = trie.New(root, b.triedb)
b.trie, err = trie.New(common.Hash{}, root, b.triedb)
if err != nil && b.odr != nil {
err = b.fetchMissingNodes(ctx, section, root)
if err == nil {
b.trie, err = trie.New(root, b.triedb)
b.trie, err = trie.New(common.Hash{}, root, b.triedb)
}
}
b.section = section

12
light/trie.go
View File

@ -169,7 +169,11 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
for {
var err error
if t.trie == nil {
t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database()))
var owner common.Hash
if len(t.id.AccKey) > 0 {
owner = common.BytesToHash(t.id.AccKey)
}
t.trie, err = trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database()))
}
if err == nil {
err = fn()
@ -195,7 +199,11 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator {
// Open the actual non-ODR trie if that hasn't happened yet.
if t.trie == nil {
it.do(func() error {
t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database()))
var owner common.Hash
if len(t.id.AccKey) > 0 {
owner = common.BytesToHash(t.id.AccKey)
}
t, err := trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database()))
if err == nil {
it.t.trie = t
}

4
tests/fuzzers/les/les-fuzzer.go
View File

@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) {
}
func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) {
chtTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
bloomTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < testChainLen; i++ {
// The element in CHT is <big-endian block number> -> <block hash>
key := make([]byte, 8)

2
tests/fuzzers/rangeproof/rangeproof-fuzzer.go
View File

@ -62,7 +62,7 @@ func (f *fuzzer) readInt() uint64 {
}
func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {
trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
size := f.readInt()
// Fill it with some fluff

3
tests/fuzzers/stacktrie/trie_fuzzer.go
View File

@ -25,7 +25,6 @@ import (
"io"
"sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"golang.org/x/crypto/sha3"
@ -144,7 +143,7 @@ func (f *fuzzer) fuzz() int {
var (
spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbA = trie.NewDatabase(spongeA)
trieA, _ = trie.New(common.Hash{}, dbA)
trieA = trie.NewEmpty(dbA)
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
trieB = trie.NewStackTrie(spongeB)
vals kvs

6
tests/fuzzers/trie/trie-fuzzer.go
View File

@ -144,7 +144,7 @@ func runRandTest(rt randTest) error {
triedb := trie.NewDatabase(memorydb.New())
tr, _ := trie.New(common.Hash{}, triedb)
tr := trie.NewEmpty(triedb)
values := make(map[string]string) // tracks content of the trie
for i, step := range rt {
@ -170,13 +170,13 @@ func runRandTest(rt randTest) error {
if err != nil {
return err
}
newtr, err := trie.New(hash, triedb)
newtr, err := trie.New(common.Hash{}, hash, triedb)
if err != nil {
return err
}
tr = newtr
case opItercheckhash:
checktr, _ := trie.New(common.Hash{}, triedb)
checktr := trie.NewEmpty(triedb)
it := trie.NewIterator(tr.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)

13
trie/errors.go
View File

@ -26,10 +26,21 @@ import (
// in the case where a trie node is not present in the local database. It contains
// information necessary for retrieving the missing node.
type MissingNodeError struct {
Owner common.Hash // owner of the trie if it's 2-layered trie
NodeHash common.Hash // hash of the missing node
Path []byte // hex-encoded path to the missing node
err error // concrete error for missing trie node
}
// Unwrap returns the concrete error for missing trie node which
// allows us for further analysis outside.
func (err *MissingNodeError) Unwrap() error {
return err.err
}
func (err *MissingNodeError) Error() string {
return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path)
if err.Owner == (common.Hash{}) {
return fmt.Sprintf("missing trie node %x (path %x) %v", err.NodeHash, err.Path, err.err)
}
return fmt.Sprintf("missing trie node %x (owner %x) (path %x) %v", err.NodeHash, err.Owner, err.Path, err.err)
}

14
trie/iterator_test.go
View File

@ -297,7 +297,7 @@ func TestUnionIterator(t *testing.T) {
}
func TestIteratorNoDups(t *testing.T) {
tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v))
}
@ -312,7 +312,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
tr, _ := New(common.Hash{}, triedb)
tr := NewEmpty(triedb)
for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v))
}
@ -337,7 +337,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
}
for i := 0; i < 20; i++ {
// Create trie that will load all nodes from DB.
tr, _ := New(tr.Hash(), triedb)
tr, _ := New(common.Hash{}, tr.Hash(), triedb)
// Remove a random node from the database. It can't be the root node
// because that one is already loaded.
@ -403,7 +403,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
ctr, _ := New(common.Hash{}, triedb)
ctr := NewEmpty(triedb)
for _, val := range testdata1 {
ctr.Update([]byte(val.k), []byte(val.v))
}
@ -425,7 +425,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
}
// Create a new iterator that seeks to "bars". Seeking can't proceed because
// the node is missing.
tr, _ := New(root, triedb)
tr, _ := New(common.Hash{}, root, triedb)
it := tr.NodeIterator([]byte("bars"))
missing, ok := it.Error().(*MissingNodeError)
if !ok {
@ -513,7 +513,7 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) {
// Create an empty trie
logDb := &loggingDb{0, memorydb.New()}
triedb := NewDatabase(logDb)
trie, _ := NewSecure(common.Hash{}, triedb)
trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb)
// Fill it with some arbitrary data
for i := 0; i < 10000; i++ {
@ -548,7 +548,7 @@ func TestIteratorNodeBlob(t *testing.T) {
var (
db = memorydb.New()
triedb = NewDatabase(db)
trie, _ = New(common.Hash{}, triedb)
trie = NewEmpty(triedb)
)
vals := []struct{ k, v string }{
{"do", "verb"},

20
trie/proof_test.go
View File

@ -80,7 +80,7 @@ func TestProof(t *testing.T) {
}
func TestOneElementProof(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "k", "v")
for i, prover := range makeProvers(trie) {
proof := prover([]byte("k"))
@ -131,7 +131,7 @@ func TestBadProof(t *testing.T) {
// Tests that missing keys can also be proven. The test explicitly uses a single
// entry trie and checks for missing keys both before and after the single entry.
func TestMissingKeyProof(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "k", "v")
for i, key := range []string{"a", "j", "l", "z"} {
@ -387,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) {
}
// Test the mini trie with only a single element.
tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
entry := &kv{randBytes(32), randBytes(20), false}
tinyTrie.Update(entry.k, entry.v)
@ -459,7 +459,7 @@ func TestAllElementsProof(t *testing.T) {
// TestSingleSideRangeProof tests the range starts from zero.
func TestSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
@ -494,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) {
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
func TestReverseSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
@ -601,7 +601,7 @@ func TestBadRangeProof(t *testing.T) {
// TestGappedRangeProof focuses on the small trie with embedded nodes.
// If the gapped node is embedded in the trie, it should be detected too.
func TestGappedRangeProof(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries []*kv // Sorted entries
for i := byte(0); i < 10; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -675,7 +675,7 @@ func TestSameSideProofs(t *testing.T) {
}
func TestHasRightElement(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
@ -1028,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) {
}
func randomTrie(n int) (*Trie, map[string]*kv) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -1053,7 +1053,7 @@ func randBytes(n int) []byte {
}
func nonRandomTrie(n int) (*Trie, map[string]*kv) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
max := uint64(0xffffffffffffffff)
for i := uint64(0); i < uint64(n); i++ {
@ -1078,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) {
common.Hex2Bytes("02"),
common.Hex2Bytes("03"),
}
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i, key := range keys {
trie.Update(key, vals[i])
}

4
trie/secure_trie.go
View File

@ -53,11 +53,11 @@ type SecureTrie struct {
// Loaded nodes are kept around until their 'cache generation' expires.
// A new cache generation is created by each call to Commit.
// cachelimit sets the number of past cache generations to keep.
func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) {
func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) {
if db == nil {
panic("trie.NewSecure called without a database")
}
trie, err := New(root, db)
trie, err := New(owner, root, db)
if err != nil {
return nil, err
}

4
trie/secure_trie_test.go
View File

@ -28,7 +28,7 @@ import (
)
func newEmptySecure() *SecureTrie {
trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New()))
trie, _ := NewSecure(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New()))
return trie
}
@ -36,7 +36,7 @@ func newEmptySecure() *SecureTrie {
func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(memorydb.New())
trie, _ := NewSecure(common.Hash{}, triedb)
trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb)
// Fill it with some arbitrary data
content := make(map[string][]byte)

47
trie/stacktrie.go
View File

@ -38,9 +38,10 @@ var stPool = sync.Pool{
},
}
func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie {
func stackTrieFromPool(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie {
st := stPool.Get().(*StackTrie)
st.db = db
st.owner = owner
return st
}
@ -53,6 +54,7 @@ func returnToPool(st *StackTrie) {
// in order. Once it determines that a subtree will no longer be inserted
// into, it will hash it and free up the memory it uses.
type StackTrie struct {
owner common.Hash // the owner of the trie
nodeType uint8 // node type (as in branch, ext, leaf)
val []byte // value contained by this node if it's a leaf
key []byte // key chunk covered by this (leaf|ext) node
@ -68,6 +70,16 @@ func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie {
}
}
// NewStackTrieWithOwner allocates and initializes an empty trie, but with
// the additional owner field.
func NewStackTrieWithOwner(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie {
return &StackTrie{
owner: owner,
nodeType: emptyNode,
db: db,
}
}
// NewFromBinary initialises a serialized stacktrie with the given db.
func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) {
var st StackTrie
@ -88,10 +100,12 @@ func (st *StackTrie) MarshalBinary() (data []byte, err error) {
w = bufio.NewWriter(&b)
)
if err := gob.NewEncoder(w).Encode(struct {
Nodetype uint8
Owner common.Hash
NodeType uint8
Val []byte
Key []byte
}{
st.owner,
st.nodeType,
st.val,
st.key,
@ -122,12 +136,14 @@ func (st *StackTrie) UnmarshalBinary(data []byte) error {
func (st *StackTrie) unmarshalBinary(r io.Reader) error {
var dec struct {
Nodetype uint8
Owner common.Hash
NodeType uint8
Val []byte
Key []byte
}
gob.NewDecoder(r).Decode(&dec)
st.nodeType = dec.Nodetype
st.owner = dec.Owner
st.nodeType = dec.NodeType
st.val = dec.Val
st.key = dec.Key
@ -154,16 +170,16 @@ func (st *StackTrie) setDb(db ethdb.KeyValueWriter) {
}
}
func newLeaf(key, val []byte, db ethdb.KeyValueWriter) *StackTrie {
st := stackTrieFromPool(db)
func newLeaf(owner common.Hash, key, val []byte, db ethdb.KeyValueWriter) *StackTrie {
st := stackTrieFromPool(db, owner)
st.nodeType = leafNode
st.key = append(st.key, key...)
st.val = val
return st
}
func newExt(key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie {
st := stackTrieFromPool(db)
func newExt(owner common.Hash, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie {
st := stackTrieFromPool(db, owner)
st.nodeType = extNode
st.key = append(st.key, key...)
st.children[0] = child
@ -196,6 +212,7 @@ func (st *StackTrie) Update(key, value []byte) {
}
func (st *StackTrie) Reset() {
st.owner = common.Hash{}
st.db = nil
st.key = st.key[:0]
st.val = nil
@ -236,7 +253,7 @@ func (st *StackTrie) insert(key, value []byte) {
// Add new child
if st.children[idx] == nil {
st.children[idx] = newLeaf(key[1:], value, st.db)
st.children[idx] = newLeaf(st.owner, key[1:], value, st.db)
} else {
st.children[idx].insert(key[1:], value)
}
@ -262,7 +279,7 @@ func (st *StackTrie) insert(key, value []byte) {
// node directly.
var n *StackTrie
if diffidx < len(st.key)-1 {
n = newExt(st.key[diffidx+1:], st.children[0], st.db)
n = newExt(st.owner, st.key[diffidx+1:], st.children[0], st.db)
} else {
// Break on the last byte, no need to insert
// an extension node: reuse the current node
@ -282,12 +299,12 @@ func (st *StackTrie) insert(key, value []byte) {
// the common prefix is at least one byte
// long, insert a new intermediate branch
// node.
st.children[0] = stackTrieFromPool(st.db)
st.children[0] = stackTrieFromPool(st.db, st.owner)
st.children[0].nodeType = branchNode
p = st.children[0]
}
// Create a leaf for the inserted part
o := newLeaf(key[diffidx+1:], value, st.db)
o := newLeaf(st.owner, key[diffidx+1:], value, st.db)
// Insert both child leaves where they belong:
origIdx := st.key[diffidx]
@ -323,7 +340,7 @@ func (st *StackTrie) insert(key, value []byte) {
// Convert current node into an ext,
// and insert a child branch node.
st.nodeType = extNode
st.children[0] = NewStackTrie(st.db)
st.children[0] = NewStackTrieWithOwner(st.db, st.owner)
st.children[0].nodeType = branchNode
p = st.children[0]
}
@ -332,11 +349,11 @@ func (st *StackTrie) insert(key, value []byte) {
// value and another containing the new value. The child leaf
// is hashed directly in order to free up some memory.
origIdx := st.key[diffidx]
p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db)
p.children[origIdx] = newLeaf(st.owner, st.key[diffidx+1:], st.val, st.db)
p.children[origIdx].hash()
newIdx := key[diffidx]
p.children[newIdx] = newLeaf(key[diffidx+1:], value, st.db)
p.children[newIdx] = newLeaf(st.owner, key[diffidx+1:], value, st.db)
// Finally, cut off the key part that has been passed
// over to the children.

14
trie/stacktrie_test.go
View File

@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) {
func TestSizeBug(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
nt := NewEmpty(NewDatabase(memorydb.New()))
leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) {
func TestEmptyBug(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
nt := NewEmpty(NewDatabase(memorydb.New()))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) {
func TestValLength56(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
nt := NewEmpty(NewDatabase(memorydb.New()))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -254,7 +254,8 @@ func TestValLength56(t *testing.T) {
// which causes a lot of node-within-node. This case was found via fuzzing.
func TestUpdateSmallNodes(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
nt := NewEmpty(NewDatabase(memorydb.New()))
kvs := []struct {
K string
V string
@ -282,7 +283,8 @@ func TestUpdateSmallNodes(t *testing.T) {
func TestUpdateVariableKeys(t *testing.T) {
t.SkipNow()
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
nt := NewEmpty(NewDatabase(memorydb.New()))
kvs := []struct {
K string
V string
@ -352,7 +354,7 @@ func TestStacktrieNotModifyValues(t *testing.T) {
func TestStacktrieSerialization(t *testing.T) {
var (
st = NewStackTrie(nil)
nt, _ = New(common.Hash{}, NewDatabase(memorydb.New()))
nt = NewEmpty(NewDatabase(memorydb.New()))
keyB = big.NewInt(1)
keyDelta = big.NewInt(1)
vals [][]byte

10
trie/sync_test.go
View File

@ -29,7 +29,7 @@ import (
func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(memorydb.New())
trie, _ := NewSecure(common.Hash{}, triedb)
trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb)
// Fill it with some arbitrary data
content := make(map[string][]byte)
@ -60,7 +60,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) {
// content map.
func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) {
// Check root availability and trie contents
trie, err := NewSecure(common.BytesToHash(root), db)
trie, err := NewSecure(common.Hash{}, common.BytesToHash(root), db)
if err != nil {
t.Fatalf("failed to create trie at %x: %v", root, err)
}
@ -77,7 +77,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri
// checkTrieConsistency checks that all nodes in a trie are indeed present.
func checkTrieConsistency(db *Database, root common.Hash) error {
// Create and iterate a trie rooted in a subnode
trie, err := NewSecure(root, db)
trie, err := NewSecure(common.Hash{}, root, db)
if err != nil {
return nil // Consider a non existent state consistent
}
@ -91,8 +91,8 @@ func checkTrieConsistency(db *Database, root common.Hash) error {
func TestEmptySync(t *testing.T) {
dbA := NewDatabase(memorydb.New())
dbB := NewDatabase(memorydb.New())
emptyA, _ := New(common.Hash{}, dbA)
emptyB, _ := New(emptyRoot, dbB)
emptyA := NewEmpty(dbA)
emptyB, _ := New(common.Hash{}, emptyRoot, dbB)
for i, trie := range []*Trie{emptyA, emptyB} {
sync := NewSync(trie.Hash(), memorydb.New(), nil)

64
trie/trie.go
View File

@ -63,6 +63,7 @@ type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent commo
type Trie struct {
db *Database
root node
owner common.Hash
// Keep track of the number leaves which have been inserted since the last
// hashing operation. This number will not directly map to the number of
@ -84,33 +85,27 @@ func (t *Trie) Copy() *Trie {
return &Trie{
db: t.db,
root: t.root,
owner: t.owner,
unhashed: t.unhashed,
tracer: t.tracer.copy(),
}
}
// New creates a trie with an existing root node from db.
// New creates a trie with an existing root node from db and an assigned
// owner for storage proximity.
//
// If root is the zero hash or the sha3 hash of an empty string, the
// trie is initially empty and does not require a database. Otherwise,
// New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand.
func New(root common.Hash, db *Database) (*Trie, error) {
if db == nil {
panic("trie.New called without a database")
}
trie := &Trie{
db: db,
//tracer: newTracer(),
}
if root != (common.Hash{}) && root != emptyRoot {
rootnode, err := trie.resolveHash(root[:], nil)
if err != nil {
return nil, err
}
trie.root = rootnode
}
return trie, nil
func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
return newTrie(owner, root, db)
}
// NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
func NewEmpty(db *Database) *Trie {
tr, _ := newTrie(common.Hash{}, common.Hash{}, db)
return tr
}
// newWithRootNode initializes the trie with the given root node.
@ -123,6 +118,26 @@ func newWithRootNode(root node) *Trie {
}
}
// newTrie is the internal function used to construct the trie with given parameters.
func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
if db == nil {
panic("trie.New called without a database")
}
trie := &Trie{
db: db,
owner: owner,
//tracer: newTracer(),
}
if root != (common.Hash{}) && root != emptyRoot {
rootnode, err := trie.resolveHash(root[:], nil)
if err != nil {
return nil, err
}
trie.root = rootnode
}
return trie, nil
}
// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
// the key after the given start key.
func (t *Trie) NodeIterator(start []byte) NodeIterator {
@ -562,7 +577,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
if node := t.db.node(hash); node != nil {
return node, nil
}
return nil, &MissingNodeError{NodeHash: hash, Path: prefix}
return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix}
}
func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
@ -571,7 +586,7 @@ func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
if len(blob) != 0 {
return blob, nil
}
return nil, &MissingNodeError{NodeHash: hash, Path: prefix}
return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix}
}
// Hash returns the root hash of the trie. It does not write to the
@ -602,7 +617,10 @@ func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
// Do a quick check if we really need to commit, before we spin
// up goroutines. This can happen e.g. if we load a trie for reading storage
// values, but don't write to it.
if _, dirty := t.root.cache(); !dirty {
if hashedNode, dirty := t.root.cache(); !dirty {
// Replace the root node with the origin hash in order to
// ensure all resolved nodes are dropped after the commit.
t.root = hashedNode
return rootHash, 0, nil
}
var wg sync.WaitGroup
@ -647,6 +665,12 @@ func (t *Trie) hashRoot() (node, node, error) {
// Reset drops the referenced root node and cleans all internal state.
func (t *Trie) Reset() {
t.root = nil
t.owner = common.Hash{}
t.unhashed = 0
t.tracer.reset()
}
// Owner returns the associated trie owner.
func (t *Trie) Owner() common.Hash {
return t.owner
}

54
trie/trie_test.go
View File

@ -48,12 +48,12 @@ func init() {
// Used for testing
func newEmpty() *Trie {
trie, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
trie := NewEmpty(NewDatabase(memorydb.New()))
return trie
}
func TestEmptyTrie(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
res := trie.Hash()
exp := emptyRoot
if res != exp {
@ -62,7 +62,7 @@ func TestEmptyTrie(t *testing.T) {
}
func TestNull(t *testing.T) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
key := make([]byte, 32)
value := []byte("test")
trie.Update(key, value)
@ -72,7 +72,7 @@ func TestNull(t *testing.T) {
}
func TestMissingRoot(t *testing.T) {
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New()))
trie, err := New(common.Hash{}, common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New()))
if trie != nil {
t.Error("New returned non-nil trie for invalid root")
}
@ -88,7 +88,7 @@ func testMissingNode(t *testing.T, memonly bool) {
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
trie, _ := New(common.Hash{}, triedb)
trie := NewEmpty(triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _, _ := trie.Commit(nil)
@ -96,27 +96,27 @@ func testMissingNode(t *testing.T, memonly bool) {
triedb.Commit(root, true, nil)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
_, err := trie.TryGet([]byte("120000"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("120099"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryDelete([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
@ -129,27 +129,27 @@ func testMissingNode(t *testing.T, memonly bool) {
diskdb.Delete(hash[:])
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("120000"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
_, err = trie.TryGet([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, triedb)
trie, _ = New(common.Hash{}, root, triedb)
err = trie.TryDelete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
@ -277,7 +277,7 @@ func TestReplication(t *testing.T) {
}
// create a new trie on top of the database and check that lookups work.
trie2, err := New(exp, trie.db)
trie2, err := New(common.Hash{}, exp, trie.db)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
@ -411,9 +411,9 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
func runRandTest(rt randTest) bool {
var (
triedb = NewDatabase(memorydb.New())
tr, _ = New(common.Hash{}, triedb)
tr = NewEmpty(triedb)
values = make(map[string]string) // tracks content of the trie
origTrie, _ = New(common.Hash{}, triedb)
origTrie = NewEmpty(triedb)
)
tr.tracer = newTracer()
@ -445,7 +445,7 @@ func runRandTest(rt randTest) bool {
rt[i].err = err
return false
}
newtr, err := New(hash, triedb)
newtr, err := New(common.Hash{}, hash, triedb)
if err != nil {
rt[i].err = err
return false
@ -455,7 +455,7 @@ func runRandTest(rt randTest) bool {
origTrie = tr.Copy()
case opItercheckhash:
checktr, _ := New(common.Hash{}, triedb)
checktr := NewEmpty(triedb)
it := NewIterator(tr.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)
@ -542,10 +542,10 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000
func benchGet(b *testing.B, commit bool) {
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
if commit {
tmpdb := tempDB(b)
trie, _ = New(common.Hash{}, tmpdb)
trie = NewEmpty(tmpdb)
}
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
@ -664,7 +664,7 @@ func TestTinyTrie(t *testing.T) {
if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
t.Errorf("3: got %x, exp %x", root, exp)
}
checktr, _ := New(common.Hash{}, trie.db)
checktr := NewEmpty(trie.db)
it := NewIterator(trie.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)
@ -790,7 +790,7 @@ func TestCommitSequence(t *testing.T) {
// This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(s)
trie, _ := New(common.Hash{}, db)
trie := NewEmpty(db)
// Another sponge is used to check the callback-sequence
callbackSponge := sha3.NewLegacyKeccak256()
// Fill the trie with elements
@ -832,7 +832,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) {
// This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(s)
trie, _ := New(common.Hash{}, db)
trie := NewEmpty(db)
// Another sponge is used to check the callback-sequence
callbackSponge := sha3.NewLegacyKeccak256()
// Fill the trie with elements
@ -871,7 +871,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
// This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(s)
trie, _ := New(common.Hash{}, db)
trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(stackTrieSponge)
@ -927,7 +927,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
func TestCommitSequenceSmallRoot(t *testing.T) {
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(s)
trie, _ := New(common.Hash{}, db)
trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(stackTrieSponge)

7
trie/util_test.go
View File

@ -19,14 +19,12 @@ package trie
import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
)
// Tests if the trie diffs are tracked correctly.
func TestTrieTracer(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase())
trie, _ := New(common.Hash{}, db)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie.tracer = newTracer()
// Insert a batch of entries, all the nodes should be marked as inserted
@ -93,8 +91,7 @@ func TestTrieTracer(t *testing.T) {
}
func TestTrieTracerNoop(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase())
trie, _ := New(common.Hash{}, db)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie.tracer = newTracer()
// Insert a batch of entries, all the nodes should be marked as inserted

38
trie/utils.go
View File

@ -29,24 +29,39 @@ package trie
// This tool can track all of them no matter the node is embedded in its
// parent or not, but valueNode is never tracked.
//
// Besides, it's also used for recording the original value of the nodes
// when they are resolved from the disk. The pre-value of the nodes will
// be used to construct reverse-diffs in the future.
//
// Note tracer is not thread-safe, callers should be responsible for handling
// the concurrency issues by themselves.
type tracer struct {
insert map[string]struct{}
delete map[string]struct{}
origin map[string][]byte
}
// newTracer initializes trie node diff tracer.
// newTracer initializes the tracer for capturing trie changes.
func newTracer() *tracer {
return &tracer{
insert: make(map[string]struct{}),
delete: make(map[string]struct{}),
origin: make(map[string][]byte),
}
}
// onInsert tracks the newly inserted trie node. If it's already
// in the deletion set(resurrected node), then just wipe it from
// the deletion set as it's untouched.
// onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally.
// Don't change the value outside of function since it's not deep-copied.
func (t *tracer) onRead(key []byte, val []byte) {
// Tracer isn't used right now, remove this check later.
if t == nil {
return
}
t.origin[string(key)] = val
}
// onInsert tracks the newly inserted trie node. If it's already in the deletion set
// (resurrected node), then just wipe it from the deletion set as the "untouched".
func (t *tracer) onInsert(key []byte) {
// Tracer isn't used right now, remove this check later.
if t == nil {
@ -100,6 +115,15 @@ func (t *tracer) deleteList() [][]byte {
return ret
}
// getPrev returns the cached original value of the specified node.
func (t *tracer) getPrev(key []byte) []byte {
// Don't panic on uninitialized tracer, it's possible in testing.
if t == nil {
return nil
}
return t.origin[string(key)]
}
// reset clears the content tracked by tracer.
func (t *tracer) reset() {
// Tracer isn't used right now, remove this check later.
@ -108,6 +132,7 @@ func (t *tracer) reset() {
}
t.insert = make(map[string]struct{})
t.delete = make(map[string]struct{})
t.origin = make(map[string][]byte)
}
// copy returns a deep copied tracer instance.
@ -119,6 +144,7 @@ func (t *tracer) copy() *tracer {
var (
insert = make(map[string]struct{})
delete = make(map[string]struct{})
origin = make(map[string][]byte)
)
for key := range t.insert {
insert[key] = struct{}{}
@ -126,8 +152,12 @@ func (t *tracer) copy() *tracer {
for key := range t.delete {
delete[key] = struct{}{}
}
for key, val := range t.origin {
origin[key] = val
}
return &tracer{
insert: insert,
delete: delete,
origin: origin,
}
}