roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

cmd, core, eth, les, light: track deleted nodes (#25757)

Browse Source 
* cmd, core, eth, les, light: track deleted nodes

* trie: add docs

* trie: address comments

* cmd, core, eth, les, light, trie: trie id

* trie: add tests

* trie, core: updates

* trie: fix imports

* trie: add utility print-method for nodeset

* trie: import err

* trie: fix go vet warnings

Co-authored-by: Martin Holst Swende <martin@swende.se>
This commit is contained in:
rjl493456442
2022-09-27 10:01:02 +02:00
committed by GitHub
co-authored by Martin Holst Swende
parent fc3e6d0162
commit bff84a99fe
41 changed files with 866 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/blockchain.go
+8 -5
View File
@@ -70,6 +70,8 @@ var (
snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil)
snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil)
triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil)
blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil)
@@ -737,10 +739,10 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x..]", hash[:4])
}
if _, err := trie.NewStateTrie(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil {
return err
root := block.Root()
if !bc.HasState(root) {
return fmt.Errorf("non existent state [%x..]", root[:4])
}
// If all checks out, manually set the head block.
if !bc.chainmu.TryLock() {
return errChainStopped
@@ -752,7 +754,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
// Destroy any existing state snapshot and regenerate it in the background,
// also resuming the normal maintenance of any previously paused snapshot.
if bc.snaps != nil {
bc.snaps.Rebuild(block.Root())
bc.snaps.Rebuild(root)
}
log.Info("Committed new head block", "number", block.Number(), "hash", hash)
return nil
@@ -1750,8 +1752,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool)
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them
triedbCommitTimer.Update(statedb.TrieDBCommits) // Triedb commits are complete, we can mark them
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits)
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits - statedb.TrieDBCommits)
blockInsertTimer.UpdateSince(start)
// Report the import stats before returning the various results
core/state/database.go
+4 -4
View File
@@ -43,7 +43,7 @@ type Database interface {
OpenTrie(root common.Hash) (Trie, error)
// OpenStorageTrie opens the storage trie of an account.
OpenStorageTrie(addrHash, root common.Hash) (Trie, error)
OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error)
// CopyTrie returns an independent copy of the given trie.
CopyTrie(Trie) Trie
@@ -148,7 +148,7 @@ type cachingDB struct {
// OpenTrie opens the main account trie at a specific root hash.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
tr, err := trie.NewStateTrie(common.Hash{}, root, db.db)
tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.db)
if err != nil {
return nil, err
}
@@ -156,8 +156,8 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
}
// OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
tr, err := trie.NewStateTrie(addrHash, root, db.db)
func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) {
tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, addrHash, root), db.db)
if err != nil {
return nil, err
}
core/state/iterator.go
+1 -1
View File
@@ -109,7 +109,7 @@ func (it *NodeIterator) step() error {
if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil {
return err
}
dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root)
dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root)
if err != nil {
return err
}
core/state/metrics.go
+8 -6
View File
@@ -19,10 +19,12 @@ package state
import "github.com/ethereum/go-ethereum/metrics"
var (
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountTrieUpdatedMeter = metrics.NewRegisteredMeter("state/update/accountnodes", nil)
storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil)
accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil)
storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil)
)
core/state/pruner/pruner.go
+4 -3
View File
@@ -93,7 +93,7 @@ type Pruner struct {
func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
headBlock := rawdb.ReadHeadBlock(db)
if headBlock == nil {
return nil, errors.New("Failed to load head block")
return nil, errors.New("failed to load head block")
}
snapconfig := snapshot.Config{
CacheSize: 256,
@@ -427,7 +427,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil {
return errors.New("missing genesis block")
}
t, err := trie.NewStateTrie(common.Hash{}, genesis.Root(), trie.NewDatabase(db))
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
if err != nil {
return err
}
@@ -447,7 +447,8 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
return err
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db))
id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
if err != nil {
return err
}
core/state/snapshot/generate.go
+15 -12
View File
@@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error)
//
// The proof result will be returned if the range proving is finished, otherwise
// the error will be returned to abort the entire procedure.
func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
var (
keys [][]byte
vals [][]byte
@@ -233,8 +233,9 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root c
}(time.Now())
// The snap state is exhausted, pass the entire key/val set for verification
root := trieId.Root
if origin == nil && !diskMore {
stackTr := trie.NewStackTrieWithOwner(nil, owner)
stackTr := trie.NewStackTrie(nil)
for i, key := range keys {
stackTr.TryUpdate(key, vals[i])
}
@@ -248,7 +249,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root c
return &proofResult{keys: keys, vals: vals}, nil
}
// Snap state is chunked, generate edge proofs for verification.
tr, err := trie.New(owner, root, dl.triedb)
tr, err := trie.New(trieId, dl.triedb)
if err != nil {
ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return nil, errMissingTrie
@@ -313,9 +314,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
// generateRange generates the state segment with particular prefix. Generation can
// either verify the correctness of existing state through range-proof and skip
// generation, or iterate trie to regenerate state on demand.
func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
// Use range prover to check the validity of the flat state in the range
result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn)
result, err := dl.proveRange(ctx, trieId, prefix, kind, origin, max, valueConvertFn)
if err != nil {
return false, nil, err
}
@@ -363,7 +364,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo
if len(result.keys) > 0 {
snapNodeCache = memorydb.New()
snapTrieDb := trie.NewDatabase(snapNodeCache)
snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb)
snapTrie := trie.NewEmpty(snapTrieDb)
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])
}
@@ -377,7 +378,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo
// if it's already opened with some nodes resolved.
tr := result.tr
if tr == nil {
tr, err = trie.New(owner, root, dl.triedb)
tr, err = trie.New(trieId, dl.triedb)
if err != nil {
ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return false, nil, errMissingTrie
@@ -460,7 +461,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo
} else {
snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
}
logger.Debug("Regenerated state range", "root", root, "last", hexutil.Encode(last),
logger.Debug("Regenerated state range", "root", trieId.Root, "last", hexutil.Encode(last),
"count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted)
// If there are either more trie items, or there are more snap items
@@ -511,7 +512,7 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error
// generateStorages generates the missing storage slots of the specific contract.
// It's supposed to restart the generation from the given origin position.
func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, storageRoot common.Hash, storeMarker []byte) error {
func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Hash, account common.Hash, storageRoot common.Hash, storeMarker []byte) error {
onStorage := func(key []byte, val []byte, write bool, delete bool) error {
defer func(start time.Time) {
snapStorageWriteCounter.Inc(time.Since(start).Nanoseconds())
@@ -540,7 +541,8 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash,
// Loop for re-generating the missing storage slots.
var origin = common.CopyBytes(storeMarker)
for {
exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
id := trie.StorageTrieID(stateRoot, account, storageRoot)
exhausted, last, err := dl.generateRange(ctx, id, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
@@ -624,7 +626,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
if accMarker != nil && bytes.Equal(account[:], accMarker) && len(dl.genMarker) > common.HashLength {
storeMarker = dl.genMarker[common.HashLength:]
}
if err := generateStorages(ctx, dl, account, acc.Root, storeMarker); err != nil {
if err := generateStorages(ctx, dl, dl.root, account, acc.Root, storeMarker); err != nil {
return err
}
}
@@ -640,7 +642,8 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
}
origin := common.CopyBytes(accMarker)
for {
exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
id := trie.StateTrieID(dl.root)
exhausted, last, err := dl.generateRange(ctx, id, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
core/state/snapshot/generate_test.go
+3 -2
View File
@@ -149,7 +149,7 @@ type testHelper struct {
func newHelper() *testHelper {
diskdb := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(diskdb)
accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, triedb)
accTrie, _ := trie.NewStateTrie(trie.StateTrieID(common.Hash{}), triedb)
return &testHelper{
diskdb: diskdb,
triedb: triedb,
@@ -182,7 +182,8 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string)
}
func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte {
stTrie, _ := trie.NewStateTrie(owner, common.Hash{}, t.triedb)
id := trie.StorageTrieID(stateRoot, owner, common.Hash{})
stTrie, _ := trie.NewStateTrie(id, t.triedb)
for i, k := range keys {
stTrie.Update([]byte(k), []byte(vals[i]))
}
core/state/state_object.go
+2 -2
View File
@@ -159,9 +159,9 @@ func (s *stateObject) getTrie(db Database) Trie {
}
if s.trie == nil {
var err error
s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root)
s.trie, err = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, s.data.Root)
if err != nil {
s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{})
s.trie, _ = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, common.Hash{})
s.setError(fmt.Errorf("can't create storage trie: %v", err))
}
}
core/state/statedb.go
+35 -14
View File
@@ -120,6 +120,7 @@ type StateDB struct {
SnapshotAccountReads time.Duration
SnapshotStorageReads time.Duration
SnapshotCommits time.Duration
TrieDBCommits time.Duration
AccountUpdated int
StorageUpdated int
@@ -904,9 +905,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
// Commit objects to the trie, measuring the elapsed time
var (
accountTrieNodes int
storageTrieNodes int
nodes = trie.NewMergedNodeSet()
accountTrieNodesUpdated int
accountTrieNodesDeleted int
storageTrieNodesUpdated int
storageTrieNodesDeleted int
nodes = trie.NewMergedNodeSet()
)
codeWriter := s.db.DiskDB().NewBatch()
for addr := range s.stateObjectsDirty {
@@ -926,7 +929,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
if err := nodes.Merge(set); err != nil {
return common.Hash{}, err
}
storageTrieNodes += set.Len()
updates, deleted := set.Size()
storageTrieNodesUpdated += updates
storageTrieNodesDeleted += deleted
}
}
}
@@ -952,7 +957,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
if err := nodes.Merge(set); err != nil {
return common.Hash{}, err
}
accountTrieNodes = set.Len()
accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size()
}
if metrics.EnabledExpensive {
s.AccountCommits += time.Since(start)
@@ -961,16 +966,16 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
storageUpdatedMeter.Mark(int64(s.StorageUpdated))
accountDeletedMeter.Mark(int64(s.AccountDeleted))
storageDeletedMeter.Mark(int64(s.StorageDeleted))
accountTrieCommittedMeter.Mark(int64(accountTrieNodes))
storageTriesCommittedMeter.Mark(int64(storageTrieNodes))
accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated))
accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted))
storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated))
storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted))
s.AccountUpdated, s.AccountDeleted = 0, 0
s.StorageUpdated, s.StorageDeleted = 0, 0
}
// If snapshotting is enabled, update the snapshot tree with this new version
if s.snap != nil {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now())
}
start := time.Now()
// Only update if there's a state transition (skip empty Clique blocks)
if parent := s.snap.Root(); parent != root {
if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil {
@@ -984,13 +989,29 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err)
}
}
if metrics.EnabledExpensive {
s.SnapshotCommits += time.Since(start)
}
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
}
if err := s.db.TrieDB().Update(nodes); err != nil {
return common.Hash{}, err
if root == (common.Hash{}) {
root = emptyRoot
}
s.originalRoot = root
return root, err
origin := s.originalRoot
if origin == (common.Hash{}) {
origin = emptyRoot
}
if root != origin {
start := time.Now()
if err := s.db.TrieDB().Update(nodes); err != nil {
return common.Hash{}, err
}
s.originalRoot = root
if metrics.EnabledExpensive {
s.TrieDBCommits += time.Since(start)
}
}
return root, nil
}
// PrepareAccessList handles the preparatory steps for executing a state transition with
core/state/sync_test.go
+4 -3
View File
@@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.KeyValueStore, root common.Hash) error {
if v, _ := db.Get(root[:]); v == nil {
return nil // Consider a non existent state consistent.
}
trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db))
trie, err := trie.New(trie.StateTrieID(root), trie.NewDatabase(db))
if err != nil {
return err
}
@@ -174,7 +174,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
if commit {
srcDb.TrieDB().Commit(srcRoot, false, nil)
}
srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB())
srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB())
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
@@ -222,7 +222,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil {
t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err)
}
stTrie, err := trie.New(common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB())
id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root)
stTrie, err := trie.New(id, srcDb.TrieDB())
if err != nil {
t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err)
}
core/state/trie_prefetcher.go
+5 -3
View File
@@ -150,7 +150,7 @@ func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]
id := p.trieID(owner, root)
fetcher := p.fetchers[id]
if fetcher == nil {
fetcher = newSubfetcher(p.db, owner, root)
fetcher = newSubfetcher(p.db, p.root, owner, root)
p.fetchers[id] = fetcher
}
fetcher.schedule(keys)
@@ -206,6 +206,7 @@ func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string {
// the trie being worked on is retrieved from the prefetcher.
type subfetcher struct {
db Database // Database to load trie nodes through
state common.Hash // Root hash of the state to prefetch
owner common.Hash // Owner of the trie, usually account hash
root common.Hash // Root hash of the trie to prefetch
trie Trie // Trie being populated with nodes
@@ -225,9 +226,10 @@ type subfetcher struct {
// newSubfetcher creates a goroutine to prefetch state items belonging to a
// particular root hash.
func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher {
func newSubfetcher(db Database, state common.Hash, owner common.Hash, root common.Hash) *subfetcher {
sf := &subfetcher{
db: db,
state: state,
owner: owner,
root: root,
wake: make(chan struct{}, 1),
@@ -298,7 +300,7 @@ func (sf *subfetcher) loop() {
}
sf.trie = trie
} else {
trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root)
trie, err := sf.db.OpenStorageTrie(sf.state, sf.owner, sf.root)
if err != nil {
log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err)
return