Patch for concurrent iterator & others (onto v1.11.6) #386

Closed
roysc wants to merge 1565 commits from v1.11.6-statediff-v5 into master
3 changed files with 77 additions and 51 deletions
Showing only changes of commit 711afbc7fd - Show all commits

View File

@ -173,6 +173,8 @@ type BlockChain struct {
snaps *snapshot.Tree // Snapshot tree for fast trie leaf access snaps *snapshot.Tree // Snapshot tree for fast trie leaf access
triegc *prque.Prque // Priority queue mapping block numbers to tries to gc triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping gcproc time.Duration // Accumulates canonical block processing for trie dumping
lastWrite uint64 // Last block when the state was flushed
flushInterval int64 // Time interval (processing time) after which to flush a state
triedb *trie.Database // The database handler for maintaining trie nodes. triedb *trie.Database // The database handler for maintaining trie nodes.
stateCache state.Database // State database to reuse between imports (contains state cache) stateCache state.Database // State database to reuse between imports (contains state cache)
@ -258,6 +260,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
cacheConfig: cacheConfig, cacheConfig: cacheConfig,
db: db, db: db,
triedb: triedb, triedb: triedb,
flushInterval: int64(cacheConfig.TrieTimeLimit),
triegc: prque.New(nil), triegc: prque.New(nil),
quit: make(chan struct{}), quit: make(chan struct{}),
chainmu: syncx.NewClosableMutex(), chainmu: syncx.NewClosableMutex(),
@ -1248,8 +1251,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
return 0, nil return 0, nil
} }
var lastWrite uint64
// writeBlockWithoutState writes only the block and its metadata to the database, // writeBlockWithoutState writes only the block and its metadata to the database,
// but does not write any state. This is used to construct competing side forks // but does not write any state. This is used to construct competing side forks
// up to the point where they exceed the canonical total difficulty. // up to the point where they exceed the canonical total difficulty.
@ -1311,12 +1312,16 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
// If we're running an archive node, always flush // If we're running an archive node, always flush
if bc.cacheConfig.TrieDirtyDisabled { if bc.cacheConfig.TrieDirtyDisabled {
return bc.triedb.Commit(root, false, nil) return bc.triedb.Commit(root, false, nil)
} else { }
// Full but not archive node, do proper garbage collection // Full but not archive node, do proper garbage collection
bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -int64(block.NumberU64())) bc.triegc.Push(root, -int64(block.NumberU64()))
if current := block.NumberU64(); current > TriesInMemory { current := block.NumberU64()
// Flush limits are not considered for the first TriesInMemory blocks.
if current <= TriesInMemory {
return nil
}
// If we exceeded our memory allowance, flush matured singleton nodes to disk // If we exceeded our memory allowance, flush matured singleton nodes to disk
var ( var (
nodes, imgs = bc.triedb.Size() nodes, imgs = bc.triedb.Size()
@ -1327,9 +1332,9 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
} }
// Find the next state trie we need to commit // Find the next state trie we need to commit
chosen := current - TriesInMemory chosen := current - TriesInMemory
flushInterval := time.Duration(atomic.LoadInt64(&bc.flushInterval))
// If we exceeded out time allowance, flush an entire trie to disk // If we exceeded time allowance, flush an entire trie to disk
if bc.gcproc > bc.cacheConfig.TrieTimeLimit { if bc.gcproc > flushInterval {
// If the header is missing (canonical chain behind), we're reorging a low // If the header is missing (canonical chain behind), we're reorging a low
// diff sidechain. Suspend committing until this operation is completed. // diff sidechain. Suspend committing until this operation is completed.
header := bc.GetHeaderByNumber(chosen) header := bc.GetHeaderByNumber(chosen)
@ -1338,12 +1343,12 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
} else { } else {
// If we're exceeding limits but haven't reached a large enough memory gap, // If we're exceeding limits but haven't reached a large enough memory gap,
// warn the user that the system is becoming unstable. // warn the user that the system is becoming unstable.
if chosen < lastWrite+TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { if chosen < bc.lastWrite+TriesInMemory && bc.gcproc >= 2*flushInterval {
log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/TriesInMemory) log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval, "optimum", float64(chosen-bc.lastWrite)/TriesInMemory)
} }
// Flush an entire trie and restart the counters // Flush an entire trie and restart the counters
bc.triedb.Commit(header.Root, true, nil) bc.triedb.Commit(header.Root, true, nil)
lastWrite = chosen bc.lastWrite = chosen
bc.gcproc = 0 bc.gcproc = 0
} }
} }
@ -1356,8 +1361,6 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
} }
bc.triedb.Dereference(root.(common.Hash)) bc.triedb.Dereference(root.(common.Hash))
} }
}
}
return nil return nil
} }
@ -2436,3 +2439,10 @@ func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Pro
bc.validator = v bc.validator = v
bc.processor = p bc.processor = p
} }
// SetTrieFlushInterval configures how often in-memory tries are persisted to disk.
// The interval is in terms of block processing time, not wall clock.
// It is thread-safe and can be called repeatedly without side effects.
func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) {
atomic.StoreInt64(&bc.flushInterval, int64(interval))
}

View File

@ -590,3 +590,14 @@ func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error
} }
return 0, errors.New("no state found") return 0, errors.New("no state found")
} }
// SetTrieFlushInterval configures how often in-memory tries are persisted
// to disk. The value is in terms of block processing time, not wall clock.
func (api *DebugAPI) SetTrieFlushInterval(interval string) error {
t, err := time.ParseDuration(interval)
if err != nil {
return err
}
api.eth.blockchain.SetTrieFlushInterval(t)
return nil
}

View File

@ -490,6 +490,11 @@ web3._extend({
call: 'debug_dbAncients', call: 'debug_dbAncients',
params: 0 params: 0
}), }),
new web3._extend.Method({
name: 'setTrieFlushInterval',
call: 'debug_setTrieFlushInterval',
params: 1
}),
], ],
properties: [] properties: []
}); });