plugeth/core/state/snapshot/difflayer.go
2020-02-25 12:51:05 +02:00

403 lines
14 KiB
Go

// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"fmt"
"sort"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
)
// diffLayer represents a collection of modifications made to a state snapshot
// after running a block on top. It contains one sorted list for the account trie
// and one-one list for each storage tries.
//
// The goal of a diff layer is to act as a journal, tracking recent modifications
// made to the state, that have not yet graduated into a semi-immutable state.
type diffLayer struct {
parent snapshot // Parent snapshot modified by this one, never nil
memory uint64 // Approximate guess as to how much memory we use
number uint64 // Block number to which this snapshot diff belongs to
root common.Hash // Root hash to which this snapshot diff belongs to
stale bool // Signals that the layer became stale (state progressed)
accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil
accountData map[common.Hash][]byte // Keyed accounts for direct retrival (nil means deleted)
storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil
storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrival. one per account (nil means deleted)
lock sync.RWMutex
}
// newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
// level persistent database or a hierarchical diff already.
func newDiffLayer(parent snapshot, number uint64, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
// Create the new layer with some pre-allocated data segments
dl := &diffLayer{
parent: parent,
number: number,
root: root,
accountData: accounts,
storageData: storage,
}
// Determine mem size
for _, data := range accounts {
dl.memory += uint64(len(data))
}
// Fill the storage hashes and sort them for the iterator
dl.storageList = make(map[common.Hash][]common.Hash)
for accountHash, slots := range storage {
// If the slots are nil, sanity check that it's a deleted account
if slots == nil {
// Ensure that the account was just marked as deleted
if account, ok := accounts[accountHash]; account != nil || !ok {
panic(fmt.Sprintf("storage in %#x nil, but account conflicts (%#x, exists: %v)", accountHash, account, ok))
}
// Everything ok, store the deletion mark and continue
dl.storageList[accountHash] = nil
continue
}
// Storage slots are not nil so entire contract was not deleted, ensure the
// account was just updated.
if account, ok := accounts[accountHash]; account == nil || !ok {
log.Error(fmt.Sprintf("storage in %#x exists, but account nil (exists: %v)", accountHash, ok))
}
// Determine mem size
for _, data := range slots {
dl.memory += uint64(len(data))
}
}
dl.memory += uint64(len(dl.storageList) * common.HashLength)
return dl
}
// Info returns the block number and root hash for which this snapshot was made.
func (dl *diffLayer) Info() (uint64, common.Hash) {
return dl.number, dl.root
}
// Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format.
func (dl *diffLayer) Account(hash common.Hash) (*Account, error) {
data, err := dl.AccountRLP(hash)
if err != nil {
return nil, err
}
if len(data) == 0 { // can be both nil and []byte{}
return nil, nil
}
account := new(Account)
if err := rlp.DecodeBytes(data, account); err != nil {
panic(err)
}
return account, nil
}
// AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format.
func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.stale {
return nil, ErrSnapshotStale
}
// If the account is known locally, return it. Note, a nil account means it was
// deleted, and is a different notion than an unknown account!
if data, ok := dl.accountData[hash]; ok {
return data, nil
}
// Account unknown to this diff, resolve from parent
return dl.parent.AccountRLP(hash)
}
// Storage directly retrieves the storage data associated with a particular hash,
// within a particular account. If the slot is unknown to this diff, it's parent
// is consulted.
func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.stale {
return nil, ErrSnapshotStale
}
// If the account is known locally, try to resolve the slot locally. Note, a nil
// account means it was deleted, and is a different notion than an unknown account!
if storage, ok := dl.storageData[accountHash]; ok {
if storage == nil {
return nil, nil
}
if data, ok := storage[storageHash]; ok {
return data, nil
}
}
// Account - or slot within - unknown to this diff, resolve from parent
return dl.parent.Storage(accountHash, storageHash)
}
// Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items.
func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, dl.number+1, blockRoot, accounts, storage)
}
// Cap traverses downwards the diff tree until the number of allowed layers are
// crossed. All diffs beyond the permitted number are flattened downwards. If
// the layer limit is reached, memory cap is also enforced (but not before). The
// block numbers for the disk layer and first diff layer are returned for GC.
func (dl *diffLayer) Cap(layers int, memory uint64) (uint64, uint64) {
// Dive until we run out of layers or reach the persistent database
if layers > 2 {
// If we still have diff layers below, recurse
if parent, ok := dl.parent.(*diffLayer); ok {
return parent.Cap(layers-1, memory)
}
// Diff stack too shallow, return block numbers without modifications
return dl.parent.(*diskLayer).number, dl.number
}
// We're out of layers, flatten anything below, stopping if it's the disk or if
// the memory limit is not yet exceeded.
switch parent := dl.parent.(type) {
case *diskLayer:
return parent.number, dl.number
case *diffLayer:
// Flatten the parent into the grandparent. The flattening internally obtains a
// write lock on grandparent.
flattened := parent.flatten().(*diffLayer)
dl.lock.Lock()
defer dl.lock.Unlock()
dl.parent = flattened
if flattened.memory < memory {
diskNumber, _ := flattened.parent.Info()
return diskNumber, flattened.number
}
default:
panic(fmt.Sprintf("unknown data layer: %T", parent))
}
// If the bottommost layer is larger than our memory cap, persist to disk
var (
parent = dl.parent.(*diffLayer)
base = parent.parent.(*diskLayer)
batch = base.db.NewBatch()
)
parent.lock.RLock()
defer parent.lock.RUnlock()
// Start by temporarily deleting the current snapshot block marker. This
// ensures that in the case of a crash, the entire snapshot is invalidated.
rawdb.DeleteSnapshotBlock(batch)
// Mark the original base as stale as we're going to create a new wrapper
base.lock.Lock()
if base.stale {
panic("parent disk layer is stale") // we've committed into the same base from two children, boo
}
base.stale = true
base.lock.Unlock()
// Push all the accounts into the database
for hash, data := range parent.accountData {
if len(data) > 0 {
// Account was updated, push to disk
rawdb.WriteAccountSnapshot(batch, hash, data)
base.cache.Set(string(hash[:]), data)
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
log.Crit("Failed to write account snapshot", "err", err)
}
batch.Reset()
}
} else {
// Account was deleted, remove all storage slots too
rawdb.DeleteAccountSnapshot(batch, hash)
base.cache.Set(string(hash[:]), nil)
it := rawdb.IterateStorageSnapshots(base.db, hash)
for it.Next() {
if key := it.Key(); len(key) == 65 { // TODO(karalabe): Yuck, we should move this into the iterator
batch.Delete(key)
base.cache.Delete(string(key[1:]))
}
}
it.Release()
}
}
// Push all the storage slots into the database
for accountHash, storage := range parent.storageData {
for storageHash, data := range storage {
if len(data) > 0 {
rawdb.WriteStorageSnapshot(batch, accountHash, storageHash, data)
base.cache.Set(string(append(accountHash[:], storageHash[:]...)), data)
} else {
rawdb.DeleteStorageSnapshot(batch, accountHash, storageHash)
base.cache.Set(string(append(accountHash[:], storageHash[:]...)), nil)
}
}
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
log.Crit("Failed to write storage snapshot", "err", err)
}
batch.Reset()
}
}
// Update the snapshot block marker and write any remainder data
newBase := &diskLayer{
root: parent.root,
number: parent.number,
cache: base.cache,
db: base.db,
journal: base.journal,
}
rawdb.WriteSnapshotBlock(batch, newBase.number, newBase.root)
if err := batch.Write(); err != nil {
log.Crit("Failed to write leftover snapshot", "err", err)
}
dl.parent = newBase
return newBase.number, dl.number
}
// flatten pushes all data from this point downwards, flattening everything into
// a single diff at the bottom. Since usually the lowermost diff is the largest,
// the flattening bulds up from there in reverse.
func (dl *diffLayer) flatten() snapshot {
// If the parent is not diff, we're the first in line, return unmodified
parent, ok := dl.parent.(*diffLayer)
if !ok {
return dl
}
// Parent is a diff, flatten it first (note, apart from weird corned cases,
// flatten will realistically only ever merge 1 layer, so there's no need to
// be smarter about grouping flattens together).
parent = parent.flatten().(*diffLayer)
parent.lock.Lock()
defer parent.lock.Unlock()
// Before actually writing all our data to the parent, first ensure that the
// parent hasn't been 'corrupted' by someone else already flattening into it
if parent.stale {
panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo
}
parent.stale = true
// Overwrite all the updated accounts blindly, merge the sorted list
for hash, data := range dl.accountData {
parent.accountData[hash] = data
}
// Overwrite all the updates storage slots (individually)
for accountHash, storage := range dl.storageData {
// If storage didn't exist (or was deleted) in the parent; or if the storage
// was freshly deleted in the child, overwrite blindly
if parent.storageData[accountHash] == nil || storage == nil {
parent.storageData[accountHash] = storage
continue
}
// Storage exists in both parent and child, merge the slots
comboData := parent.storageData[accountHash]
for storageHash, data := range storage {
comboData[storageHash] = data
}
parent.storageData[accountHash] = comboData
}
// Return the combo parent
return &diffLayer{
parent: parent.parent,
number: dl.number,
root: dl.root,
storageList: parent.storageList,
storageData: parent.storageData,
accountList: parent.accountList,
accountData: parent.accountData,
memory: parent.memory + dl.memory,
}
}
// Journal commits an entire diff hierarchy to disk into a single journal file.
// This is meant to be used during shutdown to persist the snapshot without
// flattening everything down (bad for reorgs).
func (dl *diffLayer) Journal() error {
dl.lock.RLock()
defer dl.lock.RUnlock()
writer, err := dl.journal()
if err != nil {
return err
}
writer.Close()
return nil
}
// AccountList returns a sorted list of all accounts in this difflayer.
func (dl *diffLayer) AccountList() []common.Hash {
dl.lock.Lock()
defer dl.lock.Unlock()
if dl.accountList != nil {
return dl.accountList
}
accountList := make([]common.Hash, len(dl.accountData))
i := 0
for k, _ := range dl.accountData {
accountList[i] = k
i++
// This would be a pretty good opportunity to also
// calculate the size, if we want to
}
sort.Sort(hashes(accountList))
dl.accountList = accountList
return dl.accountList
}
// StorageList returns a sorted list of all storage slot hashes
// in this difflayer for the given account.
func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash {
dl.lock.Lock()
defer dl.lock.Unlock()
if dl.storageList[accountHash] != nil {
return dl.storageList[accountHash]
}
accountStorageMap := dl.storageData[accountHash]
accountStorageList := make([]common.Hash, len(accountStorageMap))
i := 0
for k, _ := range accountStorageMap {
accountStorageList[i] = k
i++
// This would be a pretty good opportunity to also
// calculate the size, if we want to
}
sort.Sort(hashes(accountStorageList))
dl.storageList[accountHash] = accountStorageList
return accountStorageList
}