plugeth/core/state/snapshot/generate.go

213 lines
6.9 KiB
Go
Raw Normal View History

// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"fmt"
"math/big"
"time"
"github.com/allegro/bigcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
var (
// emptyRoot is the known root hash of an empty trie.
emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
// emptyCode is the known hash of the empty EVM bytecode.
emptyCode = crypto.Keccak256Hash(nil)
)
// wipeSnapshot iterates over the entire key-value database and deletes all the
// data associated with the snapshot (accounts, storage, metadata). After all is
// done, the snapshot range of the database is compacted to free up unused data
// blocks.
func wipeSnapshot(db ethdb.KeyValueStore) error {
// Batch deletions together to avoid holding an iterator for too long
var (
batch = db.NewBatch()
items int
)
// Iterate over the snapshot key-range and delete all of them
log.Info("Deleting previous snapshot leftovers")
start, logged := time.Now(), time.Now()
it := db.NewIteratorWithStart(rawdb.StateSnapshotPrefix)
for it.Next() {
// Skip any keys with the correct prefix but wrong lenth (trie nodes)
key := it.Key()
if !bytes.HasPrefix(key, rawdb.StateSnapshotPrefix) {
break
}
if len(key) != len(rawdb.StateSnapshotPrefix)+common.HashLength && len(key) != len(rawdb.StateSnapshotPrefix)+2*common.HashLength {
continue
}
// Delete the key and periodically recreate the batch and iterator
batch.Delete(key)
items++
if items%10000 == 0 {
// Batch too large (or iterator too long lived, flush and recreate)
it.Release()
if err := batch.Write(); err != nil {
return err
}
batch.Reset()
it = db.NewIteratorWithStart(key)
if time.Since(logged) > 8*time.Second {
log.Info("Deleting previous snapshot leftovers", "wiped", items, "elapsed", time.Since(start))
logged = time.Now()
}
}
}
it.Release()
rawdb.DeleteSnapshotBlock(batch)
if err := batch.Write(); err != nil {
return err
}
log.Info("Deleted previous snapshot leftovers", "wiped", items, "elapsed", time.Since(start))
// Compact the snapshot section of the database to get rid of unused space
log.Info("Compacting snapshot area in database")
start = time.Now()
end := common.CopyBytes(rawdb.StateSnapshotPrefix)
end[len(end)-1]++
if err := db.Compact(rawdb.StateSnapshotPrefix, end); err != nil {
return err
}
log.Info("Compacted snapshot area in database", "elapsed", time.Since(start))
return nil
}
// generateSnapshot regenerates a brand new snapshot based on an existing state database and head block.
func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64, headRoot common.Hash) (snapshot, error) {
// Wipe any previously existing snapshot from the database
if err := wipeSnapshot(db); err != nil {
return nil, err
}
// Iterate the entire storage trie and re-generate the state snapshot
var (
accountCount int
storageCount int
storageNodes int
accountSize common.StorageSize
storageSize common.StorageSize
logged time.Time
)
batch := db.NewBatch()
triedb := trie.NewDatabase(db)
accTrie, err := trie.NewSecure(headRoot, triedb)
if err != nil {
return nil, err
}
accIt := trie.NewIterator(accTrie.NodeIterator(nil))
for accIt.Next() {
var (
curStorageCount int
curStorageNodes int
curAccountSize common.StorageSize
curStorageSize common.StorageSize
)
var acc struct {
Nonce uint64
Balance *big.Int
Root common.Hash
CodeHash []byte
}
if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil {
return nil, err
}
data := AccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash)
curAccountSize += common.StorageSize(1 + common.HashLength + len(data))
rawdb.WriteAccountSnapshot(batch, common.BytesToHash(accIt.Key), data)
if batch.ValueSize() > ethdb.IdealBatchSize {
batch.Write()
batch.Reset()
}
if acc.Root != emptyRoot {
storeTrie, err := trie.NewSecure(acc.Root, triedb)
if err != nil {
return nil, err
}
storeIt := trie.NewIterator(storeTrie.NodeIterator(nil))
for storeIt.Next() {
curStorageSize += common.StorageSize(1 + 2*common.HashLength + len(storeIt.Value))
curStorageCount++
rawdb.WriteStorageSnapshot(batch, common.BytesToHash(accIt.Key), common.BytesToHash(storeIt.Key), storeIt.Value)
if batch.ValueSize() > ethdb.IdealBatchSize {
batch.Write()
batch.Reset()
}
}
curStorageNodes = storeIt.Nodes
}
accountCount++
storageCount += curStorageCount
accountSize += curAccountSize
storageSize += curStorageSize
storageNodes += curStorageNodes
if time.Since(logged) > 8*time.Second {
fmt.Printf("%#x: %9s + %9s (%6d slots, %6d nodes), total %9s (%d accs, %d nodes) + %9s (%d slots, %d nodes)\n", accIt.Key, curAccountSize.TerminalString(), curStorageSize.TerminalString(), curStorageCount, curStorageNodes, accountSize.TerminalString(), accountCount, accIt.Nodes, storageSize.TerminalString(), storageCount, storageNodes)
logged = time.Now()
}
}
fmt.Printf("Totals: %9s (%d accs, %d nodes) + %9s (%d slots, %d nodes)\n", accountSize.TerminalString(), accountCount, accIt.Nodes, storageSize.TerminalString(), storageCount, storageNodes)
// Update the snapshot block marker and write any remainder data
rawdb.WriteSnapshotBlock(batch, headNumber, headRoot)
batch.Write()
batch.Reset()
// Compact the snapshot section of the database to get rid of unused space
log.Info("Compacting snapshot in chain database")
if err := db.Compact([]byte{'s'}, []byte{'s' + 1}); err != nil {
return nil, err
}
// New snapshot generated, construct a brand new base layer
cache, _ := bigcache.NewBigCache(bigcache.Config{ // TODO(karalabe): dedup
Shards: 1024,
LifeWindow: time.Hour,
MaxEntriesInWindow: 512 * 1024,
MaxEntrySize: 512,
HardMaxCacheSize: 512,
})
return &diskLayer{
journal: journal,
db: db,
cache: cache,
number: headNumber,
root: headRoot,
}, nil
}