forked from cerc-io/plugeth
core/rawdb: refactor db inspector for extending multiple ancient store (#25896)
This PR ports a few changes from PBSS: - Fix the snapshot generator waiter in case the generation is not even initialized - Refactor db inspector for ancient store
This commit is contained in:
parent
a1fc0d8144
commit
60e30a940b
@ -123,7 +123,7 @@ const (
|
||||
BlockChainVersion uint64 = 8
|
||||
)
|
||||
|
||||
// CacheConfig contains the configuration values for the trie caching/pruning
|
||||
// CacheConfig contains the configuration values for the trie database
|
||||
// that's resident in a blockchain.
|
||||
type CacheConfig struct {
|
||||
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
|
||||
@ -1408,7 +1408,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
|
||||
if len(logs) > 0 {
|
||||
bc.logsFeed.Send(logs)
|
||||
}
|
||||
// In theory we should fire a ChainHeadEvent when we inject
|
||||
// In theory, we should fire a ChainHeadEvent when we inject
|
||||
// a canonical block, but sometimes we can insert a batch of
|
||||
// canonical blocks. Avoid firing too many ChainHeadEvents,
|
||||
// we will fire an accumulated ChainHeadEvent and disable fire
|
||||
|
@ -4007,3 +4007,89 @@ func TestTxIndexer(t *testing.T) {
|
||||
os.RemoveAll(frdir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateThenDeletePreByzantium(t *testing.T) {
|
||||
// We use Ropsten chain config instead of Testchain config, this is
|
||||
// deliberate: we want to use pre-byz rules where we have intermediate state roots
|
||||
// between transactions.
|
||||
testCreateThenDelete(t, params.RopstenChainConfig)
|
||||
}
|
||||
func TestCreateThenDeletePostByzantium(t *testing.T) {
|
||||
testCreateThenDelete(t, params.TestChainConfig)
|
||||
}
|
||||
|
||||
// testCreateThenDelete tests a creation and subsequent deletion of a contract, happening
|
||||
// within the same block.
|
||||
func testCreateThenDelete(t *testing.T, config *params.ChainConfig) {
|
||||
var (
|
||||
engine = ethash.NewFaker()
|
||||
// A sender who makes transactions, has some funds
|
||||
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
|
||||
address = crypto.PubkeyToAddress(key.PublicKey)
|
||||
destAddress = crypto.CreateAddress(address, 0)
|
||||
funds = big.NewInt(1000000000000000)
|
||||
)
|
||||
|
||||
// runtime code is 0x60ffff : PUSH1 0xFF SELFDESTRUCT, a.k.a SELFDESTRUCT(0xFF)
|
||||
code := append([]byte{0x60, 0xff, 0xff}, make([]byte, 32-3)...)
|
||||
initCode := []byte{
|
||||
// SSTORE 1:1
|
||||
byte(vm.PUSH1), 0x1,
|
||||
byte(vm.PUSH1), 0x1,
|
||||
byte(vm.SSTORE),
|
||||
// Get the runtime-code on the stack
|
||||
byte(vm.PUSH32)}
|
||||
initCode = append(initCode, code...)
|
||||
initCode = append(initCode, []byte{
|
||||
byte(vm.PUSH1), 0x0, // offset
|
||||
byte(vm.MSTORE),
|
||||
byte(vm.PUSH1), 0x3, // size
|
||||
byte(vm.PUSH1), 0x0, // offset
|
||||
byte(vm.RETURN), // return 3 bytes of zero-code
|
||||
}...)
|
||||
gspec := &Genesis{
|
||||
Config: config,
|
||||
Alloc: GenesisAlloc{
|
||||
address: {Balance: funds},
|
||||
},
|
||||
}
|
||||
nonce := uint64(0)
|
||||
signer := types.HomesteadSigner{}
|
||||
_, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2, func(i int, b *BlockGen) {
|
||||
fee := big.NewInt(1)
|
||||
if b.header.BaseFee != nil {
|
||||
fee = b.header.BaseFee
|
||||
}
|
||||
b.SetCoinbase(common.Address{1})
|
||||
tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{
|
||||
Nonce: nonce,
|
||||
GasPrice: new(big.Int).Set(fee),
|
||||
Gas: 100000,
|
||||
Data: initCode,
|
||||
})
|
||||
nonce++
|
||||
b.AddTx(tx)
|
||||
tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{
|
||||
Nonce: nonce,
|
||||
GasPrice: new(big.Int).Set(fee),
|
||||
Gas: 100000,
|
||||
To: &destAddress,
|
||||
})
|
||||
b.AddTx(tx)
|
||||
nonce++
|
||||
})
|
||||
// Import the canonical chain
|
||||
chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{
|
||||
//Debug: true,
|
||||
//Tracer: logger.NewJSONLogger(nil, os.Stdout),
|
||||
}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create tester chain: %v", err)
|
||||
}
|
||||
// Import the blocks
|
||||
for _, block := range blocks {
|
||||
if _, err := chain.InsertChain([]*types.Block{block}); err != nil {
|
||||
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -16,8 +16,6 @@
|
||||
|
||||
package rawdb
|
||||
|
||||
import "fmt"
|
||||
|
||||
// The list of table names of chain freezer.
|
||||
const (
|
||||
// chainFreezerHeaderTable indicates the name of the freezer header table.
|
||||
@ -53,34 +51,3 @@ var (
|
||||
|
||||
// freezers the collections of all builtin freezers.
|
||||
var freezers = []string{chainFreezerName}
|
||||
|
||||
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
|
||||
// ancient indicates the path of root ancient directory where the chain freezer can
|
||||
// be opened. Start and end specify the range for dumping out indexes.
|
||||
// Note this function can only be used for debugging purposes.
|
||||
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
|
||||
var (
|
||||
path string
|
||||
tables map[string]bool
|
||||
)
|
||||
switch freezerName {
|
||||
case chainFreezerName:
|
||||
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
|
||||
default:
|
||||
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
||||
}
|
||||
noSnappy, exist := tables[tableName]
|
||||
if !exist {
|
||||
var names []string
|
||||
for name := range tables {
|
||||
names = append(names, name)
|
||||
}
|
||||
return fmt.Errorf("unknown table, supported ones: %v", names)
|
||||
}
|
||||
table, err := newFreezerTable(path, tableName, noSnappy, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
table.dumpIndexStdout(start, end)
|
||||
return nil
|
||||
}
|
||||
|
121
core/rawdb/ancient_utils.go
Normal file
121
core/rawdb/ancient_utils.go
Normal file
@ -0,0 +1,121 @@
|
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
)
|
||||
|
||||
type tableSize struct {
|
||||
name string
|
||||
size common.StorageSize
|
||||
}
|
||||
|
||||
// freezerInfo contains the basic information of the freezer.
|
||||
type freezerInfo struct {
|
||||
name string // The identifier of freezer
|
||||
head uint64 // The number of last stored item in the freezer
|
||||
tail uint64 // The number of first stored item in the freezer
|
||||
sizes []tableSize // The storage size per table
|
||||
}
|
||||
|
||||
// count returns the number of stored items in the freezer.
|
||||
func (info *freezerInfo) count() uint64 {
|
||||
return info.head - info.tail + 1
|
||||
}
|
||||
|
||||
// size returns the storage size of the entire freezer.
|
||||
func (info *freezerInfo) size() common.StorageSize {
|
||||
var total common.StorageSize
|
||||
for _, table := range info.sizes {
|
||||
total += table.size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// inspectFreezers inspects all freezers registered in the system.
|
||||
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
|
||||
var infos []freezerInfo
|
||||
for _, freezer := range freezers {
|
||||
switch freezer {
|
||||
case chainFreezerName:
|
||||
// Chain ancient store is a bit special. It's always opened along
|
||||
// with the key-value store, inspect the chain store directly.
|
||||
info := freezerInfo{name: freezer}
|
||||
// Retrieve storage size of every contained table.
|
||||
for table := range chainFreezerNoSnappy {
|
||||
size, err := db.AncientSize(table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
|
||||
}
|
||||
// Retrieve the number of last stored item
|
||||
ancients, err := db.Ancients()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info.head = ancients - 1
|
||||
|
||||
// Retrieve the number of first stored item
|
||||
tail, err := db.Tail()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info.tail = tail
|
||||
infos = append(infos, info)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
||||
}
|
||||
}
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
|
||||
// ancient indicates the path of root ancient directory where the chain freezer can
|
||||
// be opened. Start and end specify the range for dumping out indexes.
|
||||
// Note this function can only be used for debugging purposes.
|
||||
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
|
||||
var (
|
||||
path string
|
||||
tables map[string]bool
|
||||
)
|
||||
switch freezerName {
|
||||
case chainFreezerName:
|
||||
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
|
||||
default:
|
||||
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
||||
}
|
||||
noSnappy, exist := tables[tableName]
|
||||
if !exist {
|
||||
var names []string
|
||||
for name := range tables {
|
||||
names = append(names, name)
|
||||
}
|
||||
return fmt.Errorf("unknown table, supported ones: %v", names)
|
||||
}
|
||||
table, err := newFreezerTable(path, tableName, noSnappy, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
table.dumpIndexStdout(start, end)
|
||||
return nil
|
||||
}
|
@ -22,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@ -379,13 +380,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
||||
beaconHeaders stat
|
||||
cliqueSnaps stat
|
||||
|
||||
// Ancient store statistics
|
||||
ancientHeadersSize common.StorageSize
|
||||
ancientBodiesSize common.StorageSize
|
||||
ancientReceiptsSize common.StorageSize
|
||||
ancientTdsSize common.StorageSize
|
||||
ancientHashesSize common.StorageSize
|
||||
|
||||
// Les statistic
|
||||
chtTrieNodes stat
|
||||
bloomTrieNodes stat
|
||||
@ -473,20 +467,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
// Inspect append-only file store then.
|
||||
ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
|
||||
for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
|
||||
if size, err := db.AncientSize(category); err == nil {
|
||||
*ancientSizes[i] += common.StorageSize(size)
|
||||
total += common.StorageSize(size)
|
||||
}
|
||||
}
|
||||
// Get number of ancient rows inside the freezer
|
||||
ancients := counter(0)
|
||||
if count, err := db.Ancients(); err == nil {
|
||||
ancients = counter(count)
|
||||
}
|
||||
// Display the database statistic.
|
||||
// Display the database statistic of key-value store.
|
||||
stats := [][]string{
|
||||
{"Key-Value store", "Headers", headers.Size(), headers.Count()},
|
||||
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
|
||||
@ -504,14 +485,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
||||
{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()},
|
||||
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
|
||||
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
|
||||
{"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
|
||||
{"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
|
||||
{"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
|
||||
{"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
|
||||
{"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
|
||||
{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
|
||||
{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
|
||||
}
|
||||
// Inspect all registered append-only file store then.
|
||||
ancients, err := inspectFreezers(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ancient := range ancients {
|
||||
for _, table := range ancient.sizes {
|
||||
stats = append(stats, []string{
|
||||
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
|
||||
strings.Title(table.name),
|
||||
table.size.String(),
|
||||
fmt.Sprintf("%d", ancient.count()),
|
||||
})
|
||||
}
|
||||
total += ancient.size()
|
||||
}
|
||||
table := tablewriter.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"Database", "Category", "Size", "Items"})
|
||||
table.SetFooter([]string{"", "Total", total.String(), " "})
|
||||
@ -521,6 +513,5 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
||||
if unaccounted.size > 0 {
|
||||
log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -187,8 +187,9 @@ type Tree struct {
|
||||
// If the memory layers in the journal do not match the disk layer (e.g. there is
|
||||
// a gap) or the journal is missing, there are two repair cases:
|
||||
//
|
||||
// - if the 'recovery' parameter is true, all memory diff-layers will be discarded.
|
||||
// This case happens when the snapshot is 'ahead' of the state trie.
|
||||
// - if the 'recovery' parameter is true, memory diff-layers and the disk-layer
|
||||
// will all be kept. This case happens when the snapshot is 'ahead' of the
|
||||
// state trie.
|
||||
// - otherwise, the entire snapshot is considered invalid and will be recreated on
|
||||
// a background thread.
|
||||
func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) {
|
||||
@ -199,16 +200,16 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root
|
||||
triedb: triedb,
|
||||
layers: make(map[common.Hash]snapshot),
|
||||
}
|
||||
// Create the building waiter iff the background generation is allowed
|
||||
if !config.NoBuild && !config.AsyncBuild {
|
||||
defer snap.waitBuild()
|
||||
}
|
||||
// Attempt to load a previously persisted snapshot and rebuild one if failed
|
||||
head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild)
|
||||
if disabled {
|
||||
log.Warn("Snapshot maintenance disabled (syncing)")
|
||||
return snap, nil
|
||||
}
|
||||
// Create the building waiter iff the background generation is allowed
|
||||
if !config.NoBuild && !config.AsyncBuild {
|
||||
defer snap.waitBuild()
|
||||
}
|
||||
if err != nil {
|
||||
log.Warn("Failed to load snapshot", "err", err)
|
||||
if !config.NoBuild {
|
||||
|
@ -63,7 +63,7 @@ func (s Storage) Copy() Storage {
|
||||
// The usage pattern is as follows:
|
||||
// First you need to obtain a state object.
|
||||
// Account values can be accessed and modified through the object.
|
||||
// Finally, call CommitTrie to write the modified storage trie into a database.
|
||||
// Finally, call commitTrie to write the modified storage trie into a database.
|
||||
type stateObject struct {
|
||||
address common.Address
|
||||
addrHash common.Hash // hash of ethereum address of the account
|
||||
@ -374,9 +374,9 @@ func (s *stateObject) updateRoot(db Database) {
|
||||
s.data.Root = s.trie.Hash()
|
||||
}
|
||||
|
||||
// CommitTrie the storage trie of the object to db.
|
||||
// This updates the trie root.
|
||||
func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
|
||||
// commitTrie submits the storage changes into the storage trie and re-computes
|
||||
// the root. Besides, all trie changes will be collected in a nodeset and returned.
|
||||
func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
|
||||
// If nothing changed, don't bother with hashing anything
|
||||
if s.updateTrie(db) == nil {
|
||||
return nil, nil
|
||||
|
@ -920,7 +920,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
|
||||
obj.dirtyCode = false
|
||||
}
|
||||
// Write any storage changes in the state object to its storage trie
|
||||
set, err := obj.CommitTrie(s.db)
|
||||
set, err := obj.commitTrie(s.db)
|
||||
if err != nil {
|
||||
return common.Hash{}, err
|
||||
}
|
||||
@ -934,6 +934,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
|
||||
storageTrieNodesDeleted += deleted
|
||||
}
|
||||
}
|
||||
// If the contract is destructed, the storage is still left in the
|
||||
// database as dangling data. Theoretically it's should be wiped from
|
||||
// database as well, but in hash-based-scheme it's extremely hard to
|
||||
// determine that if the trie nodes are also referenced by other storage,
|
||||
// and in path-based-scheme some technical challenges are still unsolved.
|
||||
// Although it won't affect the correctness but please fix it TODO(rjl493456442).
|
||||
}
|
||||
if len(s.stateObjectsDirty) > 0 {
|
||||
s.stateObjectsDirty = make(map[common.Address]struct{})
|
||||
|
@ -113,6 +113,7 @@ func NewNodeSet(owner common.Hash) *NodeSet {
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
// NewNodeSetWithDeletion initializes the nodeset with provided deletion set.
|
||||
func NewNodeSetWithDeletion(owner common.Hash, paths [][]byte, prev [][]byte) *NodeSet {
|
||||
set := NewNodeSet(owner)
|
||||
@ -121,6 +122,7 @@ func NewNodeSetWithDeletion(owner common.Hash, paths [][]byte, prev [][]byte) *N
|
||||
}
|
||||
return set
|
||||
}
|
||||
*/
|
||||
|
||||
// markUpdated marks the node as dirty(newly-inserted or updated) with provided
|
||||
// node path, node object along with its previous value.
|
||||
|
Loading…
Reference in New Issue
Block a user