roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

all: activate pbss as experimental feature (#26274)

Browse Source 
* all: activate pbss

* core/rawdb: fix compilation error

* cma, core, eth, les, trie: address comments

* cmd, core, eth, trie: polish code

* core, cmd, eth: address comments

* cmd, core, eth, les, light, tests: address comment

* cmd/utils: shorten log message

* trie/triedb/pathdb: limit node buffer size to 1gb

* cmd/utils: fix opening non-existing db

* cmd/utils: rename flag name

* cmd, core: group chain history flags and fix tests

* core, eth, trie: fix memory leak in snapshot generation

* cmd, eth, internal: deprecate flags

* all: enable state tests for pathdb, fixes

* cmd, core: polish code

* trie/triedb/pathdb: limit the node buffer size to 256mb

---------

Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
rjl493456442
2023-08-10 22:21:36 +03:00
committed by GitHub
co-authored by Martin Holst Swende Péter Szilágyi
parent 5e89ff4d6b
commit 503f1f7ada
96 changed files with 2372 additions and 1021 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/rawdb/accessors_chain_test.go
+1 -1
View File
@@ -435,12 +435,12 @@ func checkReceiptsRLP(have, want types.Receipts) error {
func TestAncientStorage(t *testing.T) {
// Freezer style fast import the chain.
frdir := t.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
defer db.Close()
// Create a test block
block := types.NewBlockWithHeader(&types.Header{
Number: big.NewInt(0),
core/rawdb/accessors_trie.go
+24 -2
View File
@@ -36,7 +36,7 @@ import (
//
// Now this scheme is still kept for backward compatibility, and it will be used
// for archive node and some other tries(e.g. light trie).
const HashScheme = "hashScheme"
const HashScheme = "hash"
// PathScheme is the new path-based state scheme with which trie nodes are stored
// in the disk with node path as the database key. This scheme will only store one
@@ -44,7 +44,7 @@ const HashScheme = "hashScheme"
// is native. At the same time, this scheme will put adjacent trie nodes in the same
// area of the disk with good data locality property. But this scheme needs to rely
// on extra state diffs to survive deep reorg.
const PathScheme = "pathScheme"
const PathScheme = "path"
// hasher is used to compute the sha256 hash of the provided data.
type hasher struct{ sha crypto.KeccakState }
@@ -263,3 +263,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
}
// ReadStateScheme reads the state scheme of persistent state, or none
// if the state is not present in database.
func ReadStateScheme(db ethdb.Reader) string {
// Check if state in path-based scheme is present
blob, _ := ReadAccountTrieNode(db, nil)
if len(blob) != 0 {
return PathScheme
}
// In a hash-based scheme, the genesis state is consistently stored
// on the disk. To assess the scheme of the persistent state, it
// suffices to inspect the scheme of the genesis state.
header := ReadHeader(db, ReadCanonicalHash(db, 0), 0)
if header == nil {
return "" // empty datadir
}
blob = ReadLegacyTrieNode(db, header.Root)
if len(blob) == 0 {
return "" // no state in disk
}
return HashScheme
}
core/rawdb/ancient_scheme.go
+4 -4
View File
@@ -58,7 +58,7 @@ const (
stateHistoryStorageData = "storage.data"
)
var stateHistoryFreezerNoSnappy = map[string]bool{
var stateFreezerNoSnappy = map[string]bool{
stateHistoryMeta: true,
stateHistoryAccountIndex: false,
stateHistoryStorageIndex: false,
@@ -75,7 +75,7 @@ var (
// freezers the collections of all builtin freezers.
var freezers = []string{chainFreezerName, stateFreezerName}
// NewStateHistoryFreezer initializes the freezer for state history.
func NewStateHistoryFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateHistoryFreezerNoSnappy)
// NewStateFreezer initializes the freezer for state history.
func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
}
core/rawdb/ancient_utils.go
+39 -17
View File
@@ -50,36 +50,58 @@ func (info *freezerInfo) size() common.StorageSize {
return total
}
func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
info := freezerInfo{name: name}
for t := range order {
size, err := reader.AncientSize(t)
if err != nil {
return freezerInfo{}, err
}
info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := reader.Ancients()
if err != nil {
return freezerInfo{}, err
}
info.head = ancients - 1
// Retrieve the number of first stored item
tail, err := reader.Tail()
if err != nil {
return freezerInfo{}, err
}
info.tail = tail
return info, nil
}
// inspectFreezers inspects all freezers registered in the system.
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
var infos []freezerInfo
for _, freezer := range freezers {
switch freezer {
case chainFreezerName:
// Chain ancient store is a bit special. It's always opened along
// with the key-value store, inspect the chain store directly.
info := freezerInfo{name: freezer}
// Retrieve storage size of every contained table.
for table := range chainFreezerNoSnappy {
size, err := db.AncientSize(table)
if err != nil {
return nil, err
}
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := db.Ancients()
info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db)
if err != nil {
return nil, err
}
info.head = ancients - 1
infos = append(infos, info)
// Retrieve the number of first stored item
tail, err := db.Tail()
case stateFreezerName:
datadir, err := db.AncientDatadir()
if err != nil {
return nil, err
}
f, err := NewStateFreezer(datadir, true)
if err != nil {
return nil, err
}
defer f.Close()
info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f)
if err != nil {
return nil, err
}
info.tail = tail
infos = append(infos, info)
default:
core/rawdb/database.go
+17 -4
View File
@@ -463,7 +463,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
tds stat
numHashPairings stat
hashNumPairings stat
tries stat
legacyTries stat
stateLookups stat
accountTries stat
storageTries stat
codes stat
txLookups stat
accountSnaps stat
@@ -504,8 +507,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
numHashPairings.Add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
hashNumPairings.Add(size)
case len(key) == common.HashLength:
tries.Add(size)
case IsLegacyTrieNode(key, it.Value()):
legacyTries.Add(size)
case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
stateLookups.Add(size)
case IsAccountTrieNode(key):
accountTries.Add(size)
case IsStorageTrieNode(key):
storageTries.Add(size)
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
codes.Add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
@@ -543,6 +552,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey,
} {
if bytes.Equal(key, meta) {
metadata.Add(size)
@@ -571,7 +581,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},
{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},
core/rawdb/schema.go
+21 -5
View File
@@ -273,9 +273,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
return bytes.Equal(key, crypto.Keccak256(val))
}
// IsAccountTrieNode reports whether a provided database entry is an account
// trie node in path-based state scheme.
func IsAccountTrieNode(key []byte) (bool, []byte) {
// ResolveAccountTrieNodeKey reports whether a provided database entry is an
// account trie node in path-based state scheme, and returns the resolved
// node path if so.
func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
return false, nil
}
@@ -288,9 +289,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) {
return true, key[len(trieNodeAccountPrefix):]
}
// IsStorageTrieNode reports whether a provided database entry is a storage
// IsAccountTrieNode reports whether a provided database entry is an account
// trie node in path-based state scheme.
func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
func IsAccountTrieNode(key []byte) bool {
ok, _ := ResolveAccountTrieNodeKey(key)
return ok
}
// ResolveStorageTrieNode reports whether a provided database entry is a storage
// trie node in path-based state scheme, and returns the resolved account hash
// and node path if so.
func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
return false, common.Hash{}, nil
}
@@ -306,3 +315,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
}
// IsStorageTrieNode reports whether a provided database entry is a storage
// trie node in path-based state scheme.
func IsStorageTrieNode(key []byte) bool {
ok, _, _ := ResolveStorageTrieNode(key)
return ok
}