roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

all: activate pbss as experimental feature (#26274)

Browse Source 
* all: activate pbss

* core/rawdb: fix compilation error

* cma, core, eth, les, trie: address comments

* cmd, core, eth, trie: polish code

* core, cmd, eth: address comments

* cmd, core, eth, les, light, tests: address comment

* cmd/utils: shorten log message

* trie/triedb/pathdb: limit node buffer size to 1gb

* cmd/utils: fix opening non-existing db

* cmd/utils: rename flag name

* cmd, core: group chain history flags and fix tests

* core, eth, trie: fix memory leak in snapshot generation

* cmd, eth, internal: deprecate flags

* all: enable state tests for pathdb, fixes

* cmd, core: polish code

* trie/triedb/pathdb: limit the node buffer size to 256mb

---------

Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
rjl493456442
2023-08-10 22:21:36 +03:00
committed by GitHub
co-authored by Martin Holst Swende Péter Szilágyi
parent 5e89ff4d6b
commit 503f1f7ada
96 changed files with 2372 additions and 1021 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/block_test.go
+12 -4
View File
@@ -18,6 +18,8 @@ package tests
import (
"testing"
"github.com/ethereum/go-ethereum/core/rawdb"
)
func TestBlockchain(t *testing.T) {
@@ -48,11 +50,17 @@ func TestBlockchain(t *testing.T) {
bt.skipLoad(`.*randomStatetest94.json.*`)
bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
if err := bt.checkFailure(t, test.Run(false, nil)); err != nil {
t.Errorf("test without snapshotter failed: %v", err)
if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil {
t.Errorf("test in hash mode without snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(true, nil)); err != nil {
t.Errorf("test with snapshotter failed: %v", err)
if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil {
t.Errorf("test in hash mode with snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil {
t.Errorf("test in path mode without snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil {
t.Errorf("test in path mode with snapshotter failed: %v", err)
}
})
// There is also a LegacyTests folder, containing blockchain tests generated
tests/block_test_util.go
+22 -5
View File
@@ -38,6 +38,9 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
// A BlockTest checks handling of entire blocks.
@@ -100,16 +103,30 @@ type btHeaderMarshaling struct {
BaseFeePerGas *math.HexOrDecimal256
}
func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error {
func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error {
config, ok := Forks[t.json.Network]
if !ok {
return UnsupportedForkError{t.json.Network}
}
// import pre accounts & construct test genesis block & state root
db := rawdb.NewMemoryDatabase()
var (
db = rawdb.NewMemoryDatabase()
tconf = &trie.Config{}
)
if scheme == rawdb.PathScheme {
tconf.PathDB = pathdb.Defaults
} else {
tconf.HashDB = hashdb.Defaults
}
// Commit genesis state
gspec := t.genesis(config)
gblock := gspec.MustCommit(db)
triedb := trie.NewDatabase(db, tconf)
gblock, err := gspec.Commit(db, triedb)
if err != nil {
return err
}
triedb.Close() // close the db to prevent memory leak
if gblock.Hash() != t.json.Genesis.Hash {
return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6])
}
@@ -119,7 +136,7 @@ func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error {
// Wrap the original engine within the beacon-engine
engine := beacon.New(ethash.NewFaker())
cache := &core.CacheConfig{TrieCleanLimit: 0}
cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme}
if snapshotter {
cache.SnapshotLimit = 1
cache.SnapshotWait = true
tests/fuzzers/les/les-fuzzer.go
+2 -2
View File
@@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addresses []common.Address, txHashes []co
}
func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) {
chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults))
bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults))
for i := 0; i < testChainLen; i++ {
// The element in CHT is <big-endian block number> -> <block hash>
key := make([]byte, 8)
tests/fuzzers/rangeproof/rangeproof-fuzzer.go
+1 -1
View File
@@ -56,7 +56,7 @@ func (f *fuzzer) readInt() uint64 {
}
func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv)
size := f.readInt()
// Fill it with some fluff
tests/fuzzers/stacktrie/trie_fuzzer.go
+2 -2
View File
@@ -136,10 +136,10 @@ func (f *fuzzer) fuzz() int {
// This spongeDb is used to check the sequence of disk-db-writes
var (
spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA))
dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA), nil)
trieA = trie.NewEmpty(dbA)
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB))
dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB), nil)
trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme())
})
tests/fuzzers/trie/trie-fuzzer.go
+1 -1
View File
@@ -143,7 +143,7 @@ func Fuzz(input []byte) int {
func runRandTest(rt randTest) error {
var (
triedb = trie.NewDatabase(rawdb.NewMemoryDatabase())
triedb = trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)
tr = trie.NewEmpty(triedb)
origin = types.EmptyRootHash
values = make(map[string]string) // tracks content of the trie
tests/state_test.go
+45 -11
View File
@@ -30,6 +30,8 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
@@ -78,21 +80,52 @@ func TestState(t *testing.T) {
subtest := subtest
key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
t.Run(key+"/trie", func(t *testing.T) {
t.Run(key+"/hash/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
_, _, err := test.Run(subtest, vmconfig, false)
return st.checkFailure(t, err)
var result error
test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/snap", func(t *testing.T) {
t.Run(key+"/hash/snap", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
snaps, statedb, err := test.Run(subtest, vmconfig, true)
if snaps != nil && statedb != nil {
if _, err := snaps.Journal(statedb.IntermediateRoot(false)); err != nil {
return err
var result error
test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if snaps != nil && state != nil {
if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil {
result = err
return
}
}
}
return st.checkFailure(t, err)
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/path/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
var result error
test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/path/snap", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
var result error
test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if snaps != nil && state != nil {
if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil {
result = err
return
}
}
result = st.checkFailure(t, err)
})
return result
})
})
}
@@ -190,7 +223,8 @@ func runBenchmark(b *testing.B, t *StateTest) {
vmconfig.ExtraEips = eips
block := t.genesis(config).ToBlock()
_, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false)
triedb, _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme)
defer triedb.Close()
var baseFee *big.Int
if rules.IsLondon {
tests/state_test_util.go
+46 -25
View File
@@ -39,6 +39,8 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"golang.org/x/crypto/sha3"
)
@@ -187,43 +189,50 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error {
}
// Run executes a specific subtest and verifies the post-state and logs
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) {
snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter)
if checkedErr := t.checkError(subtest, err); checkedErr != nil {
return snaps, statedb, checkedErr
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) {
triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme)
// Invoke the callback at the end of function for further analysis.
defer func() {
postCheck(result, snaps, statedb)
if triedb != nil {
triedb.Close()
}
}()
checkedErr := t.checkError(subtest, err)
if checkedErr != nil {
return checkedErr
}
// The error has been checked; if it was unexpected, it's already returned.
if err != nil {
// Here, an error exists but it was expected.
// We do not check the post state or logs.
return snaps, statedb, nil
return nil
}
post := t.json.Post[subtest.Fork][subtest.Index]
// N.B: We need to do this in a two-step process, because the first Commit takes care
// of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed.
if root != common.Hash(post.Root) {
return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
}
if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) {
return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
}
// Re-init the post-state instance for further operation
statedb, err = state.New(root, statedb.Database(), snaps)
if err != nil {
return nil, nil, err
}
return snaps, statedb, nil
statedb, _ = state.New(root, statedb.Database(), snaps)
return nil
}
// RunNoVerify runs a specific subtest and returns the statedb and post-state root
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, common.Hash, error) {
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) {
config, eips, err := GetChainConfig(subtest.Fork)
if err != nil {
return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork}
return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork}
}
vmconfig.ExtraEips = eips
block := t.genesis(config).ToBlock()
snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter)
triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme)
var baseFee *big.Int
if config.IsLondon(new(big.Int)) {
@@ -237,7 +246,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
post := t.json.Post[subtest.Fork][subtest.Index]
msg, err := t.json.Tx.toMessage(post, baseFee)
if err != nil {
return nil, nil, common.Hash{}, err
triedb.Close()
return nil, nil, nil, common.Hash{}, err
}
// Try to recover tx with current signer
@@ -245,11 +255,13 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
var ttx types.Transaction
err := ttx.UnmarshalBinary(post.TxBytes)
if err != nil {
return nil, nil, common.Hash{}, err
triedb.Close()
return nil, nil, nil, common.Hash{}, err
}
if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil {
return nil, nil, common.Hash{}, err
triedb.Close()
return nil, nil, nil, common.Hash{}, err
}
}
@@ -268,6 +280,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
context.Difficulty = big.NewInt(0)
}
evm := vm.NewEVM(context, txContext, statedb, config, vmconfig)
// Execute the message.
snapshot := statedb.Snapshot()
gaspool := new(core.GasPool)
@@ -282,17 +295,25 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
// - there are only 'bad' transactions, which aren't executed. In those cases,
// the coinbase gets no txfee, so isn't created, and thus needs to be touched
statedb.AddBalance(block.Coinbase(), new(big.Int))
// Commit block
// Commit state mutations into database.
root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number()))
return snaps, statedb, root, err
return triedb, snaps, statedb, root, err
}
func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas]
}
func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) (*snapshot.Tree, *state.StateDB) {
sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true})
func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) {
tconf := &trie.Config{Preimages: true}
if scheme == rawdb.HashScheme {
tconf.HashDB = hashdb.Defaults
} else {
tconf.PathDB = pathdb.Defaults
}
triedb := trie.NewDatabase(db, tconf)
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ := state.New(types.EmptyRootHash, sdb, nil)
for addr, a := range accounts {
statedb.SetCode(addr, a.Code)
@@ -313,10 +334,10 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo
NoBuild: false,
AsyncBuild: false,
}
snaps, _ = snapshot.New(snapconfig, db, sdb.TrieDB(), root)
snaps, _ = snapshot.New(snapconfig, db, triedb, root)
}
statedb, _ = state.New(root, sdb, snaps)
return snaps, statedb
return triedb, snaps, statedb
}
func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis {