roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

all: activate pbss as experimental feature (#26274)

Browse Source 
* all: activate pbss

* core/rawdb: fix compilation error

* cma, core, eth, les, trie: address comments

* cmd, core, eth, trie: polish code

* core, cmd, eth: address comments

* cmd, core, eth, les, light, tests: address comment

* cmd/utils: shorten log message

* trie/triedb/pathdb: limit node buffer size to 1gb

* cmd/utils: fix opening non-existing db

* cmd/utils: rename flag name

* cmd, core: group chain history flags and fix tests

* core, eth, trie: fix memory leak in snapshot generation

* cmd, eth, internal: deprecate flags

* all: enable state tests for pathdb, fixes

* cmd, core: polish code

* trie/triedb/pathdb: limit the node buffer size to 256mb

---------

Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
rjl493456442 2023-08-11 03:21:36 +08:00 committed by GitHub
parent 5e89ff4d6b
commit 503f1f7ada
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
96 changed files with 2372 additions and 1021 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/evm/blockrunner.go
View File

@ -22,6 +22,7 @@ import (
"fmt"
"os"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/log"
@ -64,7 +65,7 @@ func blockTestCmd(ctx *cli.Context) error {
return err
}
for i, test := range tests {
if err := test.Run(false, tracer); err != nil {
if err := test.Run(false, rawdb.HashScheme, tracer); err != nil {
return fmt.Errorf("test %v: %w", i, err)
}
}

18
cmd/evm/runner.go
View File

@ -42,6 +42,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/urfave/cli/v2"
)
@ -142,12 +143,23 @@ func runCmd(ctx *cli.Context) error {
gen := readGenesis(ctx.String(GenesisFlag.Name))
genesisConfig = gen
db := rawdb.NewMemoryDatabase()
genesis := gen.MustCommit(db)
sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: preimages})
triedb := trie.NewDatabase(db, &trie.Config{
Preimages: preimages,
HashDB: hashdb.Defaults,
})
defer triedb.Close()
genesis := gen.MustCommit(db, triedb)
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ = state.New(genesis.Root(), sdb, nil)
chainConfig = gen.Config
} else {
sdb := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: preimages})
db := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(db, &trie.Config{
Preimages: preimages,
HashDB: hashdb.Defaults,
})
defer triedb.Close()
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
genesisConfig = new(core.Genesis)
}

35
cmd/evm/staterunner.go
View File

@ -23,7 +23,9 @@ import (
"os"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/log"
@ -104,25 +106,22 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error {
for _, st := range test.Subtests() {
// Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
_, s, err := test.Run(st, cfg, false)
// print state root for evmlab tracing
if s != nil {
root := s.IntermediateRoot(false)
result.Root = &root
if jsonOut {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if err != nil {
// Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error()
if dump {
dump := state.RawDump(nil)
result.State = &dump
}
} else {
root := state.IntermediateRoot(false)
result.Root = &root
if jsonOut {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
}
}
}
if err != nil {
// Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error()
if dump && s != nil {
s, _ = state.New(*result.Root, s.Database(), nil)
dump := s.RawDump(nil)
result.State = &dump
}
}
})
results = append(results, *result)
}
}

30
cmd/geth/chaincmd.go
View File

@ -39,7 +39,6 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/trie"
"github.com/urfave/cli/v2"
)
@ -49,7 +48,10 @@ var (
Name: "init",
Usage: "Bootstrap and initialize a new genesis block",
ArgsUsage: "<genesisPath>",
Flags: flags.Merge([]cli.Flag{utils.CachePreimagesFlag}, utils.DatabasePathFlags),
Flags: flags.Merge([]cli.Flag{
utils.CachePreimagesFlag,
utils.StateSchemeFlag,
}, utils.DatabasePathFlags),
Description: `
The init command initializes a new genesis block and definition for the network.
This is a destructive action and changes the network in which you will be
@ -94,6 +96,9 @@ if one is set. Otherwise it prints the genesis from the datadir.`,
utils.MetricsInfluxDBBucketFlag,
utils.MetricsInfluxDBOrganizationFlag,
utils.TxLookupLimitFlag,
utils.TransactionHistoryFlag,
utils.StateSchemeFlag,
utils.StateHistoryFlag,
}, utils.DatabasePathFlags),
Description: `
The import command imports blocks from an RLP-encoded form. The form can be one file
@ -110,6 +115,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
Flags: flags.Merge([]cli.Flag{
utils.CacheFlag,
utils.SyncModeFlag,
utils.StateSchemeFlag,
}, utils.DatabasePathFlags),
Description: `
Requires a first argument of the file to write to.
@ -159,6 +165,7 @@ It's deprecated, please use "geth db export" instead.
utils.IncludeIncompletesFlag,
utils.StartKeyFlag,
utils.DumpLimitFlag,
utils.StateSchemeFlag,
}, utils.DatabasePathFlags),
Description: `
This command dumps out the state for a given block (or latest, if none provided).
@ -195,14 +202,15 @@ func initGenesis(ctx *cli.Context) error {
if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}
triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{
Preimages: ctx.Bool(utils.CachePreimagesFlag.Name),
})
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
defer triedb.Close()
_, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis)
if err != nil {
utils.Fatalf("Failed to write genesis block: %v", err)
}
chaindb.Close()
log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
}
return nil
@ -241,7 +249,7 @@ func dumpGenesis(ctx *cli.Context) error {
if ctx.IsSet(utils.DataDirFlag.Name) {
utils.Fatalf("no existing datadir at %s", stack.Config().DataDir)
}
utils.Fatalf("no network preset provided. no exisiting genesis in the default datadir")
utils.Fatalf("no network preset provided, no existing genesis in the default datadir")
return nil
}
@ -465,10 +473,10 @@ func dump(ctx *cli.Context) error {
if err != nil {
return err
}
config := &trie.Config{
Preimages: true, // always enable preimage lookup
}
state, err := state.New(root, state.NewDatabaseWithConfig(db, config), nil)
triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup
defer triedb.Close()
state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
if err != nil {
return err
}

6
cmd/geth/dbcmd.go
View File

@ -151,6 +151,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
Flags: flags.Merge([]cli.Flag{
utils.SyncModeFlag,
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: "This command looks up the specified database key from the database.",
}
@ -482,6 +483,9 @@ func dbDumpTrie(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
triedb := utils.MakeTrieDatabase(ctx, db, false, true)
defer triedb.Close()
var (
state []byte
storage []byte
@ -515,7 +519,7 @@ func dbDumpTrie(ctx *cli.Context) error {
}
}
id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage))
theTrie, err := trie.New(id, trie.NewDatabase(db))
theTrie, err := trie.New(id, triedb)
if err != nil {
return err
}

4
cmd/geth/genesis_test.go
View File

@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) {
{ // Can't start pebble on top of leveldb
initArgs: []string{"--db.engine", "leveldb"},
execArgs: []string{"--db.engine", "pebble"},
execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
},
{ // Can't start leveldb on top of pebble
initArgs: []string{"--db.engine", "pebble"},
execArgs: []string{"--db.engine", "leveldb"},
execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
},
{ // Reject invalid backend choice
initArgs: []string{"--db.engine", "mssql"},

3
cmd/geth/main.go
View File

@ -88,6 +88,9 @@ var (
utils.GCModeFlag,
utils.SnapshotFlag,
utils.TxLookupLimitFlag,
utils.TransactionHistoryFlag,
utils.StateSchemeFlag,
utils.StateHistoryFlag,
utils.LightServeFlag,
utils.LightIngressFlag,
utils.LightEgressFlag,

54
cmd/geth/snapshot.go
View File

@ -61,10 +61,7 @@ two version states are available: genesis and the specific one.
The default pruning target is the HEAD-127 state.
WARNING: It's necessary to delete the trie clean cache after the pruning.
If you specify another directory for the trie clean cache via "--cache.trie.journal"
during the use of Geth, please also specify it here for correct deletion. Otherwise
the trie clean cache with default directory will be deleted.
WARNING: it's only supported in hash mode(--state.scheme=hash)".
`,
},
{
@ -72,7 +69,9 @@ the trie clean cache with default directory will be deleted.
Usage: "Recalculate state hash based on the snapshot for verification",
ArgsUsage: "<root>",
Action: verifyState,
Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
Flags: flags.Merge([]cli.Flag{
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: `
geth snapshot verify-state <state-root>
will traverse the whole accounts and storages set based on the specified
@ -107,7 +106,9 @@ information about the specified address.
Usage: "Traverse the state with given root hash and perform quick verification",
ArgsUsage: "<root>",
Action: traverseState,
Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
Flags: flags.Merge([]cli.Flag{
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: `
geth snapshot traverse-state <state-root>
will traverse the whole state from the given state root and will abort if any
@ -122,7 +123,9 @@ It's also usable without snapshot enabled.
Usage: "Traverse the state with given root hash and perform detailed verification",
ArgsUsage: "<root>",
Action: traverseRawState,
Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
Flags: flags.Merge([]cli.Flag{
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: `
geth snapshot traverse-rawstate <state-root>
will traverse the whole state from the given root and will abort if any referenced
@ -143,6 +146,7 @@ It's also usable without snapshot enabled.
utils.ExcludeStorageFlag,
utils.StartKeyFlag,
utils.DumpLimitFlag,
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: `
This command is semantically equivalent to 'geth dump', but uses the snapshots
@ -165,6 +169,9 @@ func pruneState(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, false)
defer chaindb.Close()
if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
log.Crit("Offline pruning is not required for path scheme")
}
prunerconfig := pruner.Config{
Datadir: stack.ResolvePath(""),
BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name),
@ -205,13 +212,16 @@ func verifyState(ctx *cli.Context) error {
log.Error("Failed to load head block")
return errors.New("no head block")
}
snapconfig := snapshot.Config{
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
snapConfig := snapshot.Config{
CacheSize: 256,
Recovery: false,
NoBuild: true,
AsyncBuild: false,
}
snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root())
snaptree, err := snapshot.New(snapConfig, chaindb, triedb, headBlock.Root())
if err != nil {
log.Error("Failed to open snapshot tree", "err", err)
return err
@ -253,6 +263,11 @@ func traverseState(ctx *cli.Context) error {
defer stack.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
log.Error("Failed to load head block")
@ -277,7 +292,6 @@ func traverseState(ctx *cli.Context) error {
root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb)
if err != nil {
log.Error("Failed to open trie", "root", root, "err", err)
@ -353,6 +367,11 @@ func traverseRawState(ctx *cli.Context) error {
defer stack.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
log.Error("Failed to load head block")
@ -377,7 +396,6 @@ func traverseRawState(ctx *cli.Context) error {
root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb)
if err != nil {
log.Error("Failed to open trie", "root", root, "err", err)
@ -398,6 +416,11 @@ func traverseRawState(ctx *cli.Context) error {
log.Error("Failed to open iterator", "root", root, "err", err)
return err
}
reader, err := triedb.Reader(root)
if err != nil {
log.Error("State is non-existent", "root", root)
return nil
}
for accIter.Next(true) {
nodes += 1
node := accIter.Hash()
@ -405,7 +428,7 @@ func traverseRawState(ctx *cli.Context) error {
// Check the present for non-empty hash node(embedded node doesn't
// have their own hash).
if node != (common.Hash{}) {
blob := rawdb.ReadLegacyTrieNode(chaindb, node)
blob, _ := reader.Node(common.Hash{}, accIter.Path(), node)
if len(blob) == 0 {
log.Error("Missing trie node(account)", "hash", node)
return errors.New("missing account")
@ -446,7 +469,7 @@ func traverseRawState(ctx *cli.Context) error {
// Check the presence for non-empty hash node(embedded node doesn't
// have their own hash).
if node != (common.Hash{}) {
blob := rawdb.ReadLegacyTrieNode(chaindb, node)
blob, _ := reader.Node(common.BytesToHash(accIter.LeafKey()), storageIter.Path(), node)
if len(blob) == 0 {
log.Error("Missing trie node(storage)", "hash", node)
return errors.New("missing storage")
@ -506,13 +529,16 @@ func dumpState(ctx *cli.Context) error {
if err != nil {
return err
}
triedb := utils.MakeTrieDatabase(ctx, db, false, true)
defer triedb.Close()
snapConfig := snapshot.Config{
CacheSize: 256,
Recovery: false,
NoBuild: true,
AsyncBuild: false,
}
snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db), root)
snaptree, err := snapshot.New(snapConfig, db, triedb, root)
if err != nil {
return err
}

185
cmd/utils/flags.go
View File

@ -48,7 +48,7 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth"
ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst"
"github.com/ethereum/go-ethereum/eth/catalyst"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/filters"
@ -74,6 +74,9 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
pcsclite "github.com/gballet/go-libpcsclite"
gopsutil "github.com/shirou/gopsutil/mem"
"github.com/urfave/cli/v2"
@ -218,30 +221,12 @@ var (
}
defaultSyncMode = ethconfig.Defaults.SyncMode
SyncModeFlag = &flags.TextMarshalerFlag{
Name: "syncmode",
Usage: `Blockchain sync mode ("snap", "full" or "light")`,
Value: &defaultSyncMode,
Category: flags.EthCategory,
}
GCModeFlag = &cli.StringFlag{
Name: "gcmode",
Usage: `Blockchain garbage collection mode ("full", "archive")`,
Value: "full",
Category: flags.EthCategory,
}
SnapshotFlag = &cli.BoolFlag{
SnapshotFlag = &cli.BoolFlag{
Name: "snapshot",
Usage: `Enables snapshot-database mode (default = enable)`,
Value: true,
Category: flags.EthCategory,
}
TxLookupLimitFlag = &cli.Uint64Flag{
Name: "txlookuplimit",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
Value: ethconfig.Defaults.TxLookupLimit,
Category: flags.EthCategory,
}
LightKDFFlag = &cli.BoolFlag{
Name: "lightkdf",
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@ -268,6 +253,36 @@ var (
Usage: "Manually specify the Verkle fork timestamp, overriding the bundled setting",
Category: flags.EthCategory,
}
SyncModeFlag = &flags.TextMarshalerFlag{
Name: "syncmode",
Usage: `Blockchain sync mode ("snap", "full" or "light")`,
Value: &defaultSyncMode,
Category: flags.StateCategory,
}
GCModeFlag = &cli.StringFlag{
Name: "gcmode",
Usage: `Blockchain garbage collection mode, only relevant in state.scheme=hash ("full", "archive")`,
Value: "full",
Category: flags.StateCategory,
}
StateSchemeFlag = &cli.StringFlag{
Name: "state.scheme",
Usage: "Scheme to use for storing ethereum state ('hash' or 'path')",
Value: rawdb.HashScheme,
Category: flags.StateCategory,
}
StateHistoryFlag = &cli.Uint64Flag{
Name: "history.state",
Usage: "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)",
Value: ethconfig.Defaults.StateHistory,
Category: flags.StateCategory,
}
TransactionHistoryFlag = &cli.Uint64Flag{
Name: "history.transactions",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.StateCategory,
}
// Light server and client settings
LightServeFlag = &cli.IntFlag{
Name: "light.serve",
@ -1624,13 +1639,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
CheckExclusive(ctx, MainnetFlag, DeveloperFlag, GoerliFlag, SepoliaFlag)
CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light")
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 {
ctx.Set(TxLookupLimitFlag.Name, "0")
log.Warn("Disable transaction unindexing for archive node")
}
if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 {
log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited")
}
// Set configurations from CLI flags
setEtherbase(ctx, cfg)
setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light")
setTxPool(ctx, &cfg.TxPool)
@ -1687,8 +1697,36 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
cfg.Preimages = true
log.Info("Enabling recording of key preimages since archive mode is used")
}
if ctx.IsSet(TxLookupLimitFlag.Name) {
cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name)
if ctx.IsSet(StateHistoryFlag.Name) {
cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
}
// Parse state scheme, abort the process if it's not compatible.
chaindb := tryMakeReadOnlyDatabase(ctx, stack)
scheme, err := ParseStateScheme(ctx, chaindb)
chaindb.Close()
if err != nil {
Fatalf("%v", err)
}
cfg.StateScheme = scheme
// Parse transaction history flag, if user is still using legacy config
// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
log.Warn("The config option 'TxLookupLimit' is deprecated and will be removed, please use 'TransactionHistory'")
cfg.TransactionHistory = cfg.TxLookupLimit
}
if ctx.IsSet(TransactionHistoryFlag.Name) {
cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name)
} else if ctx.IsSet(TxLookupLimitFlag.Name) {
log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions")
cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name)
}
if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
cfg.TransactionHistory = 0
log.Warn("Disabled transaction unindexing for archive node")
}
if ctx.IsSet(LightServeFlag.Name) && cfg.TransactionHistory != 0 {
log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited")
}
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
@ -1814,15 +1852,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
// Create a new developer genesis block or reuse existing one
cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address)
if ctx.IsSet(DataDirFlag.Name) {
// If datadir doesn't exist we need to open db in write-mode
// so leveldb can create files.
readonly := true
if !common.FileExist(stack.ResolvePath("chaindata")) {
readonly = false
}
// Check if we have an already initialized chain and fall back to
// that if so. Otherwise we need to generate a new genesis spec.
chaindb := MakeChainDatabase(ctx, stack, readonly)
chaindb := tryMakeReadOnlyDatabase(ctx, stack)
if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
cfg.Genesis = nil // fallback to db content
}
@ -1930,7 +1960,7 @@ func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, path string) {
if err := rlp.DecodeBytes(rlpBlob, &block); err != nil {
Fatalf("Failed to decode block: %v", err)
}
ethcatalyst.RegisterFullSyncTester(stack, eth, &block)
catalyst.RegisterFullSyncTester(stack, eth, &block)
log.Info("Registered full-sync tester", "number", block.NumberU64(), "hash", block.Hash())
}
@ -2040,6 +2070,18 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
return chainDb
}
// tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
// or fallback to write mode if the database is not initialized.
func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
// If datadir doesn't exist we need to open db in write-mode
// so database engine can create files.
readonly := true
if !common.FileExist(stack.ResolvePath("chaindata")) {
readonly = false
}
return MakeChainDatabase(ctx, stack, readonly)
}
func IsNetworkPreset(ctx *cli.Context) bool {
for _, flag := range NetworkFlags {
bFlag, _ := flag.(*cli.BoolFlag)
@ -2106,6 +2148,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
scheme, err := ParseStateScheme(ctx, chainDb)
if err != nil {
Fatalf("%v", err)
}
cache := &core.CacheConfig{
TrieCleanLimit: ethconfig.Defaults.TrieCleanCache,
TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name),
@ -2114,6 +2160,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
TrieTimeLimit: ethconfig.Defaults.TrieTimeout,
SnapshotLimit: ethconfig.Defaults.SnapshotCache,
Preimages: ctx.Bool(CachePreimagesFlag.Name),
StateScheme: scheme,
StateHistory: ctx.Uint64(StateHistoryFlag.Name),
}
if cache.TrieDirtyDisabled && !cache.Preimages {
cache.Preimages = true
@ -2158,3 +2206,62 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
}
return preloads
}
// ParseStateScheme resolves scheme identifier from CLI flag. If the provided
// state scheme is not compatible with the one of persistent scheme, an error
// will be returned.
//
// - none: use the scheme consistent with persistent state, or fallback
// to hash-based scheme if state is empty.
// - hash: use hash-based scheme or error out if not compatible with
// persistent state scheme.
// - path: use path-based scheme or error out if not compatible with
// persistent state scheme.
func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) {
// If state scheme is not specified, use the scheme consistent
// with persistent state, or fallback to hash mode if database
// is empty.
stored := rawdb.ReadStateScheme(disk)
if !ctx.IsSet(StateSchemeFlag.Name) {
if stored == "" {
// use default scheme for empty database, flip it when
// path mode is chosen as default
log.Info("State schema set to default", "scheme", "hash")
return rawdb.HashScheme, nil
}
log.Info("State scheme set to already existing", "scheme", stored)
return stored, nil // reuse scheme of persistent scheme
}
// If state scheme is specified, ensure it's compatible with
// persistent state.
scheme := ctx.String(StateSchemeFlag.Name)
if stored == "" || scheme == stored {
log.Info("State scheme set by user", "scheme", scheme)
return scheme, nil
}
return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme)
}
// MakeTrieDatabase constructs a trie database based on the configured scheme.
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database {
config := &trie.Config{
Preimages: preimage,
}
scheme, err := ParseStateScheme(ctx, disk)
if err != nil {
Fatalf("%v", err)
}
if scheme == rawdb.HashScheme {
// Read-only mode is not implemented in hash mode,
// ignore the parameter silently. TODO(rjl493456442)
// please config it if read mode is implemented.
config.HashDB = hashdb.Defaults
return trie.NewDatabase(disk, config)
}
if readOnly {
config.PathDB = pathdb.ReadOnly
} else {
config.PathDB = pathdb.Defaults
}
return trie.NewDatabase(disk, config)
}

9
cmd/utils/flags_legacy.go
View File

@ -19,6 +19,7 @@ package utils
import (
"fmt"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/urfave/cli/v2"
)
@ -37,6 +38,7 @@ var DeprecatedFlags = []cli.Flag{
CacheTrieJournalFlag,
CacheTrieRejournalFlag,
LegacyDiscoveryV5Flag,
TxLookupLimitFlag,
}
var (
@ -68,6 +70,13 @@ var (
Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)",
Category: flags.DeprecatedCategory,
}
// Deprecated August 2023
TxLookupLimitFlag = &cli.Uint64Flag{
Name: "txlookuplimit",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain) (deprecated, use history.transactions instead)",
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.DeprecatedCategory,
}
)
// showDeprecated displays deprecated flags that will be soon removed from the codebase.

7
core/block_validator_test.go
View File

@ -35,6 +35,11 @@ import (
// Tests that simple header verification works, for both good and bad blocks.
func TestHeaderVerification(t *testing.T) {
testHeaderVerification(t, rawdb.HashScheme)
testHeaderVerification(t, rawdb.PathScheme)
}
func testHeaderVerification(t *testing.T, scheme string) {
// Create a simple chain to verify
var (
gspec = &Genesis{Config: params.TestChainConfig}
@ -45,7 +50,7 @@ func TestHeaderVerification(t *testing.T) {
headers[i] = block.Header()
}
// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
defer chain.Stop()
for i := 0; i < len(blocks); i++ {

163
core/blockchain.go
View File

@ -48,6 +48,8 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"golang.org/x/exp/slices"
)
@ -128,7 +130,7 @@ const (
)
// CacheConfig contains the configuration values for the trie database
// that's resident in a blockchain.
// and state snapshot these are resident in a blockchain.
type CacheConfig struct {
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks
@ -137,11 +139,31 @@ type CacheConfig struct {
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory
Preimages bool // Whether to store preimage of trie key to the disk
StateHistory uint64 // Number of blocks from head whose state histories are reserved.
StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top
SnapshotNoBuild bool // Whether the background generation is allowed
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
}
// triedbConfig derives the configures for trie database.
func (c *CacheConfig) triedbConfig() *trie.Config {
config := &trie.Config{Preimages: c.Preimages}
if c.StateScheme == rawdb.HashScheme {
config.HashDB = &hashdb.Config{
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
}
}
if c.StateScheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{
StateHistory: c.StateHistory,
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024,
}
}
return config
}
// defaultCacheConfig are the default caching values if none are specified by the
// user (also used during testing).
var defaultCacheConfig = &CacheConfig{
@ -150,6 +172,15 @@ var defaultCacheConfig = &CacheConfig{
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256,
SnapshotWait: true,
StateScheme: rawdb.HashScheme,
}
// DefaultCacheConfigWithScheme returns a deep copied default cache config with
// a provided trie node scheme.
func DefaultCacheConfigWithScheme(scheme string) *CacheConfig {
config := *defaultCacheConfig
config.StateScheme = scheme
return &config
}
// BlockChain represents the canonical chain given a database with a genesis
@ -235,10 +266,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
cacheConfig = defaultCacheConfig
}
// Open trie database with provided config
triedb := trie.NewDatabaseWithConfig(db, &trie.Config{
Cache: cacheConfig.TrieCleanLimit,
Preimages: cacheConfig.Preimages,
})
triedb := trie.NewDatabase(db, cacheConfig.triedbConfig())
// Setup the genesis block, commit the provided genesis specification
// to database if the genesis block is not present yet, or load the
// stored one from database.
@ -612,6 +641,25 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
pivot := rawdb.ReadLastPivotNumber(bc.db)
frozen, _ := bc.db.Ancients()
// resetState resets the persistent state to genesis if it's not available.
resetState := func() {
// Short circuit if the genesis state is already present.
if bc.HasState(bc.genesisBlock.Root()) {
return
}
// Reset the state database to empty for committing genesis state.
// Note, it should only happen in path-based scheme and Reset function
// is also only call-able in this mode.
if bc.triedb.Scheme() == rawdb.PathScheme {
if err := bc.triedb.Reset(types.EmptyRootHash); err != nil {
log.Crit("Failed to clean state", "err", err) // Shouldn't happen
}
}
// Write genesis state into database.
if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil {
log.Crit("Failed to commit genesis state", "err", err)
}
}
updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) {
// Rewind the blockchain, ensuring we don't end up with a stateless head
// block. Note, depth equality is permitted to allow using SetHead as a
@ -621,6 +669,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
if newHeadBlock == nil {
log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash())
newHeadBlock = bc.genesisBlock
resetState()
} else {
// Block exists, keep rewinding until we find one with state,
// keeping rewinding until we exceed the optional threshold
@ -632,7 +681,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root {
beyondRoot, rootNumber = true, newHeadBlock.NumberU64()
}
if !bc.HasState(newHeadBlock.Root()) {
if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) {
log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
if pivot == nil || newHeadBlock.NumberU64() > *pivot {
parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1)
@ -649,16 +698,12 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
}
if beyondRoot || newHeadBlock.NumberU64() == 0 {
if newHeadBlock.NumberU64() == 0 {
// Recommit the genesis state into disk in case the rewinding destination
// is genesis block and the relevant state is gone. In the future this
// rewinding destination can be the earliest block stored in the chain
// if the historical chain pruning is enabled. In that case the logic
// needs to be improved here.
if !bc.HasState(bc.genesisBlock.Root()) {
if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil {
log.Crit("Failed to commit genesis state", "err", err)
}
log.Debug("Recommitted genesis state to disk")
resetState()
} else if !bc.HasState(newHeadBlock.Root()) {
// Rewind to a block with recoverable state. If the state is
// missing, run the state recovery here.
if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil {
log.Crit("Failed to rollback state", "err", err) // Shouldn't happen
}
}
log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
@ -772,7 +817,13 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x..]", hash[:4])
}
// Reset the trie database with the fresh snap synced state.
root := block.Root()
if bc.triedb.Scheme() == rawdb.PathScheme {
if err := bc.triedb.Reset(root); err != nil {
return err
}
}
if !bc.HasState(root) {
return fmt.Errorf("non existent state [%x..]", root[:4])
}
@ -937,41 +988,47 @@ func (bc *BlockChain) Stop() {
log.Error("Failed to journal state snapshot", "err", err)
}
}
if bc.triedb.Scheme() == rawdb.PathScheme {
// Ensure that the in-memory trie nodes are journaled to disk properly.
if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
log.Info("Failed to journal in-memory trie nodes", "err", err)
}
} else {
// Ensure the state of a recent block is also stored to disk before exiting.
// We're writing three different states to catch different restart scenarios:
// - HEAD: So we don't need to reprocess any blocks in the general case
// - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle
// - HEAD-127: So we have a hard limit on the number of blocks reexecuted
if !bc.cacheConfig.TrieDirtyDisabled {
triedb := bc.triedb
// Ensure the state of a recent block is also stored to disk before exiting.
// We're writing three different states to catch different restart scenarios:
// - HEAD: So we don't need to reprocess any blocks in the general case
// - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle
// - HEAD-127: So we have a hard limit on the number of blocks reexecuted
if !bc.cacheConfig.TrieDirtyDisabled {
triedb := bc.triedb
for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
if number := bc.CurrentBlock().Number.Uint64(); number > offset {
recent := bc.GetBlockByNumber(number - offset)
for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
if number := bc.CurrentBlock().Number.Uint64(); number > offset {
recent := bc.GetBlockByNumber(number - offset)
log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
if err := triedb.Commit(recent.Root(), true); err != nil {
log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
if err := triedb.Commit(recent.Root(), true); err != nil {
log.Error("Failed to commit recent state trie", "err", err)
}
}
}
if snapBase != (common.Hash{}) {
log.Info("Writing snapshot state to disk", "root", snapBase)
if err := triedb.Commit(snapBase, true); err != nil {
log.Error("Failed to commit recent state trie", "err", err)
}
}
}
if snapBase != (common.Hash{}) {
log.Info("Writing snapshot state to disk", "root", snapBase)
if err := triedb.Commit(snapBase, true); err != nil {
log.Error("Failed to commit recent state trie", "err", err)
for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem())
}
if size, _ := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup")
}
}
for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem())
}
if size, _ := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup")
}
}
// Flush the collected preimages to disk
if err := bc.stateCache.TrieDB().Close(); err != nil {
log.Error("Failed to close trie db", "err", err)
// Close the trie database, release all the held resources as the last step.
if err := bc.triedb.Close(); err != nil {
log.Error("Failed to close trie database", "err", err)
}
log.Info("Blockchain stopped")
}
@ -1341,6 +1398,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
if err != nil {
return err
}
// If node is running in path mode, skip explicit gc operation
// which is unnecessary in this mode.
if bc.triedb.Scheme() == rawdb.PathScheme {
return nil
}
// If we're running an archive node, always flush
if bc.cacheConfig.TrieDirtyDisabled {
return bc.triedb.Commit(root, false)
@ -1349,8 +1411,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -int64(block.NumberU64()))
current := block.NumberU64()
// Flush limits are not considered for the first TriesInMemory blocks.
current := block.NumberU64()
if current <= TriesInMemory {
return nil
}
@ -1936,6 +1998,12 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
)
parent := it.previous()
for parent != nil && !bc.HasState(parent.Root) {
if bc.stateRecoverable(parent.Root) {
if err := bc.triedb.Recover(parent.Root); err != nil {
return 0, err
}
break
}
hashes = append(hashes, parent.Hash())
numbers = append(numbers, parent.Number.Uint64())
@ -1992,6 +2060,12 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
parent = block
)
for parent != nil && !bc.HasState(parent.Root()) {
if bc.stateRecoverable(parent.Root()) {
if err := bc.triedb.Recover(parent.Root()); err != nil {
return common.Hash{}, err
}
break
}
hashes = append(hashes, parent.Hash())
numbers = append(numbers, parent.NumberU64())
parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
@ -2393,6 +2467,7 @@ func (bc *BlockChain) maintainTxIndex() {
return
}
defer sub.Unsubscribe()
log.Info("Initialized transaction indexer", "limit", bc.TxLookupLimit())
for {
select {

14
core/blockchain_reader.go
View File

@ -293,10 +293,16 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
return bc.HasState(block.Root())
}
// TrieNode retrieves a blob of data associated with a trie node
// either from ephemeral in-memory cache, or from persistent storage.
func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) {
return bc.stateCache.TrieDB().Node(hash)
// stateRecoverable checks if the specified state is recoverable.
// Note, this function assumes the state is not present, because
// state is not treated as recoverable if it's available, thus
// false will be returned in this case.
func (bc *BlockChain) stateRecoverable(root common.Hash) bool {
if bc.triedb.Scheme() == rawdb.HashScheme {
return false
}
result, _ := bc.triedb.Recoverable(root)
return result
}
// ContractCodeWithPrefix retrieves a blob of data associated with a contract

56
core/blockchain_repair_test.go
View File

@ -22,6 +22,7 @@ package core
import (
"math/big"
"path"
"testing"
"time"
@ -1749,16 +1750,23 @@ func testLongReorgedSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
}
func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
testRepairWithScheme(t, tt, snapshots, scheme)
}
}
func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
// It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump(true))
// Create a temporary persistent database
datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: datadir,
AncientsDirectory: ancient,
})
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
@ -1777,6 +1785,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, // Disable snapshot by default
StateScheme: scheme,
}
)
defer engine.Close()
@ -1806,7 +1815,9 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
if tt.commitBlock > 0 {
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil {
t.Fatalf("Failed to flush trie state: %v", err)
}
if snapshots {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
@ -1828,21 +1839,21 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
rawdb.WriteLastPivotNumber(db, *tt.pivotBlock)
}
// Pull the plug on the database, simulating a hard crash
chain.triedb.Close()
db.Close()
chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us
db, err = rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: datadir,
AncientsDirectory: ancient,
})
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
}
defer db.Close()
newChain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil)
newChain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
@ -1885,17 +1896,22 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
// In this case the snapshot layer of B3 is not created because of existent
// state.
func TestIssue23496(t *testing.T) {
testIssue23496(t, rawdb.HashScheme)
testIssue23496(t, rawdb.PathScheme)
}
func testIssue23496(t *testing.T, scheme string) {
// It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// Create a temporary persistent database
datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: datadir,
AncientsDirectory: ancient,
})
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
}
@ -1908,15 +1924,8 @@ func TestIssue23496(t *testing.T) {
BaseFee: big.NewInt(params.InitialBaseFee),
}
engine = ethash.NewFullFaker()
config = &CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256,
SnapshotWait: true,
}
)
chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to create chain: %v", err)
}
@ -1929,7 +1938,7 @@ func TestIssue23496(t *testing.T) {
if _, err := chain.InsertChain(blocks[:1]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
chain.stateCache.TrieDB().Commit(blocks[0].Root(), false)
chain.triedb.Commit(blocks[0].Root(), false)
// Insert block B2 and commit the snapshot into disk
if _, err := chain.InsertChain(blocks[1:2]); err != nil {
@ -1943,7 +1952,7 @@ func TestIssue23496(t *testing.T) {
if _, err := chain.InsertChain(blocks[2:3]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
chain.stateCache.TrieDB().Commit(blocks[2].Root(), false)
chain.triedb.Commit(blocks[2].Root(), false)
// Insert the remaining blocks
if _, err := chain.InsertChain(blocks[3:]); err != nil {
@ -1951,20 +1960,21 @@ func TestIssue23496(t *testing.T) {
}
// Pull the plug on the database, simulating a hard crash
chain.triedb.Close()
db.Close()
chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us
db, err = rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: datadir,
AncientsDirectory: ancient,
})
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
}
defer db.Close()
chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil)
chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
@ -1976,8 +1986,12 @@ func TestIssue23496(t *testing.T) {
if head := chain.CurrentSnapBlock(); head.Number.Uint64() != uint64(4) {
t.Errorf("Head fast block mismatch: have %d, want %d", head.Number, uint64(4))
}
if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(1) {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(1))
expHead := uint64(1)
if scheme == rawdb.PathScheme {
expHead = uint64(2)
}
if head := chain.CurrentBlock(); head.Number.Uint64() != expHead {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead)
}
// Reinsert B2-B4

33
core/blockchain_sethead_test.go
View File

@ -22,6 +22,7 @@ package core
import (
"fmt"
"math/big"
"path"
"strings"
"testing"
"time"
@ -29,9 +30,13 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
// rewindTest is a test case for chain rollback upon user request.
@ -1949,16 +1954,23 @@ func testLongReorgedSnapSyncingDeepSetHead(t *testing.T, snapshots bool) {
}
func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
testSetHeadWithScheme(t, tt, snapshots, scheme)
}
}
func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
// It's hard to follow the test case, visualize the input
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump(false))
// Create a temporary persistent database
datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: datadir,
AncientsDirectory: ancient,
})
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
@ -1977,6 +1989,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, // Disable snapshot
StateScheme: scheme,
}
)
if snapshots {
@ -2007,7 +2020,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
if tt.commitBlock > 0 {
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false)
if snapshots {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
@ -2017,13 +2030,17 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
}
// Manually dereference anything not committed to not have to work with 128+ tries
for _, block := range sideblocks {
chain.stateCache.TrieDB().Dereference(block.Root())
}
for _, block := range canonblocks {
chain.stateCache.TrieDB().Dereference(block.Root())
// Reopen the trie database without persisting in-memory dirty nodes.
chain.triedb.Close()
dbconfig := &trie.Config{}
if scheme == rawdb.PathScheme {
dbconfig.PathDB = pathdb.Defaults
} else {
dbconfig.HashDB = hashdb.Defaults
}
chain.triedb = trie.NewDatabase(chain.db, dbconfig)
chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb)
// Force run a freeze cycle
type freezer interface {
Freeze(threshold uint64) error

260
core/blockchain_snapshot_test.go
View File

@ -24,6 +24,7 @@ import (
"fmt"
"math/big"
"os"
"path"
"strings"
"testing"
"time"
@ -39,6 +40,7 @@ import (
// snapshotTestBasic wraps the common testing fields in the snapshot tests.
type snapshotTestBasic struct {
scheme string // Disk scheme used for storing trie nodes
chainBlocks int // Number of blocks to generate for the canonical chain
snapshotBlock uint64 // Block number of the relevant snapshot disk layer
commitBlock uint64 // Block number for which to commit the state to disk
@ -51,6 +53,7 @@ type snapshotTestBasic struct {
// share fields, set in runtime
datadir string
ancient string
db ethdb.Database
genDb ethdb.Database
engine consensus.Engine
@ -60,10 +63,11 @@ type snapshotTestBasic struct {
func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) {
// Create a temporary persistent database
datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: datadir,
AncientsDirectory: ancient,
})
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
@ -75,13 +79,8 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
Config: params.AllEthashProtocolChanges,
}
engine = ethash.NewFullFaker()
// Snapshot is enabled, the first snapshot is created from the Genesis.
// The snapshot memory allowance is 256MB, it means no snapshot flush
// will happen during the block insertion.
cacheConfig = defaultCacheConfig
)
chain, err := NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil)
chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to create chain: %v", err)
}
@ -102,7 +101,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
startPoint = point
if basic.commitBlock > 0 && basic.commitBlock == point {
chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false)
chain.TrieDB().Commit(blocks[point-1].Root(), false)
}
if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
// Flushing the entire snap tree into the disk, the
@ -121,6 +120,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
// Set runtime fields
basic.datadir = datadir
basic.ancient = ancient
basic.db = db
basic.genDb = genDb
basic.engine = engine
@ -210,6 +210,7 @@ func (basic *snapshotTestBasic) teardown() {
basic.db.Close()
basic.genDb.Close()
os.RemoveAll(basic.datadir)
os.RemoveAll(basic.ancient)
}
// snapshotTest is a test case type for normal snapshot recovery.
@ -226,7 +227,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
// Restart the chain normally
chain.Stop()
newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
@ -235,7 +236,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
snaptest.verify(t, newchain, blocks)
}
// crashSnapshotTest is a test case type for innormal snapshot recovery.
// crashSnapshotTest is a test case type for irregular snapshot recovery.
// It can be used for testing that restart Geth after the crash.
type crashSnapshotTest struct {
snapshotTestBasic
@ -251,13 +252,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
db := chain.db
db.Close()
chain.stopWithoutSaving()
chain.triedb.Close()
// Start a new blockchain back up and see where the repair leads us
newdb, err := rawdb.Open(rawdb.OpenOptions{
Directory: snaptest.datadir,
AncientsDirectory: snaptest.datadir,
AncientsDirectory: snaptest.ancient,
})
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
}
@ -267,13 +268,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
// the crash, we do restart twice here: one after the crash and one
// after the normal stop. It's used to ensure the broken snapshot
// can be detected all the time.
newchain, err := NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
newchain.Stop()
newchain, err = NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
@ -300,7 +301,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
// Insert blocks without enabling snapshot if gapping is required.
chain.Stop()
gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {})
gappedBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {})
// Insert a few more blocks without enabling snapshot
var cacheConfig = &CacheConfig{
@ -308,6 +309,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0,
StateScheme: snaptest.scheme,
}
newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
@ -317,7 +319,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
newchain.Stop()
// Restart the chain with enabling the snapshot
newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
@ -345,7 +347,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) {
chain.SetHead(snaptest.setHead)
chain.Stop()
newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
@ -379,22 +381,24 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0,
StateScheme: snaptest.scheme,
}
newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {})
newBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {})
newchain.InsertChain(newBlocks)
newchain.Stop()
// Restart the chain, the wiper should starts working
// Restart the chain, the wiper should start working
config = &CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256,
SnapshotWait: false, // Don't wait rebuild
StateScheme: snaptest.scheme,
}
tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
@ -402,14 +406,15 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
}
// Simulate the blockchain crash.
tmp.triedb.Close()
tmp.stopWithoutSaving()
newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
defer newchain.Stop()
snaptest.verify(t, newchain, blocks)
newchain.Stop()
}
// Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
@ -433,20 +438,23 @@ func TestRestartWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8
// Expected head block : C8
// Expected snapshot disk : G
test := &snapshotTest{
snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 8,
expSnapshotBottom: 0, // Initial disk layer built from genesis
},
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &snapshotTest{
snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 8,
expSnapshotBottom: 0, // Initial disk layer built from genesis
},
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case the
@ -472,20 +480,23 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : C4
test := &crashSnapshotTest{
snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
},
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &crashSnapshotTest{
snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
},
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case the
@ -511,20 +522,23 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8
// Expected head block : C2
// Expected snapshot disk : C4
test := &crashSnapshotTest{
snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 2,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 2,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
},
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &crashSnapshotTest{
snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 2,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 2,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
},
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case
@ -550,20 +564,27 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : C4
test := &crashSnapshotTest{
snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 6,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
},
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
expHead := uint64(0)
if scheme == rawdb.PathScheme {
expHead = uint64(4)
}
test := &crashSnapshotTest{
snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 6,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: expHead,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
},
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}
// Tests a Geth was running with snapshot enabled. Then restarts without
@ -587,21 +608,24 @@ func TestGappedNewSnapshot(t *testing.T) {
// Expected head fast block: C10
// Expected head block : C10
// Expected snapshot disk : C10
test := &gappedSnapshotTest{
snapshotTestBasic: snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 10,
expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD
},
gapped: 2,
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &gappedSnapshotTest{
snapshotTestBasic: snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 10,
expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD
},
gapped: 2,
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}
// Tests the Geth was running with snapshot enabled and resetHead is applied.
@ -625,21 +649,24 @@ func TestSetHeadWithNewSnapshot(t *testing.T) {
// Expected head fast block: C4
// Expected head block : C4
// Expected snapshot disk : G
test := &setHeadSnapshotTest{
snapshotTestBasic: snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 4,
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
expSnapshotBottom: 0, // The initial disk layer is built from the genesis
},
setHead: 4,
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &setHeadSnapshotTest{
snapshotTestBasic: snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 4,
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
expSnapshotBottom: 0, // The initial disk layer is built from the genesis
},
setHead: 4,
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}
// Tests the Geth was running with a complete snapshot and then imports a few
@ -663,19 +690,22 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
// Expected head fast block: C10
// Expected head block : C8
// Expected snapshot disk : C10
test := &wipeCrashSnapshotTest{
snapshotTestBasic: snapshotTestBasic{
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 10,
expSnapshotBottom: 10,
},
newBlocks: 2,
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &wipeCrashSnapshotTest{
snapshotTestBasic: snapshotTestBasic{
scheme: scheme,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 10,
expSnapshotBottom: 10,
},
newBlocks: 2,
}
test.test(t)
test.teardown()
}
test.test(t)
test.teardown()
}

701
core/blockchain_test.go
View File

File diff suppressed because it is too large Load Diff

16
core/chain_makers.go
View File

@ -283,7 +283,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
}
blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
chainreader := &fakeChainReader{config: config}
genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) {
b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
b.header = makeHeader(chainreader, parent, statedb, b.engine)
@ -326,19 +326,23 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
if err != nil {
panic(fmt.Sprintf("state write error: %v", err))
}
if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
if err = triedb.Commit(root, false); err != nil {
panic(fmt.Sprintf("trie write error: %v", err))
}
return block, b.receipts
}
return nil, nil
}
// Forcibly use hash-based state scheme for retaining all nodes in disk.
triedb := trie.NewDatabase(db, trie.HashDefaults)
defer triedb.Close()
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil)
statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil)
if err != nil {
panic(err)
}
block, receipt := genblock(i, parent, statedb)
block, receipt := genblock(i, parent, triedb, statedb)
blocks[i] = block
receipts[i] = receipt
parent = block
@ -351,7 +355,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
// then generate chain on top.
func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) {
db := rawdb.NewMemoryDatabase()
_, err := genesis.Commit(db, trie.NewDatabase(db))
triedb := trie.NewDatabase(db, trie.HashDefaults)
defer triedb.Close()
_, err := genesis.Commit(db, triedb)
if err != nil {
panic(err)
}

11
core/chain_makers_test.go
View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
func TestGenerateWithdrawalChain(t *testing.T) {
@ -74,8 +75,7 @@ func TestGenerateWithdrawalChain(t *testing.T) {
Storage: storage,
Code: common.Hex2Bytes("600154600354"),
}
genesis := gspec.MustCommit(gendb)
genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults))
chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) {
tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key)
@ -146,6 +146,7 @@ func ExampleGenerateChain() {
addr2 = crypto.PubkeyToAddress(key2.PublicKey)
addr3 = crypto.PubkeyToAddress(key3.PublicKey)
db = rawdb.NewMemoryDatabase()
genDb = rawdb.NewMemoryDatabase()
)
// Ensure that key1 has some funds in the genesis block.
@ -153,13 +154,13 @@ func ExampleGenerateChain() {
Config: &params.ChainConfig{HomesteadBlock: new(big.Int)},
Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}},
}
genesis := gspec.MustCommit(db)
genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults))
// This call generates a chain of 5 blocks. The function runs for
// each block and adds different features to gen based on the
// block index.
signer := types.HomesteadSigner{}
chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 5, func(i int, gen *BlockGen) {
chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), genDb, 5, func(i int, gen *BlockGen) {
switch i {
case 0:
// In block 1, addr1 sends addr2 some ether.
@ -188,7 +189,7 @@ func ExampleGenerateChain() {
})
// Import the chain. This runs all block validation rules.
blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
defer blockchain.Stop()
if i, err := blockchain.InsertChain(chain); err != nil {

8
core/dao_test.go
View File

@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
}
bc.Stop()
@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
}
bc.Stop()
@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&proConf, conBc.GetBlockByHash(conBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})
@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&conConf, proBc.GetBlockByHash(proBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})

14
core/genesis.go
View File

@ -323,10 +323,12 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen
applyOverrides(genesis.Config)
return genesis.Config, block.Hash(), nil
}
// We have the genesis block in database(perhaps in ancient database)
// but the corresponding state is missing.
// The genesis block is present(perhaps in ancient database) while the
// state database is not initialized yet. It can happen that the node
// is initialized with an external ancient store. Commit genesis state
// in this case.
header := rawdb.ReadHeader(db, stored, 0)
if header.Root != types.EmptyRootHash && !rawdb.HasLegacyTrieNode(db, header.Root) {
if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) {
if genesis == nil {
genesis = DefaultGenesisBlock()
}
@ -526,10 +528,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block
// MustCommit writes the genesis block and state to db, panicking on error.
// The block is committed as the canonical head block.
// Note the state changes will be committed in hash-based scheme, use Commit
// if path-scheme is preferred.
func (g *Genesis) MustCommit(db ethdb.Database) *types.Block {
block, err := g.Commit(db, trie.NewDatabase(db))
func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block {
block, err := g.Commit(db, triedb)
if err != nil {
panic(err)
}

53
core/genesis_test.go
View File

@ -30,18 +30,24 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
func TestInvalidCliqueConfig(t *testing.T) {
block := DefaultGoerliGenesisBlock()
block.ExtraData = []byte{}
db := rawdb.NewMemoryDatabase()
if _, err := block.Commit(db, trie.NewDatabase(db)); err == nil {
if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil {
t.Fatal("Expected error on invalid clique config")
}
}
func TestSetupGenesis(t *testing.T) {
testSetupGenesis(t, rawdb.HashScheme)
testSetupGenesis(t, rawdb.PathScheme)
}
func testSetupGenesis(t *testing.T, scheme string) {
var (
customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50")
customg = Genesis{
@ -53,6 +59,7 @@ func TestSetupGenesis(t *testing.T) {
oldcustomg = customg
)
oldcustomg.Config = &params.ChainConfig{HomesteadBlock: big.NewInt(2)}
tests := []struct {
name string
fn func(ethdb.Database) (*params.ChainConfig, common.Hash, error)
@ -63,7 +70,7 @@ func TestSetupGenesis(t *testing.T) {
{
name: "genesis without ChainConfig",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
return SetupGenesisBlock(db, trie.NewDatabase(db), new(Genesis))
return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis))
},
wantErr: errGenesisNoConfig,
wantConfig: params.AllEthashProtocolChanges,
@ -71,7 +78,7 @@ func TestSetupGenesis(t *testing.T) {
{
name: "no block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
return SetupGenesisBlock(db, trie.NewDatabase(db), nil)
return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil)
},
wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig,
@ -79,8 +86,8 @@ func TestSetupGenesis(t *testing.T) {
{
name: "mainnet block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
DefaultGenesisBlock().MustCommit(db)
return SetupGenesisBlock(db, trie.NewDatabase(db), nil)
DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, newDbConfig(scheme)))
return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil)
},
wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig,
@ -88,8 +95,9 @@ func TestSetupGenesis(t *testing.T) {
{
name: "custom block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
customg.MustCommit(db)
return SetupGenesisBlock(db, trie.NewDatabase(db), nil)
tdb := trie.NewDatabase(db, newDbConfig(scheme))
customg.Commit(db, tdb)
return SetupGenesisBlock(db, tdb, nil)
},
wantHash: customghash,
wantConfig: customg.Config,
@ -97,8 +105,9 @@ func TestSetupGenesis(t *testing.T) {
{
name: "custom block in DB, genesis == goerli",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
customg.MustCommit(db)
return SetupGenesisBlock(db, trie.NewDatabase(db), DefaultGoerliGenesisBlock())
tdb := trie.NewDatabase(db, newDbConfig(scheme))
customg.Commit(db, tdb)
return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock())
},
wantErr: &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash},
wantHash: params.GoerliGenesisHash,
@ -107,8 +116,9 @@ func TestSetupGenesis(t *testing.T) {
{
name: "compatible config in DB",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
oldcustomg.MustCommit(db)
return SetupGenesisBlock(db, trie.NewDatabase(db), &customg)
tdb := trie.NewDatabase(db, newDbConfig(scheme))
oldcustomg.Commit(db, tdb)
return SetupGenesisBlock(db, tdb, &customg)
},
wantHash: customghash,
wantConfig: customg.Config,
@ -118,16 +128,17 @@ func TestSetupGenesis(t *testing.T) {
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
// Commit the 'old' genesis block with Homestead transition at #2.
// Advance to block #4, past the homestead transition block of customg.
genesis := oldcustomg.MustCommit(db)
tdb := trie.NewDatabase(db, newDbConfig(scheme))
oldcustomg.Commit(db, tdb)
bc, _ := NewBlockChain(db, nil, &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil)
bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil)
defer bc.Stop()
blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil)
_, blocks, _ := GenerateChainWithGenesis(&oldcustomg, ethash.NewFaker(), 4, nil)
bc.InsertChain(blocks)
// This should return a compatibility error.
return SetupGenesisBlock(db, trie.NewDatabase(db), &customg)
return SetupGenesisBlock(db, tdb, &customg)
},
wantHash: customghash,
wantConfig: customg.Config,
@ -175,7 +186,8 @@ func TestGenesisHashes(t *testing.T) {
{DefaultSepoliaGenesisBlock(), params.SepoliaGenesisHash},
} {
// Test via MustCommit
if have := c.genesis.MustCommit(rawdb.NewMemoryDatabase()).Hash(); have != c.want {
db := rawdb.NewMemoryDatabase()
if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want {
t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex())
}
// Test via ToBlock
@ -193,7 +205,7 @@ func TestGenesis_Commit(t *testing.T) {
}
db := rawdb.NewMemoryDatabase()
genesisBlock := genesis.MustCommit(db)
genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
if genesis.Difficulty != nil {
t.Fatalf("assumption wrong")
@ -242,3 +254,10 @@ func TestReadWriteGenesisAlloc(t *testing.T) {
}
}
}
func newDbConfig(scheme string) *trie.Config {
if scheme == rawdb.HashScheme {
return trie.HashDefaults
}
return &trie.Config{PathDB: pathdb.Defaults}
}

2
core/headerchain_test.go
View File

@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) {
db = rawdb.NewMemoryDatabase()
gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges}
)
gspec.Commit(db, trie.NewDatabase(db))
gspec.Commit(db, trie.NewDatabase(db, nil))
hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false })
if err != nil {
t.Fatal(err)

2
core/rawdb/accessors_chain_test.go
View File

@ -435,12 +435,12 @@ func checkReceiptsRLP(have, want types.Receipts) error {
func TestAncientStorage(t *testing.T) {
// Freezer style fast import the chain.
frdir := t.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
defer db.Close()
// Create a test block
block := types.NewBlockWithHeader(&types.Header{
Number: big.NewInt(0),

26
core/rawdb/accessors_trie.go
View File

@ -36,7 +36,7 @@ import (
//
// Now this scheme is still kept for backward compatibility, and it will be used
// for archive node and some other tries(e.g. light trie).
const HashScheme = "hashScheme"
const HashScheme = "hash"
// PathScheme is the new path-based state scheme with which trie nodes are stored
// in the disk with node path as the database key. This scheme will only store one
@ -44,7 +44,7 @@ const HashScheme = "hashScheme"
// is native. At the same time, this scheme will put adjacent trie nodes in the same
// area of the disk with good data locality property. But this scheme needs to rely
// on extra state diffs to survive deep reorg.
const PathScheme = "pathScheme"
const PathScheme = "path"
// hasher is used to compute the sha256 hash of the provided data.
type hasher struct{ sha crypto.KeccakState }
@ -263,3 +263,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
}
// ReadStateScheme reads the state scheme of persistent state, or none
// if the state is not present in database.
func ReadStateScheme(db ethdb.Reader) string {
// Check if state in path-based scheme is present
blob, _ := ReadAccountTrieNode(db, nil)
if len(blob) != 0 {
return PathScheme
}
// In a hash-based scheme, the genesis state is consistently stored
// on the disk. To assess the scheme of the persistent state, it
// suffices to inspect the scheme of the genesis state.
header := ReadHeader(db, ReadCanonicalHash(db, 0), 0)
if header == nil {
return "" // empty datadir
}
blob = ReadLegacyTrieNode(db, header.Root)
if len(blob) == 0 {
return "" // no state in disk
}
return HashScheme
}

8
core/rawdb/ancient_scheme.go
View File

@ -58,7 +58,7 @@ const (
stateHistoryStorageData = "storage.data"
)
var stateHistoryFreezerNoSnappy = map[string]bool{
var stateFreezerNoSnappy = map[string]bool{
stateHistoryMeta: true,
stateHistoryAccountIndex: false,
stateHistoryStorageIndex: false,
@ -75,7 +75,7 @@ var (
// freezers the collections of all builtin freezers.
var freezers = []string{chainFreezerName, stateFreezerName}
// NewStateHistoryFreezer initializes the freezer for state history.
func NewStateHistoryFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateHistoryFreezerNoSnappy)
// NewStateFreezer initializes the freezer for state history.
func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
}

56
core/rawdb/ancient_utils.go
View File

@ -50,36 +50,58 @@ func (info *freezerInfo) size() common.StorageSize {
return total
}
func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
info := freezerInfo{name: name}
for t := range order {
size, err := reader.AncientSize(t)
if err != nil {
return freezerInfo{}, err
}
info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := reader.Ancients()
if err != nil {
return freezerInfo{}, err
}
info.head = ancients - 1
// Retrieve the number of first stored item
tail, err := reader.Tail()
if err != nil {
return freezerInfo{}, err
}
info.tail = tail
return info, nil
}
// inspectFreezers inspects all freezers registered in the system.
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
var infos []freezerInfo
for _, freezer := range freezers {
switch freezer {
case chainFreezerName:
// Chain ancient store is a bit special. It's always opened along
// with the key-value store, inspect the chain store directly.
info := freezerInfo{name: freezer}
// Retrieve storage size of every contained table.
for table := range chainFreezerNoSnappy {
size, err := db.AncientSize(table)
if err != nil {
return nil, err
}
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := db.Ancients()
info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db)
if err != nil {
return nil, err
}
info.head = ancients - 1
infos = append(infos, info)
// Retrieve the number of first stored item
tail, err := db.Tail()
case stateFreezerName:
datadir, err := db.AncientDatadir()
if err != nil {
return nil, err
}
f, err := NewStateFreezer(datadir, true)
if err != nil {
return nil, err
}
defer f.Close()
info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f)
if err != nil {
return nil, err
}
info.tail = tail
infos = append(infos, info)
default:

21
core/rawdb/database.go
View File

@ -463,7 +463,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
tds stat
numHashPairings stat
hashNumPairings stat
tries stat
legacyTries stat
stateLookups stat
accountTries stat
storageTries stat
codes stat
txLookups stat
accountSnaps stat
@ -504,8 +507,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
numHashPairings.Add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
hashNumPairings.Add(size)
case len(key) == common.HashLength:
tries.Add(size)
case IsLegacyTrieNode(key, it.Value()):
legacyTries.Add(size)
case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
stateLookups.Add(size)
case IsAccountTrieNode(key):
accountTries.Add(size)
case IsStorageTrieNode(key):
storageTries.Add(size)
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
codes.Add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
@ -543,6 +552,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey,
} {
if bytes.Equal(key, meta) {
metadata.Add(size)
@ -571,7 +581,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},
{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},

26
core/rawdb/schema.go
View File

@ -273,9 +273,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
return bytes.Equal(key, crypto.Keccak256(val))
}
// IsAccountTrieNode reports whether a provided database entry is an account
// trie node in path-based state scheme.
func IsAccountTrieNode(key []byte) (bool, []byte) {
// ResolveAccountTrieNodeKey reports whether a provided database entry is an
// account trie node in path-based state scheme, and returns the resolved
// node path if so.
func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
return false, nil
}
@ -288,9 +289,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) {
return true, key[len(trieNodeAccountPrefix):]
}
// IsStorageTrieNode reports whether a provided database entry is a storage
// IsAccountTrieNode reports whether a provided database entry is an account
// trie node in path-based state scheme.
func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
func IsAccountTrieNode(key []byte) bool {
ok, _ := ResolveAccountTrieNodeKey(key)
return ok
}
// ResolveStorageTrieNode reports whether a provided database entry is a storage
// trie node in path-based state scheme, and returns the resolved account hash
// and node path if so.
func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
return false, common.Hash{}, nil
}
@ -306,3 +315,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
}
// IsStorageTrieNode reports whether a provided database entry is a storage
// trie node in path-based state scheme.
func IsStorageTrieNode(key []byte) bool {
ok, _, _ := ResolveStorageTrieNode(key)
return ok
}

4
core/state/database.go
View File

@ -58,7 +58,7 @@ type Database interface {
// DiskDB returns the underlying key-value disk database.
DiskDB() ethdb.KeyValueStore
// TrieDB retrieves the low level trie database used for data storage.
// TrieDB returns the underlying trie database for managing trie nodes.
TrieDB() *trie.Database
}
@ -147,7 +147,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
disk: db,
codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
triedb: trie.NewDatabaseWithConfig(db, config),
triedb: trie.NewDatabase(db, config),
}
}

15
core/state/iterator_test.go
View File

@ -26,9 +26,14 @@ import (
// Tests that the node iterator indeed walks over the entire database contents.
func TestNodeIteratorCoverage(t *testing.T) {
testNodeIteratorCoverage(t, rawdb.HashScheme)
testNodeIteratorCoverage(t, rawdb.PathScheme)
}
func testNodeIteratorCoverage(t *testing.T, scheme string) {
// Create some arbitrary test state to iterate
db, sdb, root, _ := makeTestState()
sdb.TrieDB().Commit(root, false)
db, sdb, ndb, root, _ := makeTestState(scheme)
ndb.Commit(root, false)
state, err := New(root, sdb, nil)
if err != nil {
@ -48,7 +53,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
)
it := db.NewIterator(nil, nil)
for it.Next() {
ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value())
ok, hash := isTrieNode(scheme, it.Key(), it.Value())
if !ok {
continue
}
@ -90,11 +95,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) {
return true, common.BytesToHash(key)
}
} else {
ok, _ := rawdb.IsAccountTrieNode(key)
ok := rawdb.IsAccountTrieNode(key)
if ok {
return true, crypto.Keccak256Hash(val)
}
ok, _, _ = rawdb.IsStorageTrieNode(key)
ok = rawdb.IsStorageTrieNode(key)
if ok {
return true, crypto.Keccak256Hash(val)
}

13
core/state/pruner/pruner.go
View File

@ -85,13 +85,16 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
if headBlock == nil {
return nil, errors.New("failed to load head block")
}
// Offline pruning is only supported in legacy hash based scheme.
triedb := trie.NewDatabase(db, trie.HashDefaults)
snapconfig := snapshot.Config{
CacheSize: 256,
Recovery: false,
NoBuild: true,
AsyncBuild: false,
}
snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
if err != nil {
return nil, err // The relevant snapshot(s) might not exist
}
@ -361,7 +364,9 @@ func RecoverPruning(datadir string, db ethdb.Database) error {
NoBuild: true,
AsyncBuild: false,
}
snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
// Offline pruning is only supported in legacy hash based scheme.
triedb := trie.NewDatabase(db, trie.HashDefaults)
snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
if err != nil {
return err // The relevant snapshot(s) might not exist
}
@ -403,7 +408,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil {
return errors.New("missing genesis block")
}
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults))
if err != nil {
return err
}
@ -427,7 +432,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
}
if acc.Root != types.EmptyRootHash {
id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults))
if err != nil {
return err
}

3
core/state/snapshot/generate.go
View File

@ -356,7 +356,8 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
var resolver trie.NodeResolver
if len(result.keys) > 0 {
mdb := rawdb.NewMemoryDatabase()
tdb := trie.NewDatabase(mdb)
tdb := trie.NewDatabase(mdb, trie.HashDefaults)
defer tdb.Close()
snapTrie := trie.NewEmpty(tdb)
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])

142
core/state/snapshot/generate_test.go
View File

@ -30,6 +30,8 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode"
"golang.org/x/crypto/sha3"
)
@ -45,10 +47,15 @@ func hashData(input []byte) common.Hash {
// Tests that snapshot generation from an empty database.
func TestGeneration(t *testing.T) {
testGeneration(t, rawdb.HashScheme)
testGeneration(t, rawdb.PathScheme)
}
func testGeneration(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var helper = newHelper()
var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@ -79,10 +86,15 @@ func TestGeneration(t *testing.T) {
// Tests that snapshot generation with existent flat state.
func TestGenerateExistentState(t *testing.T) {
testGenerateExistentState(t, rawdb.HashScheme)
testGenerateExistentState(t, rawdb.PathScheme)
}
func testGenerateExistentState(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
var helper = newHelper()
var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@ -148,9 +160,15 @@ type testHelper struct {
nodes *trienode.MergedNodeSet
}
func newHelper() *testHelper {
func newHelper(scheme string) *testHelper {
diskdb := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(diskdb)
config := &trie.Config{}
if scheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{} // disable caching
} else {
config.HashDB = &hashdb.Config{} // disable caching
}
triedb := trie.NewDatabase(diskdb, config)
accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb)
return &testHelper{
diskdb: diskdb,
@ -233,7 +251,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
// - extra slots in the middle
// - extra slots in the end
func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
helper := newHelper()
testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme)
testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme)
}
func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) {
helper := newHelper(scheme)
// Account one, empty root but non-empty database
helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
@ -325,7 +348,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// - wrong accounts
// - extra accounts
func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper := newHelper()
testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme)
testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme)
}
func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) {
helper := newHelper(scheme)
helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@ -380,10 +408,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
// Tests that snapshot generation errors out correctly in case of a missing trie
// node in the account trie.
func TestGenerateCorruptAccountTrie(t *testing.T) {
testGenerateCorruptAccountTrie(t, rawdb.HashScheme)
testGenerateCorruptAccountTrie(t, rawdb.PathScheme)
}
func testGenerateCorruptAccountTrie(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// without any storage slots to keep the test smaller.
helper := newHelper()
helper := newHelper(scheme)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@ -391,9 +424,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
// Delete an account trie leaf and ensure the generator chokes
helper.triedb.Commit(root, false)
helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
// Delete an account trie node and ensure the generator chokes
targetPath := []byte{0xc}
targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7")
rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
@ -414,11 +449,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
// trie node for a storage trie. It's similar to internal corruption but it is
// handled differently inside the generator.
func TestGenerateMissingStorageTrie(t *testing.T) {
testGenerateMissingStorageTrie(t, rawdb.HashScheme)
testGenerateMissingStorageTrie(t, rawdb.PathScheme)
}
func testGenerateMissingStorageTrie(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
helper := newHelper()
var (
acc1 = hashData([]byte("acc-1"))
acc3 = hashData([]byte("acc-3"))
helper = newHelper(scheme)
)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@ -427,8 +470,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
root := helper.Commit()
// Delete a storage trie root and ensure the generator chokes
helper.diskdb.Delete(stRoot.Bytes())
// Delete storage trie root of account one and three.
rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme)
rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
@ -448,10 +492,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
// Tests that snapshot generation errors out correctly in case of a missing trie
// node in a storage trie.
func TestGenerateCorruptStorageTrie(t *testing.T) {
testGenerateCorruptStorageTrie(t, rawdb.HashScheme)
testGenerateCorruptStorageTrie(t, rawdb.PathScheme)
}
func testGenerateCorruptStorageTrie(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
helper := newHelper()
helper := newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
@ -461,8 +510,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
root := helper.Commit()
// Delete a storage trie leaf and ensure the generator chokes
helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
// Delete a node in the storage trie.
targetPath := []byte{0x4}
targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371")
rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme)
rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
@ -481,7 +533,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
// Tests that snapshot generation when an extra account with storage exists in the snap state.
func TestGenerateWithExtraAccounts(t *testing.T) {
helper := newHelper()
testGenerateWithExtraAccounts(t, rawdb.HashScheme)
testGenerateWithExtraAccounts(t, rawdb.PathScheme)
}
func testGenerateWithExtraAccounts(t *testing.T, scheme string) {
helper := newHelper(scheme)
{
// Account one in the trie
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@ -549,10 +606,15 @@ func enableLogging() {
// Tests that snapshot generation when an extra account with storage exists in the snap state.
func TestGenerateWithManyExtraAccounts(t *testing.T) {
testGenerateWithManyExtraAccounts(t, rawdb.HashScheme)
testGenerateWithManyExtraAccounts(t, rawdb.PathScheme)
}
func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
if false {
enableLogging()
}
helper := newHelper()
helper := newHelper(scheme)
{
// Account one in the trie
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@ -605,11 +667,16 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
// So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted.
// But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished.
func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme)
testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme)
}
func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) {
accountCheckRange = 3
if false {
enableLogging()
}
helper := newHelper()
helper := newHelper(scheme)
{
acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
@ -642,11 +709,16 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
// TestGenerateWithMalformedSnapdata tests what happes if we have some junk
// in the snapshot database, which cannot be parsed back to an account
func TestGenerateWithMalformedSnapdata(t *testing.T) {
testGenerateWithMalformedSnapdata(t, rawdb.HashScheme)
testGenerateWithMalformedSnapdata(t, rawdb.PathScheme)
}
func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) {
accountCheckRange = 3
if false {
enableLogging()
}
helper := newHelper()
helper := newHelper(scheme)
{
acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
@ -679,10 +751,15 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
}
func TestGenerateFromEmptySnap(t *testing.T) {
testGenerateFromEmptySnap(t, rawdb.HashScheme)
testGenerateFromEmptySnap(t, rawdb.PathScheme)
}
func testGenerateFromEmptySnap(t *testing.T, scheme string) {
//enableLogging()
accountCheckRange = 10
storageCheckRange = 20
helper := newHelper()
helper := newHelper(scheme)
// Add 1K accounts to the trie
for i := 0; i < 400; i++ {
stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@ -714,8 +791,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
// This hits a case where the snap verification passes, but there are more elements in the trie
// which we must also add.
func TestGenerateWithIncompleteStorage(t *testing.T) {
testGenerateWithIncompleteStorage(t, rawdb.HashScheme)
testGenerateWithIncompleteStorage(t, rawdb.PathScheme)
}
func testGenerateWithIncompleteStorage(t *testing.T, scheme string) {
storageCheckRange = 4
helper := newHelper()
helper := newHelper(scheme)
stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
// We add 8 accounts, each one is missing exactly one of the storage slots. This means
@ -813,7 +895,12 @@ func populateDangling(disk ethdb.KeyValueStore) {
//
// This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper()
testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme)
testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme)
}
func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) {
var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@ -848,7 +935,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
//
// This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper()
testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme)
testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme)
}
func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) {
var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})

2
core/state/statedb_fuzz_test.go
View File

@ -179,7 +179,7 @@ func (test *stateTest) run() bool {
storageList = append(storageList, copy2DSet(states.Storages))
}
disk = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit})
tdb = trie.NewDatabase(disk, &trie.Config{OnCommit: onCommit})
sdb = NewDatabaseWithNodeDB(disk, tdb)
byzantium = rand.Intn(2) == 0
)

57
core/state/statedb_test.go
View File

@ -36,14 +36,19 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
// Tests that updating a state trie does not leak any database writes prior to
// actually committing the state.
func TestUpdateLeaks(t *testing.T) {
// Create an empty state database
db := rawdb.NewMemoryDatabase()
state, _ := New(types.EmptyRootHash, NewDatabase(db), nil)
var (
db = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabase(db, nil)
)
state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil)
// Update it with some accounts
for i := byte(0); i < 255; i++ {
@ -59,7 +64,7 @@ func TestUpdateLeaks(t *testing.T) {
}
root := state.IntermediateRoot(false)
if err := state.Database().TrieDB().Commit(root, false); err != nil {
if err := tdb.Commit(root, false); err != nil {
t.Errorf("can not commit trie %v to persistent database", root.Hex())
}
@ -77,8 +82,10 @@ func TestIntermediateLeaks(t *testing.T) {
// Create two state databases, one transitioning to the final state, the other final from the beginning
transDb := rawdb.NewMemoryDatabase()
finalDb := rawdb.NewMemoryDatabase()
transState, _ := New(types.EmptyRootHash, NewDatabase(transDb), nil)
finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb), nil)
transNdb := trie.NewDatabase(transDb, nil)
finalNdb := trie.NewDatabase(finalDb, nil)
transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil)
finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil)
modify := func(state *StateDB, addr common.Address, i, tweak byte) {
state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
@ -110,7 +117,7 @@ func TestIntermediateLeaks(t *testing.T) {
if err != nil {
t.Fatalf("failed to commit transition state: %v", err)
}
if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil {
if err = transNdb.Commit(transRoot, false); err != nil {
t.Errorf("can not commit trie %v to persistent database", transRoot.Hex())
}
@ -118,7 +125,7 @@ func TestIntermediateLeaks(t *testing.T) {
if err != nil {
t.Fatalf("failed to commit final state: %v", err)
}
if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil {
if err = finalNdb.Commit(finalRoot, false); err != nil {
t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex())
}
@ -747,9 +754,28 @@ func TestDeleteCreateRevert(t *testing.T) {
// the Commit operation fails with an error
// If we are missing trie nodes, we should not continue writing to the trie
func TestMissingTrieNodes(t *testing.T) {
testMissingTrieNodes(t, rawdb.HashScheme)
testMissingTrieNodes(t, rawdb.PathScheme)
}
func testMissingTrieNodes(t *testing.T, scheme string) {
// Create an initial state with a few accounts
memDb := rawdb.NewMemoryDatabase()
db := NewDatabase(memDb)
var (
triedb *trie.Database
memDb = rawdb.NewMemoryDatabase()
)
if scheme == rawdb.PathScheme {
triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{
CleanCacheSize: 0,
DirtyCacheSize: 0,
}}) // disable caching
} else {
triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{
CleanCacheSize: 0,
}}) // disable caching
}
db := NewDatabaseWithNodeDB(memDb, triedb)
var root common.Hash
state, _ := New(types.EmptyRootHash, db, nil)
addr := common.BytesToAddress([]byte("so"))
@ -762,7 +788,7 @@ func TestMissingTrieNodes(t *testing.T) {
root, _ = state.Commit(0, false)
t.Logf("root: %x", root)
// force-flush
state.Database().TrieDB().Cap(0)
triedb.Commit(root, false)
}
// Create a new state on the old root
state, _ = New(root, db, nil)
@ -969,7 +995,8 @@ func TestFlushOrderDataLoss(t *testing.T) {
// Create a state trie with many accounts and slots
var (
memdb = rawdb.NewMemoryDatabase()
statedb = NewDatabase(memdb)
triedb = trie.NewDatabase(memdb, nil)
statedb = NewDatabaseWithNodeDB(memdb, triedb)
state, _ = New(types.EmptyRootHash, statedb, nil)
)
for a := byte(0); a < 10; a++ {
@ -982,11 +1009,11 @@ func TestFlushOrderDataLoss(t *testing.T) {
if err != nil {
t.Fatalf("failed to commit state trie: %v", err)
}
statedb.TrieDB().Reference(root, common.Hash{})
if err := statedb.TrieDB().Cap(1024); err != nil {
triedb.Reference(root, common.Hash{})
if err := triedb.Cap(1024); err != nil {
t.Fatalf("failed to cap trie dirty cache: %v", err)
}
if err := statedb.TrieDB().Commit(root, false); err != nil {
if err := triedb.Commit(root, false); err != nil {
t.Fatalf("failed to commit state trie: %v", err)
}
// Reopen the state trie from flushed disk and verify it
@ -1040,7 +1067,7 @@ func TestStateDBTransientStorage(t *testing.T) {
func TestResetObject(t *testing.T) {
var (
disk = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabase(disk)
tdb = trie.NewDatabase(disk, nil)
db = NewDatabaseWithNodeDB(disk, tdb)
snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash)
state, _ = New(types.EmptyRootHash, db, snaps)

169
core/state/sync_test.go
View File

@ -28,6 +28,8 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
// testAccount is the data associated with an account used by the state tests.
@ -39,10 +41,17 @@ type testAccount struct {
}
// makeTestState create a sample test state to test node-wise reconstruction.
func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) {
// Create an empty state
config := &trie.Config{Preimages: true}
if scheme == rawdb.PathScheme {
config.PathDB = pathdb.Defaults
} else {
config.HashDB = hashdb.Defaults
}
db := rawdb.NewMemoryDatabase()
sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true})
nodeDb := trie.NewDatabase(db, config)
sdb := NewDatabaseWithNodeDB(db, nodeDb)
state, _ := New(types.EmptyRootHash, sdb, nil)
// Fill it with some arbitrary data
@ -67,24 +76,27 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
obj.SetState(hash, hash)
}
}
state.updateStateObject(obj)
accounts = append(accounts, acc)
}
root, _ := state.Commit(0, false)
// Return the generated state
return db, sdb, root, accounts
return db, sdb, nodeDb, root, accounts
}
// checkStateAccounts cross references a reconstructed state with an expected
// account array.
func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) {
var config trie.Config
if scheme == rawdb.PathScheme {
config.PathDB = pathdb.Defaults
}
// Check root availability and state contents
state, err := New(root, NewDatabase(db), nil)
state, err := New(root, NewDatabaseWithConfig(db, &config), nil)
if err != nil {
t.Fatalf("failed to create state trie at %x: %v", root, err)
}
if err := checkStateConsistency(db, root); err != nil {
if err := checkStateConsistency(db, scheme, root); err != nil {
t.Fatalf("inconsistent state trie at %x: %v", root, err)
}
for i, acc := range accounts {
@ -101,8 +113,12 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou
}
// checkStateConsistency checks that all data of a state root is present.
func checkStateConsistency(db ethdb.Database, root common.Hash) error {
state, err := New(root, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil)
func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error {
config := &trie.Config{Preimages: true}
if scheme == rawdb.PathScheme {
config.PathDB = pathdb.Defaults
}
state, err := New(root, NewDatabaseWithConfig(db, config), nil)
if err != nil {
return err
}
@ -114,8 +130,14 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
// Tests that an empty state is not scheduled for syncing.
func TestEmptyStateSync(t *testing.T) {
db := trie.NewDatabase(rawdb.NewMemoryDatabase())
sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, db.Scheme())
dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)
dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults})
sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbA.Scheme())
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
}
sync = NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbB.Scheme())
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
}
@ -124,22 +146,28 @@ func TestEmptyStateSync(t *testing.T) {
// Tests that given a root hash, a state can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go.
func TestIterativeStateSyncIndividual(t *testing.T) {
testIterativeStateSync(t, 1, false, false)
testIterativeStateSync(t, 1, false, false, rawdb.HashScheme)
testIterativeStateSync(t, 1, false, false, rawdb.PathScheme)
}
func TestIterativeStateSyncBatched(t *testing.T) {
testIterativeStateSync(t, 100, false, false)
testIterativeStateSync(t, 100, false, false, rawdb.HashScheme)
testIterativeStateSync(t, 100, false, false, rawdb.PathScheme)
}
func TestIterativeStateSyncIndividualFromDisk(t *testing.T) {
testIterativeStateSync(t, 1, true, false)
testIterativeStateSync(t, 1, true, false, rawdb.HashScheme)
testIterativeStateSync(t, 1, true, false, rawdb.PathScheme)
}
func TestIterativeStateSyncBatchedFromDisk(t *testing.T) {
testIterativeStateSync(t, 100, true, false)
testIterativeStateSync(t, 100, true, false, rawdb.HashScheme)
testIterativeStateSync(t, 100, true, false, rawdb.PathScheme)
}
func TestIterativeStateSyncIndividualByPath(t *testing.T) {
testIterativeStateSync(t, 1, false, true)
testIterativeStateSync(t, 1, false, true, rawdb.HashScheme)
testIterativeStateSync(t, 1, false, true, rawdb.PathScheme)
}
func TestIterativeStateSyncBatchedByPath(t *testing.T) {
testIterativeStateSync(t, 100, false, true)
testIterativeStateSync(t, 100, false, true, rawdb.HashScheme)
testIterativeStateSync(t, 100, false, true, rawdb.PathScheme)
}
// stateElement represents the element in the state trie(bytecode or trie node).
@ -150,17 +178,17 @@ type stateElement struct {
syncPath trie.SyncPath
}
func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, scheme string) {
// Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
if commit {
srcDb.TrieDB().Commit(srcRoot, false)
ndb.Commit(srcRoot, false)
}
srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB())
srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), ndb)
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
var (
nodeElements []stateElement
@ -175,9 +203,11 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
})
}
for i := 0; i < len(codes); i++ {
codeElements = append(codeElements, stateElement{
code: codes[i],
})
codeElements = append(codeElements, stateElement{code: codes[i]})
}
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
}
for len(nodeElements)+len(codeElements) > 0 {
var (
@ -205,7 +235,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err)
}
id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root)
stTrie, err := trie.New(id, srcDb.TrieDB())
stTrie, err := trie.New(id, ndb)
if err != nil {
t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err)
}
@ -216,7 +246,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data}
}
} else {
data, err := srcDb.TrieDB().Node(node.hash)
owner, inner := trie.ResolvePath([]byte(node.path))
data, err := reader.Node(owner, inner, node.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for key %v", []byte(node.path))
}
@ -260,18 +291,23 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned, and the others sent only later.
func TestIterativeDelayedStateSync(t *testing.T) {
testIterativeDelayedStateSync(t, rawdb.HashScheme)
testIterativeDelayedStateSync(t, rawdb.PathScheme)
}
func testIterativeDelayedStateSync(t *testing.T, scheme string) {
// Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
var (
nodeElements []stateElement
@ -286,9 +322,11 @@ func TestIterativeDelayedStateSync(t *testing.T) {
})
}
for i := 0; i < len(codes); i++ {
codeElements = append(codeElements, stateElement{
code: codes[i],
})
codeElements = append(codeElements, stateElement{code: codes[i]})
}
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
}
for len(nodeElements)+len(codeElements) > 0 {
// Sync only half of the scheduled nodes
@ -313,7 +351,8 @@ func TestIterativeDelayedStateSync(t *testing.T) {
if len(nodeElements) > 0 {
nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1)
for i, element := range nodeElements[:len(nodeResults)] {
data, err := srcDb.TrieDB().Node(element.hash)
owner, inner := trie.ResolvePath([]byte(element.path))
data, err := reader.Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve contract bytecode for %x", element.code)
}
@ -353,22 +392,28 @@ func TestIterativeDelayedStateSync(t *testing.T) {
copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
}
// Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go, however in a
// random order.
func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) }
func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) }
func TestIterativeRandomStateSyncIndividual(t *testing.T) {
testIterativeRandomStateSync(t, 1, rawdb.HashScheme)
testIterativeRandomStateSync(t, 1, rawdb.PathScheme)
}
func TestIterativeRandomStateSyncBatched(t *testing.T) {
testIterativeRandomStateSync(t, 100, rawdb.HashScheme)
testIterativeRandomStateSync(t, 100, rawdb.PathScheme)
}
func testIterativeRandomStateSync(t *testing.T, count int) {
func testIterativeRandomStateSync(t *testing.T, count int, scheme string) {
// Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
nodeQueue := make(map[string]stateElement)
codeQueue := make(map[common.Hash]struct{})
@ -383,6 +428,10 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
for _, hash := range codes {
codeQueue[hash] = struct{}{}
}
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
}
for len(nodeQueue)+len(codeQueue) > 0 {
// Fetch all the queued nodes in a random order
if len(codeQueue) > 0 {
@ -403,7 +452,8 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
if len(nodeQueue) > 0 {
results := make([]trie.NodeSyncResult, 0, len(nodeQueue))
for path, element := range nodeQueue {
data, err := srcDb.TrieDB().Node(element.hash)
owner, inner := trie.ResolvePath([]byte(element.path))
data, err := reader.Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path)
}
@ -415,7 +465,6 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
}
}
}
// Feed the retrieved results back and queue new tasks
batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil {
t.Fatalf("failed to commit data: %v", err)
@ -441,18 +490,23 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned (Even those randomly), others sent only later.
func TestIterativeRandomDelayedStateSync(t *testing.T) {
testIterativeRandomDelayedStateSync(t, rawdb.HashScheme)
testIterativeRandomDelayedStateSync(t, rawdb.PathScheme)
}
func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) {
// Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
nodeQueue := make(map[string]stateElement)
codeQueue := make(map[common.Hash]struct{})
@ -467,6 +521,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
for _, hash := range codes {
codeQueue[hash] = struct{}{}
}
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
}
for len(nodeQueue)+len(codeQueue) > 0 {
// Sync only half of the scheduled nodes, even those in random order
if len(codeQueue) > 0 {
@ -495,7 +553,8 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
for path, element := range nodeQueue {
delete(nodeQueue, path)
data, err := srcDb.TrieDB().Node(element.hash)
owner, inner := trie.ResolvePath([]byte(element.path))
data, err := reader.Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x", element.hash)
}
@ -535,14 +594,19 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
}
// Tests that at any point in time during a sync, only complete sub-tries are in
// the database.
func TestIncompleteStateSync(t *testing.T) {
testIncompleteStateSync(t, rawdb.HashScheme)
testIncompleteStateSync(t, rawdb.PathScheme)
}
func testIncompleteStateSync(t *testing.T, scheme string) {
// Create a random state to copy
db, srcDb, srcRoot, srcAccounts := makeTestState()
db, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// isCodeLookup to save some hashing
var isCode = make(map[common.Hash]struct{})
@ -555,14 +619,14 @@ func TestIncompleteStateSync(t *testing.T) {
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
var (
addedCodes []common.Hash
addedPaths []string
addedHashes []common.Hash
)
reader, err := srcDb.TrieDB().Reader(srcRoot)
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not available %x", srcRoot)
}
@ -649,12 +713,11 @@ func TestIncompleteStateSync(t *testing.T) {
for _, node := range addedCodes {
val := rawdb.ReadCode(dstDb, node)
rawdb.DeleteCode(dstDb, node)
if err := checkStateConsistency(dstDb, srcRoot); err == nil {
if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil {
t.Errorf("trie inconsistency not caught, missing: %x", node)
}
rawdb.WriteCode(dstDb, node, val)
}
scheme := srcDb.TrieDB().Scheme()
for i, path := range addedPaths {
owner, inner := trie.ResolvePath([]byte(path))
hash := addedHashes[i]
@ -663,7 +726,7 @@ func TestIncompleteStateSync(t *testing.T) {
t.Error("missing trie node")
}
rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme)
if err := checkStateConsistency(dstDb, srcRoot); err == nil {
if err := checkStateConsistency(dstDb, scheme, srcRoot); err == nil {
t.Errorf("trie inconsistency not caught, missing: %v", path)
}
rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme)

8
core/types/hashing_test.go
View File

@ -39,7 +39,7 @@ func TestDeriveSha(t *testing.T) {
t.Fatal(err)
}
for len(txs) < 1000 {
exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
got := types.DeriveSha(txs, trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@ -86,7 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
}
})
@ -107,7 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) {
rndSeed := mrand.Int()
for i := 0; i < 10; i++ {
seed := rndSeed + i
exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed))
@ -135,7 +135,7 @@ func TestDerivableList(t *testing.T) {
},
}
for i, tc := range tcs[1:] {
exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp)

2
eth/api_backend.go
View File

@ -418,7 +418,7 @@ func (b *EthAPIBackend) StartMining() error {
}
func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
return b.eth.StateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
}
func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) {

2
eth/api_debug.go
View File

@ -322,7 +322,7 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c
if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
}
triedb := api.eth.BlockChain().StateCache().TrieDB()
triedb := api.eth.BlockChain().TrieDB()
oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb)
if err != nil {

16
eth/backend.go
View File

@ -133,8 +133,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if err != nil {
return nil, err
}
if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil {
log.Error("Failed to recover state", "error", err)
// Try to recover offline state pruning only in hash-based.
if config.StateScheme == rawdb.HashScheme {
if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil {
log.Error("Failed to recover state", "error", err)
}
}
// Transfer mining-related config to the ethash config.
chainConfig, err := core.LoadChainConfig(chainDb, config.Genesis)
@ -161,7 +164,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
p2pServer: stack.Server(),
shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb),
}
bcVersion := rawdb.ReadDatabaseVersion(chainDb)
var dbVer = "<nil>"
if bcVersion != nil {
@ -191,6 +193,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
StateHistory: config.StateHistory,
StateScheme: config.StateScheme,
}
)
// Override the chain config with provided settings.
@ -201,7 +205,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle
}
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory)
if err != nil {
return nil, err
}
@ -438,7 +442,7 @@ func (s *Ethereum) StartMining() error {
}
// If mining is started, we can disable the transaction rejection mechanism
// introduced to speed sync times.
s.handler.acceptTxs.Store(true)
s.handler.enableSyncedFeatures()
go s.miner.Start()
}
@ -471,7 +475,7 @@ func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }
func (s *Ethereum) IsListening() bool { return true } // Always listening
func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader }
func (s *Ethereum) Synced() bool { return s.handler.acceptTxs.Load() }
func (s *Ethereum) SetSynced() { s.handler.acceptTxs.Store(true) }
func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() }
func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning }
func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer }
func (s *Ethereum) Merger() *consensus.Merger { return s.merger }

3
eth/downloader/testchain_test.go
View File

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
// Test chain parameters.
@ -43,7 +44,7 @@ var (
Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}},
BaseFee: big.NewInt(params.InitialBaseFee),
}
testGenesis = testGspec.MustCommit(testDB)
testGenesis = testGspec.MustCommit(testDB, trie.NewDatabase(testDB, trie.HashDefaults))
)
// The common prefix of all test chains:

10
eth/ethconfig/config.go
View File

@ -27,6 +27,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/txpool/legacypool"
"github.com/ethereum/go-ethereum/eth/downloader"
@ -61,6 +62,9 @@ var Defaults = Config{
SyncMode: downloader.SnapSync,
NetworkId: 1,
TxLookupLimit: 2350000,
TransactionHistory: 2350000,
StateHistory: params.FullImmutabilityThreshold,
StateScheme: rawdb.HashScheme,
LightPeers: 100,
DatabaseCache: 512,
TrieCleanCache: 154,
@ -97,7 +101,11 @@ type Config struct {
NoPruning bool // Whether to disable pruning and flush everything to disk
NoPrefetch bool // Whether to disable prefetching and only load state on demand
TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
// Deprecated, use 'TransactionHistory' instead.
TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
// RequiredBlocks is a set of block number -> hash mappings which must be in the
// canonical chain of all remote peers. Setting the option makes geth verify the

18
eth/ethconfig/gen_config.go
View File

@ -25,6 +25,9 @@ func (c Config) MarshalTOML() (interface{}, error) {
NoPruning bool
NoPrefetch bool
TxLookupLimit uint64 `toml:",omitempty"`
TransactionHistory uint64 `toml:",omitempty"`
StateHistory uint64 `toml:",omitempty"`
StateScheme string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"`
LightServ int `toml:",omitempty"`
LightIngress int `toml:",omitempty"`
@ -63,6 +66,9 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.NoPruning = c.NoPruning
enc.NoPrefetch = c.NoPrefetch
enc.TxLookupLimit = c.TxLookupLimit
enc.TransactionHistory = c.TransactionHistory
enc.StateHistory = c.StateHistory
enc.StateScheme = c.StateScheme
enc.RequiredBlocks = c.RequiredBlocks
enc.LightServ = c.LightServ
enc.LightIngress = c.LightIngress
@ -105,6 +111,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
NoPruning *bool
NoPrefetch *bool
TxLookupLimit *uint64 `toml:",omitempty"`
TransactionHistory *uint64 `toml:",omitempty"`
StateHistory *uint64 `toml:",omitempty"`
StateScheme *string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"`
LightServ *int `toml:",omitempty"`
LightIngress *int `toml:",omitempty"`
@ -162,6 +171,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.TxLookupLimit != nil {
c.TxLookupLimit = *dec.TxLookupLimit
}
if dec.TransactionHistory != nil {
c.TransactionHistory = *dec.TransactionHistory
}
if dec.StateHistory != nil {
c.StateHistory = *dec.StateHistory
}
if dec.StateScheme != nil {
c.StateScheme = *dec.StateScheme
}
if dec.RequiredBlocks != nil {
c.RequiredBlocks = dec.RequiredBlocks
}

2
eth/fetcher/block_fetcher_test.go
View File

@ -44,7 +44,7 @@ var (
Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}},
BaseFee: big.NewInt(params.InitialBaseFee),
}
genesis = gspec.MustCommit(testdb)
genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults))
unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil))
)

4
eth/filters/filter_test.go
View File

@ -86,7 +86,7 @@ func BenchmarkFilters(b *testing.B) {
// The test txs are not properly signed, can't simply create a chain
// and then import blocks. TODO(rjl493456442) try to get rid of the
// manual database writes.
gspec.MustCommit(db)
gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
for i, block := range chain {
rawdb.WriteBlock(db, block)
@ -180,7 +180,7 @@ func TestFilters(t *testing.T) {
// Hack: GenerateChainWithGenesis creates a new db.
// Commit the genesis manually and use GenerateChain.
_, err = gspec.Commit(db, trie.NewDatabase(db))
_, err = gspec.Commit(db, trie.NewDatabase(db, nil))
if err != nil {
t.Fatal(err)
}

15
eth/handler.go
View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth/downloader"
@ -40,6 +41,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
const (
@ -183,7 +185,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
}
// If we've successfully finished a sync cycle, accept transactions from
// the network
h.acceptTxs.Store(true)
h.enableSyncedFeatures()
}
// Construct the downloader (long sync)
h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, success)
@ -272,7 +274,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
}
n, err := h.chain.InsertChain(blocks)
if err == nil {
h.acceptTxs.Store(true) // Mark initial sync done on any fetcher import
h.enableSyncedFeatures() // Mark initial sync done on any fetcher import
}
return n, err
}
@ -674,3 +676,12 @@ func (h *handler) txBroadcastLoop() {
}
}
}
// enableSyncedFeatures enables the post-sync functionalities when the initial
// sync is finished.
func (h *handler) enableSyncedFeatures() {
h.acceptTxs.Store(true)
if h.chain.TrieDB().Scheme() == rawdb.PathScheme {
h.chain.TrieDB().SetBufferSize(pathdb.DefaultBufferSize)
}
}

2
eth/protocols/eth/handler_test.go
View File

@ -112,7 +112,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int,
panic(err)
}
for _, block := range bs {
chain.StateCache().TrieDB().Commit(block.Root(), false)
chain.TrieDB().Commit(block.Root(), false)
}
txconfig := legacypool.DefaultConfig
txconfig.Journal = "" // Don't litter the disk with test journals

7
eth/protocols/eth/handlers.go
View File

@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
@ -246,6 +247,10 @@ func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error {
// ServiceGetNodeDataQuery assembles the response to a node data query. It is
// exposed to allow external packages to test protocol behavior.
func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) [][]byte {
// Request nodes by hash is not supported in path-based scheme.
if chain.TrieDB().Scheme() == rawdb.PathScheme {
return nil
}
// Gather state data until the fetch or network limits is reached
var (
bytes int
@ -257,7 +262,7 @@ func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) []
break
}
// Retrieve the requested state entry
entry, err := chain.TrieNode(hash)
entry, err := chain.TrieDB().Node(hash)
if len(entry) == 0 || err != nil {
// Read the contract code with prefix only to save unnecessary lookups.
entry, err = chain.ContractCodeWithPrefix(hash)

8
eth/protocols/snap/handler.go
View File

@ -284,7 +284,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac
req.Bytes = softResponseLimit
}
// Retrieve the requested state and bail out if non existent
tr, err := trie.New(trie.StateTrieID(req.Root), chain.StateCache().TrieDB())
tr, err := trie.New(trie.StateTrieID(req.Root), chain.TrieDB())
if err != nil {
return nil, nil
}
@ -414,7 +414,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
if origin != (common.Hash{}) || (abort && len(storage) > 0) {
// Request started at a non-zero hash or was capped prematurely, add
// the endpoint Merkle proofs
accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.StateCache().TrieDB())
accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.TrieDB())
if err != nil {
return nil, nil
}
@ -423,7 +423,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
return nil, nil
}
id := trie.StorageTrieID(req.Root, account, acc.Root)
stTrie, err := trie.NewStateTrie(id, chain.StateCache().TrieDB())
stTrie, err := trie.NewStateTrie(id, chain.TrieDB())
if err != nil {
return nil, nil
}
@ -487,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
req.Bytes = softResponseLimit
}
// Make sure we have the state associated with the request
triedb := chain.StateCache().TrieDB()
triedb := chain.TrieDB()
accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), triedb)
if err != nil {

204
eth/protocols/snap/sync_test.go
View File

@ -35,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode"
"golang.org/x/crypto/sha3"
"golang.org/x/exp/slices"
@ -561,6 +562,11 @@ func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Has
func TestSyncBloatedProof(t *testing.T) {
t.Parallel()
testSyncBloatedProof(t, rawdb.HashScheme)
testSyncBloatedProof(t, rawdb.PathScheme)
}
func testSyncBloatedProof(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -570,7 +576,7 @@ func TestSyncBloatedProof(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
source := newTestPeer("source", t, term)
source.accountTrie = sourceAccountTrie.Copy()
source.accountValues = elems
@ -638,6 +644,11 @@ func setupSyncer(scheme string, peers ...*testPeer) *Syncer {
func TestSync(t *testing.T) {
t.Parallel()
testSync(t, rawdb.HashScheme)
testSync(t, rawdb.PathScheme)
}
func testSync(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -647,7 +658,7 @@ func TestSync(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -659,7 +670,7 @@ func TestSync(t *testing.T) {
if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil {
t.Fatalf("sync failed: %v", err)
}
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a
@ -667,6 +678,11 @@ func TestSync(t *testing.T) {
func TestSyncTinyTriePanic(t *testing.T) {
t.Parallel()
testSyncTinyTriePanic(t, rawdb.HashScheme)
testSyncTinyTriePanic(t, rawdb.PathScheme)
}
func testSyncTinyTriePanic(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -676,7 +692,7 @@ func TestSyncTinyTriePanic(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1, scheme)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -690,13 +706,18 @@ func TestSyncTinyTriePanic(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestMultiSync tests a basic sync with multiple peers
func TestMultiSync(t *testing.T) {
t.Parallel()
testMultiSync(t, rawdb.HashScheme)
testMultiSync(t, rawdb.PathScheme)
}
func testMultiSync(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -706,7 +727,7 @@ func TestMultiSync(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -720,13 +741,18 @@ func TestMultiSync(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncWithStorage tests basic sync using accounts + storage + code
func TestSyncWithStorage(t *testing.T) {
t.Parallel()
testSyncWithStorage(t, rawdb.HashScheme)
testSyncWithStorage(t, rawdb.PathScheme)
}
func testSyncWithStorage(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -736,7 +762,7 @@ func TestSyncWithStorage(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 3000, true, false)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -752,13 +778,18 @@ func TestSyncWithStorage(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all
func TestMultiSyncManyUseless(t *testing.T) {
t.Parallel()
testMultiSyncManyUseless(t, rawdb.HashScheme)
testMultiSyncManyUseless(t, rawdb.PathScheme)
}
func testMultiSyncManyUseless(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -768,7 +799,7 @@ func TestMultiSyncManyUseless(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer {
source := newTestPeer(name, t, term)
@ -801,11 +832,18 @@ func TestMultiSyncManyUseless(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all
func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) {
t.Parallel()
testMultiSyncManyUselessWithLowTimeout(t, rawdb.HashScheme)
testMultiSyncManyUselessWithLowTimeout(t, rawdb.PathScheme)
}
func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -815,7 +853,7 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer {
source := newTestPeer(name, t, term)
@ -853,11 +891,18 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all
func TestMultiSyncManyUnresponsive(t *testing.T) {
t.Parallel()
testMultiSyncManyUnresponsive(t, rawdb.HashScheme)
testMultiSyncManyUnresponsive(t, rawdb.PathScheme)
}
func testMultiSyncManyUnresponsive(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -867,7 +912,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer {
source := newTestPeer(name, t, term)
@ -903,7 +948,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
func checkStall(t *testing.T, term func()) chan struct{} {
@ -925,6 +970,11 @@ func checkStall(t *testing.T, term func()) chan struct{} {
func TestSyncBoundaryAccountTrie(t *testing.T) {
t.Parallel()
testSyncBoundaryAccountTrie(t, rawdb.HashScheme)
testSyncBoundaryAccountTrie(t, rawdb.PathScheme)
}
func testSyncBoundaryAccountTrie(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -934,7 +984,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(3000)
nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(scheme, 3000)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -952,7 +1002,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is
@ -960,6 +1010,11 @@ func TestSyncBoundaryAccountTrie(t *testing.T) {
func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
t.Parallel()
testSyncNoStorageAndOneCappedPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneCappedPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneCappedPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -969,7 +1024,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, slow bool) *testPeer {
source := newTestPeer(name, t, term)
@ -994,7 +1049,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver
@ -1002,6 +1057,11 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) {
t.Parallel()
testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneCodeCorruptPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1011,7 +1071,7 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, codeFn codeHandlerFunc) *testPeer {
source := newTestPeer(name, t, term)
@ -1034,12 +1094,17 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
t.Parallel()
testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneAccountCorruptPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1049,7 +1114,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, accFn accountHandlerFunc) *testPeer {
source := newTestPeer(name, t, term)
@ -1072,7 +1137,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes
@ -1080,6 +1145,11 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
t.Parallel()
testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneCodeCappedPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1089,7 +1159,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, codeFn codeHandlerFunc) *testPeer {
source := newTestPeer(name, t, term)
@ -1123,7 +1193,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
if threshold := 100; counter > threshold {
t.Logf("Error, expected < %d invocations, got %d", threshold, counter)
}
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncBoundaryStorageTrie tests sync against a few normal peers, but the
@ -1131,6 +1201,11 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
func TestSyncBoundaryStorageTrie(t *testing.T) {
t.Parallel()
testSyncBoundaryStorageTrie(t, rawdb.HashScheme)
testSyncBoundaryStorageTrie(t, rawdb.PathScheme)
}
func testSyncBoundaryStorageTrie(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1140,7 +1215,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(10, 1000, false, true)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 10, 1000, false, true)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -1160,7 +1235,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is
@ -1168,6 +1243,11 @@ func TestSyncBoundaryStorageTrie(t *testing.T) {
func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
t.Parallel()
testSyncWithStorageAndOneCappedPeer(t, rawdb.HashScheme)
testSyncWithStorageAndOneCappedPeer(t, rawdb.PathScheme)
}
func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1177,7 +1257,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 300, 1000, false, false)
mkSource := func(name string, slow bool) *testPeer {
source := newTestPeer(name, t, term)
@ -1202,7 +1282,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is
@ -1210,6 +1290,11 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
func TestSyncWithStorageAndCorruptPeer(t *testing.T) {
t.Parallel()
testSyncWithStorageAndCorruptPeer(t, rawdb.HashScheme)
testSyncWithStorageAndCorruptPeer(t, rawdb.PathScheme)
}
func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1219,7 +1304,7 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, handler storageHandlerFunc) *testPeer {
source := newTestPeer(name, t, term)
@ -1243,12 +1328,17 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
t.Parallel()
testSyncWithStorageAndNonProvingPeer(t, rawdb.HashScheme)
testSyncWithStorageAndNonProvingPeer(t, rawdb.PathScheme)
}
func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1258,7 +1348,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, handler storageHandlerFunc) *testPeer {
source := newTestPeer(name, t, term)
@ -1281,7 +1371,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err)
}
close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
// TestSyncWithStorage tests basic sync using accounts + storage + code, against
@ -1290,6 +1380,12 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
// did not mark the account for healing.
func TestSyncWithStorageMisbehavingProve(t *testing.T) {
t.Parallel()
testSyncWithStorageMisbehavingProve(t, rawdb.HashScheme)
testSyncWithStorageMisbehavingProve(t, rawdb.PathScheme)
}
func testSyncWithStorageMisbehavingProve(t *testing.T, scheme string) {
var (
once sync.Once
cancel = make(chan struct{})
@ -1299,7 +1395,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(10, 30, false)
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(scheme, 10, 30, false)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -1314,7 +1410,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) {
if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil {
t.Fatalf("sync failed: %v", err)
}
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
}
type kv struct {
@ -1364,9 +1460,9 @@ func getCodeByHash(hash common.Hash) []byte {
}
// makeAccountTrieNoStorage spits out a trie, along with the leafs
func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) {
func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db)
entries []*kv
)
@ -1396,12 +1492,12 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) {
// makeBoundaryAccountTrie constructs an account trie. Instead of filling
// accounts normally, this function will fill a few accounts which have
// boundary hash.
func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) {
func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) {
var (
entries []*kv
boundaries []common.Hash
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db)
)
// Initialize boundaries
@ -1457,9 +1553,9 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) {
// makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts
// has a unique storage set.
func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) {
func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db)
entries []*kv
storageRoots = make(map[common.Hash]common.Hash)
@ -1512,9 +1608,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
}
// makeAccountTrieWithStorage spits out a trie, along with the leafs
func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) {
func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db)
entries []*kv
storageRoots = make(map[common.Hash]common.Hash)
@ -1656,9 +1752,9 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo
return root, nodes, entries
}
func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
t.Helper()
triedb := trie.NewDatabase(rawdb.NewDatabase(db))
triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme))
accTrie, err := trie.New(trie.StateTrieID(root), triedb)
if err != nil {
t.Fatal(err)
@ -1700,6 +1796,13 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
// TestSyncAccountPerformance tests how efficient the snap algo is at minimizing
// state healing
func TestSyncAccountPerformance(t *testing.T) {
t.Parallel()
testSyncAccountPerformance(t, rawdb.HashScheme)
testSyncAccountPerformance(t, rawdb.PathScheme)
}
func testSyncAccountPerformance(t *testing.T, scheme string) {
// Set the account concurrency to 1. This _should_ result in the
// range root to become correct, and there should be no healing needed
defer func(old int) { accountConcurrency = old }(accountConcurrency)
@ -1714,7 +1817,7 @@ func TestSyncAccountPerformance(t *testing.T) {
})
}
)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100)
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term)
@ -1727,7 +1830,7 @@ func TestSyncAccountPerformance(t *testing.T) {
if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil {
t.Fatalf("sync failed: %v", err)
}
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t)
verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
// The trie root will always be requested, since it is added when the snap
// sync cycle starts. When popping the queue, we do not look it up again.
// Doing so would bring this number down to zero in this artificial testcase,
@ -1787,3 +1890,10 @@ func TestSlotEstimation(t *testing.T) {
}
}
}
func newDbConfig(scheme string) *trie.Config {
if scheme == rawdb.HashScheme {
return &trie.Config{}
}
return &trie.Config{PathDB: pathdb.Defaults}
}

91
eth/state_accessor.go
View File

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
@ -36,31 +37,11 @@ import (
// for releasing state.
var noopReleaser = tracers.StateReleaseFunc(func() {})
// StateAtBlock retrieves the state database associated with a certain block.
// If no state is locally available for the given block, a number of blocks
// are attempted to be reexecuted to generate the desired state. The optional
// base layer statedb can be provided which is regarded as the statedb of the
// parent block.
//
// An additional release function will be returned if the requested state is
// available. Release is expected to be invoked when the returned state is no longer needed.
// Its purpose is to prevent resource leaking. Though it can be noop in some cases.
//
// Parameters:
// - block: The block for which we want the state(state = block.Root)
// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state
// - base: If the caller is tracing multiple blocks, the caller can provide the parent
// state continuously from the callsite.
// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should
// be made from caller, e.g. perform Commit or other 'save-to-disk' changes.
// Otherwise, the trash generated by caller may be persisted permanently.
// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is
// provided, it would be preferable to start from a fresh state, if we have it
// on disk.
func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
var (
current *types.Block
database state.Database
triedb *trie.Database
report = true
origin = block.NumberU64()
)
@ -71,9 +52,9 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
// on top to prevent garbage collection and return a release
// function to deref it.
if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil {
statedb.Database().TrieDB().Reference(block.Root(), common.Hash{})
eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{})
return statedb, func() {
statedb.Database().TrieDB().Dereference(block.Root())
eth.blockchain.TrieDB().Dereference(block.Root())
}, nil
}
}
@ -84,14 +65,16 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
if preferDisk {
// Create an ephemeral trie.Database for isolating the live one. Otherwise
// the internal junks created by tracing will be persisted into the disk.
database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16})
// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
// please re-enable it for better performance.
database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults)
if statedb, err = state.New(block.Root(), database, nil); err == nil {
log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number())
return statedb, noopReleaser, nil
}
}
// The optional base statedb is given, mark the start point as parent block
statedb, database, report = base, base.Database(), false
statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false
current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
} else {
// Otherwise, try to reexec blocks until we find a state or reach our limit
@ -99,7 +82,10 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
// Create an ephemeral trie.Database for isolating the live one. Otherwise
// the internal junks created by tracing will be persisted into the disk.
database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16})
// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
// please re-enable it for better performance.
triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults)
database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb)
// If we didn't check the live database, do check state over ephemeral database,
// otherwise we would rewind past a persisted block (specific corner case is
@ -175,17 +161,58 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
}
// Hold the state reference and also drop the parent state
// to prevent accumulating too many nodes in memory.
database.TrieDB().Reference(root, common.Hash{})
triedb.Reference(root, common.Hash{})
if parent != (common.Hash{}) {
database.TrieDB().Dereference(parent)
triedb.Dereference(parent)
}
parent = root
}
if report {
nodes, imgs := database.TrieDB().Size()
nodes, imgs := triedb.Size()
log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs)
}
return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil
return statedb, func() { triedb.Dereference(block.Root()) }, nil
}
func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) {
// Check if the requested state is available in the live chain.
statedb, err := eth.blockchain.StateAt(block.Root())
if err == nil {
return statedb, noopReleaser, nil
}
// TODO historic state is not supported in path-based scheme.
// Fully archive node in pbss will be implemented by relying
// on state history, but needs more work on top.
return nil, nil, errors.New("historical state not available in path scheme yet")
}
// stateAtBlock retrieves the state database associated with a certain block.
// If no state is locally available for the given block, a number of blocks
// are attempted to be reexecuted to generate the desired state. The optional
// base layer statedb can be provided which is regarded as the statedb of the
// parent block.
//
// An additional release function will be returned if the requested state is
// available. Release is expected to be invoked when the returned state is no
// longer needed. Its purpose is to prevent resource leaking. Though it can be
// noop in some cases.
//
// Parameters:
// - block: The block for which we want the state(state = block.Root)
// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state
// - base: If the caller is tracing multiple blocks, the caller can provide the parent
// state continuously from the callsite.
// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should
// be made from caller, e.g. perform Commit or other 'save-to-disk' changes.
// Otherwise, the trash generated by caller may be persisted permanently.
// - preferDisk: This arg can be used by the caller to signal that even though the 'base' is
// provided, it would be preferable to start from a fresh state, if we have it
// on disk.
func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme {
return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk)
}
return eth.pathState(block)
}
// stateAtTransaction returns the execution environment of a certain transaction.
@ -201,7 +228,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block,
}
// Lookup the statedb of parent block from the live database,
// otherwise regenerate it on the flight.
statedb, release, err := eth.StateAtBlock(ctx, parent, reexec, nil, true, false)
statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, nil, true, false)
if err != nil {
return nil, vm.BlockContext{}, nil, nil, err
}

13
eth/tracers/internal/tracetest/calltrace_test.go
View File

@ -137,8 +137,10 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) {
GasLimit: uint64(test.Context.GasLimit),
BaseFee: test.Genesis.BaseFee,
}
_, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
)
triedb.Close()
tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)
if err != nil {
t.Fatalf("failed to create call tracer: %v", err)
@ -237,7 +239,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) {
Difficulty: (*big.Int)(test.Context.Difficulty),
GasLimit: uint64(test.Context.GasLimit),
}
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
defer triedb.Close()
b.ReportAllocs()
b.ResetTimer()
@ -363,7 +366,7 @@ func TestInternals(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(),
triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(),
core.GenesisAlloc{
to: core.GenesisAccount{
Code: tc.code,
@ -371,7 +374,9 @@ func TestInternals(t *testing.T) {
origin: core.GenesisAccount{
Balance: big.NewInt(500000000000000),
},
}, false)
}, false, rawdb.HashScheme)
defer triedb.Close()
evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer})
msg := &core.Message{
To: &to,

3
eth/tracers/internal/tracetest/flat_calltrace_test.go
View File

@ -100,7 +100,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string
Difficulty: (*big.Int)(test.Context.Difficulty),
GasLimit: uint64(test.Context.GasLimit),
}
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
defer triedb.Close()
// Create the tracer, the EVM environment and run it
tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)

4
eth/tracers/internal/tracetest/prestate_test.go
View File

@ -108,8 +108,10 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) {
GasLimit: uint64(test.Context.GasLimit),
BaseFee: test.Genesis.BaseFee,
}
_, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
)
defer triedb.Close()
tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)
if err != nil {
t.Fatalf("failed to create call tracer: %v", err)

4
eth/tracers/tracers_test.go
View File

@ -79,7 +79,9 @@ func BenchmarkTransactionTrace(b *testing.B) {
Code: []byte{},
Balance: big.NewInt(500000000000000),
}
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false)
triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme)
defer triedb.Close()
// Create the tracer, the EVM environment and run it
tracer := logger.NewStructLogger(&logger.Config{
Debug: false,

2
internal/flags/categories.go
View File

@ -22,7 +22,7 @@ const (
EthCategory = "ETHEREUM"
LightCategory = "LIGHT CLIENT"
DevCategory = "DEVELOPER CHAIN"
EthashCategory = "ETHASH"
StateCategory = "STATE HISTORY MANAGEMENT"
TxPoolCategory = "TRANSACTION POOL (EVM)"
BlobPoolCategory = "TRANSACTION POOL (BLOB)"
PerfCategory = "PERFORMANCE TUNING"

3
les/client.go
View File

@ -98,7 +98,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) {
if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle
}
chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, trie.NewDatabase(chainDb), config.Genesis, &overrides)
triedb := trie.NewDatabase(chainDb, trie.HashDefaults)
chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, triedb, config.Genesis, &overrides)
if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat {
return nil, genesisErr
}

8
les/handler_test.go
View File

@ -406,7 +406,7 @@ func testGetProofs(t *testing.T, protocol int) {
accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}}
for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ {
header := bc.GetHeaderByNumber(i)
trie, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db))
trie, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB())
for _, acc := range accounts {
req := ProofReq{
@ -457,7 +457,7 @@ func testGetStaleProof(t *testing.T, protocol int) {
var expected []rlp.RawValue
if wantOK {
proofsV2 := light.NewNodeSet()
t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db))
t, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB())
t.Prove(account, proofsV2)
expected = proofsV2.NodeList()
}
@ -513,7 +513,7 @@ func testGetCHTProofs(t *testing.T, protocol int) {
AuxData: [][]byte{rlp},
}
root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash())
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix))))
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix)), trie.HashDefaults))
trie.Prove(key, &proofsV2.Proofs)
// Assemble the requests for the different protocols
requestsV2 := []HelperTrieReq{{
@ -578,7 +578,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) {
var proofs HelperTrieResps
root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash())
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix))))
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix)), trie.HashDefaults))
trie.Prove(key, &proofs.Proofs)
// Send the proof request and verify the response

2
les/odr_test.go
View File

@ -104,7 +104,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon
for _, addr := range acc {
if bc != nil {
header := bc.GetHeaderByHash(bhash)
st, err = state.New(header.Root, state.NewDatabase(db), nil)
st, err = state.New(header.Root, bc.StateCache(), nil)
} else {
header := lc.GetHeaderByHash(bhash)
st = light.NewState(ctx, header, lc.Odr())

3
les/server_handler.go
View File

@ -390,7 +390,8 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie {
if root == (common.Hash{}) {
return nil
}
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix)))
triedb := trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix), trie.HashDefaults)
trie, _ := trie.New(trie.TrieID(root), triedb)
return trie
}

5
les/server_requests.go
View File

@ -303,9 +303,8 @@ func handleGetCode(msg Decoder) (serveRequestFn, uint64, uint64, error) {
p.bumpInvalid()
continue
}
triedb := bc.StateCache().TrieDB()
address := common.BytesToAddress(request.AccountAddress)
account, err := getAccount(triedb, header.Root, address)
account, err := getAccount(bc.TrieDB(), header.Root, address)
if err != nil {
p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", address, "err", err)
p.bumpInvalid()
@ -424,7 +423,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) {
default:
// Account key specified, open a storage trie
address := common.BytesToAddress(request.AccountAddress)
account, err := getAccount(statedb.TrieDB(), root, address)
account, err := getAccount(bc.TrieDB(), root, address)
if err != nil {
p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", address, "err", err)
p.bumpInvalid()

5
les/test_helper.go
View File

@ -49,6 +49,7 @@ import (
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
var (
@ -188,7 +189,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index
BaseFee: big.NewInt(params.InitialBaseFee),
}
)
genesis := gspec.MustCommit(db)
genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
chain, _ := light.NewLightChain(odr, gspec.Config, engine)
client := &LightEthereum{
@ -226,7 +227,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da
BaseFee: big.NewInt(params.InitialBaseFee),
}
)
genesis := gspec.MustCommit(db)
genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
// create a simulation backend and pre-commit several customized block to the database.
simulation := backends.NewSimulatedBackendWithDatabase(db, gspec.Alloc, 100000000)

5
light/lightchain_test.go
View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
// So we can deterministically seed different blockchains
@ -55,7 +56,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [
func newCanonical(n int) (ethdb.Database, *LightChain, error) {
db := rawdb.NewMemoryDatabase()
gspec := core.Genesis{Config: params.TestChainConfig}
genesis := gspec.MustCommit(db)
genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker())
// Create and inject the requested chain
@ -75,7 +76,7 @@ func newTestLightChain() *LightChain {
Difficulty: big.NewInt(1),
Config: params.TestChainConfig,
}
gspec.MustCommit(db)
gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker())
if err != nil {
panic(err)

3
light/odr_test.go
View File

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
var (
@ -282,7 +283,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) {
t.Fatal(err)
}
gspec.MustCommit(ldb)
gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults))
odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig}
lightchain, err := NewLightChain(odr, gspec.Config, ethash.NewFullFaker())
if err != nil {

4
light/postprocess.go
View File

@ -145,7 +145,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis
diskdb: db,
odr: odr,
trieTable: trieTable,
triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down
triedb: trie.NewDatabase(trieTable, trie.HashDefaults),
sectionSize: size,
disablePruning: disablePruning,
}
@ -348,7 +348,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
diskdb: db,
odr: odr,
trieTable: trieTable,
triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down
triedb: trie.NewDatabase(trieTable, trie.HashDefaults),
parentSize: parentSize,
size: size,
disablePruning: disablePruning,

6
light/trie.go
View File

@ -215,7 +215,8 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
} else {
id = trie.StateTrieID(t.id.StateRoot)
}
t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database()))
triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults)
t.trie, err = trie.New(id, triedb)
}
if err == nil {
err = fn()
@ -247,7 +248,8 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator {
} else {
id = trie.StateTrieID(t.id.StateRoot)
}
t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database()))
triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults)
t, err := trie.New(id, triedb)
if err == nil {
it.t.trie = t
}

2
light/trie_test.go
View File

@ -50,7 +50,7 @@ func TestNodeIterator(t *testing.T) {
panic(err)
}
gspec.MustCommit(lightdb)
gspec.MustCommit(lightdb, trie.NewDatabase(lightdb, trie.HashDefaults))
ctx := context.Background()
odr := &testOdr{sdb: fulldb, ldb: lightdb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig}
head := blockchain.CurrentHeader()

3
light/txpool_test.go
View File

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
type testTxRelay struct {
@ -96,7 +97,7 @@ func TestTxPool(t *testing.T) {
panic(err)
}
gspec.MustCommit(ldb)
gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults))
odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig}
relay := &testTxRelay{
send: make(chan int, 1),

5
miner/miner_test.go
View File

@ -288,8 +288,9 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) {
}
// Create chainConfig
chainDB := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(chainDB, nil)
genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345"))
chainConfig, _, err := core.SetupGenesisBlock(chainDB, trie.NewDatabase(chainDB), genesis)
chainConfig, _, err := core.SetupGenesisBlock(chainDB, triedb, genesis)
if err != nil {
t.Fatalf("can't create new chain config: %v", err)
}
@ -300,7 +301,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) {
if err != nil {
t.Fatalf("can't create new chain %v", err)
}
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(chainDB), nil)
statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache(), nil)
blockchain := &testBlockChain{chainConfig, statedb, 10000000, new(event.Feed)}
pool := legacypool.New(testTxPoolConfig, blockchain)

16
tests/block_test.go
View File

@ -18,6 +18,8 @@ package tests
import (
"testing"
"github.com/ethereum/go-ethereum/core/rawdb"
)
func TestBlockchain(t *testing.T) {
@ -48,11 +50,17 @@ func TestBlockchain(t *testing.T) {
bt.skipLoad(`.*randomStatetest94.json.*`)
bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
if err := bt.checkFailure(t, test.Run(false, nil)); err != nil {
t.Errorf("test without snapshotter failed: %v", err)
if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil {
t.Errorf("test in hash mode without snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(true, nil)); err != nil {
t.Errorf("test with snapshotter failed: %v", err)
if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil {
t.Errorf("test in hash mode with snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil {
t.Errorf("test in path mode without snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil {
t.Errorf("test in path mode with snapshotter failed: %v", err)
}
})
// There is also a LegacyTests folder, containing blockchain tests generated

27
tests/block_test_util.go
View File

@ -38,6 +38,9 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
// A BlockTest checks handling of entire blocks.
@ -100,16 +103,30 @@ type btHeaderMarshaling struct {
BaseFeePerGas *math.HexOrDecimal256
}
func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error {
func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error {
config, ok := Forks[t.json.Network]
if !ok {
return UnsupportedForkError{t.json.Network}
}
// import pre accounts & construct test genesis block & state root
db := rawdb.NewMemoryDatabase()
var (
db = rawdb.NewMemoryDatabase()
tconf = &trie.Config{}
)
if scheme == rawdb.PathScheme {
tconf.PathDB = pathdb.Defaults
} else {
tconf.HashDB = hashdb.Defaults
}
// Commit genesis state
gspec := t.genesis(config)
gblock := gspec.MustCommit(db)
triedb := trie.NewDatabase(db, tconf)
gblock, err := gspec.Commit(db, triedb)
if err != nil {
return err
}
triedb.Close() // close the db to prevent memory leak
if gblock.Hash() != t.json.Genesis.Hash {
return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6])
}
@ -119,7 +136,7 @@ func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error {
// Wrap the original engine within the beacon-engine
engine := beacon.New(ethash.NewFaker())
cache := &core.CacheConfig{TrieCleanLimit: 0}
cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme}
if snapshotter {
cache.SnapshotLimit = 1
cache.SnapshotWait = true

4
tests/fuzzers/les/les-fuzzer.go
View File

@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addresses []common.Address, txHashes []co
}
func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) {
chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults))
bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults))
for i := 0; i < testChainLen; i++ {
// The element in CHT is <big-endian block number> -> <block hash>
key := make([]byte, 8)

2
tests/fuzzers/rangeproof/rangeproof-fuzzer.go
View File

@ -56,7 +56,7 @@ func (f *fuzzer) readInt() uint64 {
}
func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv)
size := f.readInt()
// Fill it with some fluff

4
tests/fuzzers/stacktrie/trie_fuzzer.go
View File

@ -136,10 +136,10 @@ func (f *fuzzer) fuzz() int {
// This spongeDb is used to check the sequence of disk-db-writes
var (
spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA))
dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA), nil)
trieA = trie.NewEmpty(dbA)
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB))
dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB), nil)
trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme())
})

2
tests/fuzzers/trie/trie-fuzzer.go
View File

@ -143,7 +143,7 @@ func Fuzz(input []byte) int {
func runRandTest(rt randTest) error {
var (
triedb = trie.NewDatabase(rawdb.NewMemoryDatabase())
triedb = trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)
tr = trie.NewEmpty(triedb)
origin = types.EmptyRootHash
values = make(map[string]string) // tracks content of the trie

56
tests/state_test.go
View File

@ -30,6 +30,8 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
@ -78,21 +80,52 @@ func TestState(t *testing.T) {
subtest := subtest
key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
t.Run(key+"/trie", func(t *testing.T) {
t.Run(key+"/hash/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
_, _, err := test.Run(subtest, vmconfig, false)
return st.checkFailure(t, err)
var result error
test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/snap", func(t *testing.T) {
t.Run(key+"/hash/snap", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
snaps, statedb, err := test.Run(subtest, vmconfig, true)
if snaps != nil && statedb != nil {
if _, err := snaps.Journal(statedb.IntermediateRoot(false)); err != nil {
return err
var result error
test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if snaps != nil && state != nil {
if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil {
result = err
return
}
}
}
return st.checkFailure(t, err)
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/path/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
var result error
test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/path/snap", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
var result error
test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if snaps != nil && state != nil {
if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil {
result = err
return
}
}
result = st.checkFailure(t, err)
})
return result
})
})
}
@ -190,7 +223,8 @@ func runBenchmark(b *testing.B, t *StateTest) {
vmconfig.ExtraEips = eips
block := t.genesis(config).ToBlock()
_, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false)
triedb, _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme)
defer triedb.Close()
var baseFee *big.Int
if rules.IsLondon {

71
tests/state_test_util.go
View File

@ -39,6 +39,8 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"golang.org/x/crypto/sha3"
)
@ -187,43 +189,50 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error {
}
// Run executes a specific subtest and verifies the post-state and logs
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) {
snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter)
if checkedErr := t.checkError(subtest, err); checkedErr != nil {
return snaps, statedb, checkedErr
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) {
triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme)
// Invoke the callback at the end of function for further analysis.
defer func() {
postCheck(result, snaps, statedb)
if triedb != nil {
triedb.Close()
}
}()
checkedErr := t.checkError(subtest, err)
if checkedErr != nil {
return checkedErr
}
// The error has been checked; if it was unexpected, it's already returned.
if err != nil {
// Here, an error exists but it was expected.
// We do not check the post state or logs.
return snaps, statedb, nil
return nil
}
post := t.json.Post[subtest.Fork][subtest.Index]
// N.B: We need to do this in a two-step process, because the first Commit takes care
// of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed.
if root != common.Hash(post.Root) {
return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
}
if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) {
return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
}
// Re-init the post-state instance for further operation
statedb, err = state.New(root, statedb.Database(), snaps)
if err != nil {
return nil, nil, err
}
return snaps, statedb, nil
statedb, _ = state.New(root, statedb.Database(), snaps)
return nil
}
// RunNoVerify runs a specific subtest and returns the statedb and post-state root
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, common.Hash, error) {
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) {
config, eips, err := GetChainConfig(subtest.Fork)
if err != nil {
return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork}
return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork}
}
vmconfig.ExtraEips = eips
block := t.genesis(config).ToBlock()
snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter)
triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme)
var baseFee *big.Int
if config.IsLondon(new(big.Int)) {
@ -237,7 +246,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
post := t.json.Post[subtest.Fork][subtest.Index]
msg, err := t.json.Tx.toMessage(post, baseFee)
if err != nil {
return nil, nil, common.Hash{}, err
triedb.Close()
return nil, nil, nil, common.Hash{}, err
}
// Try to recover tx with current signer
@ -245,11 +255,13 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
var ttx types.Transaction
err := ttx.UnmarshalBinary(post.TxBytes)
if err != nil {
return nil, nil, common.Hash{}, err
triedb.Close()
return nil, nil, nil, common.Hash{}, err
}
if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil {
return nil, nil, common.Hash{}, err
triedb.Close()
return nil, nil, nil, common.Hash{}, err
}
}
@ -268,6 +280,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
context.Difficulty = big.NewInt(0)
}
evm := vm.NewEVM(context, txContext, statedb, config, vmconfig)
// Execute the message.
snapshot := statedb.Snapshot()
gaspool := new(core.GasPool)
@ -282,17 +295,25 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
// - there are only 'bad' transactions, which aren't executed. In those cases,
// the coinbase gets no txfee, so isn't created, and thus needs to be touched
statedb.AddBalance(block.Coinbase(), new(big.Int))
// Commit block
// Commit state mutations into database.
root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number()))
return snaps, statedb, root, err
return triedb, snaps, statedb, root, err
}
func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas]
}
func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) (*snapshot.Tree, *state.StateDB) {
sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true})
func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) {
tconf := &trie.Config{Preimages: true}
if scheme == rawdb.HashScheme {
tconf.HashDB = hashdb.Defaults
} else {
tconf.PathDB = pathdb.Defaults
}
triedb := trie.NewDatabase(db, tconf)
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ := state.New(types.EmptyRootHash, sdb, nil)
for addr, a := range accounts {
statedb.SetCode(addr, a.Code)
@ -313,10 +334,10 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo
NoBuild: false,
AsyncBuild: false,
}
snaps, _ = snapshot.New(snapconfig, db, sdb.TrieDB(), root)
snaps, _ = snapshot.New(snapconfig, db, triedb, root)
}
statedb, _ = state.New(root, sdb, snaps)
return snaps, statedb
return triedb, snaps, statedb
}
func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis {

107
trie/database.go
View File

@ -21,6 +21,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode"
@ -29,14 +30,21 @@ import (
// Config defines all necessary options for database.
type Config struct {
Cache int // Memory allowance (MB) to use for caching trie nodes in memory
Preimages bool // Flag whether the preimage of trie key is recorded
PathDB *pathdb.Config // Configs for experimental path-based scheme, not used yet.
Preimages bool // Flag whether the preimage of node key is recorded
HashDB *hashdb.Config // Configs for hash-based scheme
PathDB *pathdb.Config // Configs for experimental path-based scheme
// Testing hooks
OnCommit func(states *triestate.Set) // Hook invoked when commit is performed
}
// HashDefaults represents a config for using hash-based scheme with
// default settings.
var HashDefaults = &Config{
Preimages: false,
HashDB: hashdb.Defaults,
}
// backend defines the methods needed to access/update trie nodes in different
// state scheme.
type backend interface {
@ -91,22 +99,30 @@ func prepare(diskdb ethdb.Database, config *Config) *Database {
}
}
// NewDatabase initializes the trie database with default settings, namely
// NewDatabase initializes the trie database with default settings, note
// the legacy hash-based scheme is used by default.
func NewDatabase(diskdb ethdb.Database) *Database {
return NewDatabaseWithConfig(diskdb, nil)
}
// NewDatabaseWithConfig initializes the trie database with provided configs.
// The path-based scheme is not activated yet, always initialized with legacy
// hash-based scheme by default.
func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database {
var cleans int
if config != nil && config.Cache != 0 {
cleans = config.Cache * 1024 * 1024
func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
// Sanitize the config and use the default one if it's not specified.
if config == nil {
config = HashDefaults
}
var preimages *preimageStore
if config.Preimages {
preimages = newPreimageStore(diskdb)
}
db := &Database{
config: config,
diskdb: diskdb,
preimages: preimages,
}
if config.HashDB != nil && config.PathDB != nil {
log.Crit("Both 'hash' and 'path' mode are configured")
}
if config.PathDB != nil {
db.backend = pathdb.New(diskdb, config.PathDB)
} else {
db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{})
}
db := prepare(diskdb, config)
db.backend = hashdb.New(diskdb, cleans, mptResolver{})
return db
}
@ -240,3 +256,60 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
}
return hdb.Node(hash)
}
// Recover rollbacks the database to a specified historical point. The state is
// supported as the rollback destination only if it's canonical state and the
// corresponding trie histories are existent. It's only supported by path-based
// database and will return an error for others.
func (db *Database) Recover(target common.Hash) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.Recover(target, &trieLoader{db: db})
}
// Recoverable returns the indicator if the specified state is enabled to be
// recovered. It's only supported by path-based database and will return an
// error for others.
func (db *Database) Recoverable(root common.Hash) (bool, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return false, errors.New("not supported")
}
return pdb.Recoverable(root), nil
}
// Reset wipes all available journal from the persistent database and discard
// all caches and diff layers. Using the given root to create a new disk layer.
// It's only supported by path-based database and will return an error for others.
func (db *Database) Reset(root common.Hash) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.Reset(root)
}
// Journal commits an entire diff hierarchy to disk into a single journal entry.
// This is meant to be used during shutdown to persist the snapshot without
// flattening everything down (bad for reorgs). It's only supported by path-based
// database and will return an error for others.
func (db *Database) Journal(root common.Hash) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.Journal(root)
}
// SetBufferSize sets the node buffer size to the provided value(in bytes).
// It's only supported by path-based database and will return an error for
// others.
func (db *Database) SetBufferSize(size int) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.SetBufferSize(size)
}

2
trie/database_test.go
View File

@ -27,7 +27,7 @@ import (
func newTestDatabase(diskdb ethdb.Database, scheme string) *Database {
db := prepare(diskdb, nil)
if scheme == rawdb.HashScheme {
db.backend = hashdb.New(diskdb, 0, mptResolver{})
db.backend = hashdb.New(diskdb, &hashdb.Config{}, mptResolver{})
} else {
db.backend = pathdb.New(diskdb, &pathdb.Config{}) // disable clean/dirty cache
}

113
trie/iterator_test.go
View File

@ -18,7 +18,6 @@ package trie
import (
"bytes"
"encoding/binary"
"fmt"
"math/rand"
"testing"
@ -27,13 +26,11 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/trie/trienode"
)
func TestEmptyIterator(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
iter := trie.MustNodeIterator(nil)
seen := make(map[string]struct{})
@ -46,7 +43,7 @@ func TestEmptyIterator(t *testing.T) {
}
func TestIterator(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase())
db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db)
vals := []struct{ k, v string }{
{"do", "verb"},
@ -89,7 +86,7 @@ func (k *kv) less(other *kv) bool {
}
func TestIteratorLargeData(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv)
for i := byte(0); i < 255; i++ {
@ -208,7 +205,7 @@ var testdata2 = []kvs{
}
func TestIteratorSeek(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for _, val := range testdata1 {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
@ -249,7 +246,7 @@ func checkIteratorOrder(want []kvs, it *Iterator) error {
}
func TestDifferenceIterator(t *testing.T) {
dba := NewDatabase(rawdb.NewMemoryDatabase())
dba := NewDatabase(rawdb.NewMemoryDatabase(), nil)
triea := NewEmpty(dba)
for _, val := range testdata1 {
triea.MustUpdate([]byte(val.k), []byte(val.v))
@ -258,7 +255,7 @@ func TestDifferenceIterator(t *testing.T) {
dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil)
triea, _ = New(TrieID(rootA), dba)
dbb := NewDatabase(rawdb.NewMemoryDatabase())
dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trieb := NewEmpty(dbb)
for _, val := range testdata2 {
trieb.MustUpdate([]byte(val.k), []byte(val.v))
@ -291,7 +288,7 @@ func TestDifferenceIterator(t *testing.T) {
}
func TestUnionIterator(t *testing.T) {
dba := NewDatabase(rawdb.NewMemoryDatabase())
dba := NewDatabase(rawdb.NewMemoryDatabase(), nil)
triea := NewEmpty(dba)
for _, val := range testdata1 {
triea.MustUpdate([]byte(val.k), []byte(val.v))
@ -300,7 +297,7 @@ func TestUnionIterator(t *testing.T) {
dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil)
triea, _ = New(TrieID(rootA), dba)
dbb := NewDatabase(rawdb.NewMemoryDatabase())
dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trieb := NewEmpty(dbb)
for _, val := range testdata2 {
trieb.MustUpdate([]byte(val.k), []byte(val.v))
@ -344,7 +341,7 @@ func TestUnionIterator(t *testing.T) {
}
func TestIteratorNoDups(t *testing.T) {
tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for _, val := range testdata1 {
tr.MustUpdate([]byte(val.k), []byte(val.v))
}
@ -537,96 +534,6 @@ func TestIteratorNodeBlob(t *testing.T) {
testIteratorNodeBlob(t, rawdb.PathScheme)
}
type loggingDb struct {
getCount uint64
backend ethdb.KeyValueStore
}
func (l *loggingDb) Has(key []byte) (bool, error) {
return l.backend.Has(key)
}
func (l *loggingDb) Get(key []byte) ([]byte, error) {
l.getCount++
return l.backend.Get(key)
}
func (l *loggingDb) Put(key []byte, value []byte) error {
return l.backend.Put(key, value)
}
func (l *loggingDb) Delete(key []byte) error {
return l.backend.Delete(key)
}
func (l *loggingDb) NewBatch() ethdb.Batch {
return l.backend.NewBatch()
}
func (l *loggingDb) NewBatchWithSize(size int) ethdb.Batch {
return l.backend.NewBatchWithSize(size)
}
func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
return l.backend.NewIterator(prefix, start)
}
func (l *loggingDb) NewSnapshot() (ethdb.Snapshot, error) {
return l.backend.NewSnapshot()
}
func (l *loggingDb) Stat(property string) (string, error) {
return l.backend.Stat(property)
}
func (l *loggingDb) Compact(start []byte, limit []byte) error {
return l.backend.Compact(start, limit)
}
func (l *loggingDb) Close() error {
return l.backend.Close()
}
// makeLargeTestTrie create a sample test trie
func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) {
// Create an empty trie
logDb := &loggingDb{0, memorydb.New()}
triedb := NewDatabase(rawdb.NewDatabase(logDb))
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data
for i := 0; i < 10000; i++ {
key := make([]byte, 32)
val := make([]byte, 32)
binary.BigEndian.PutUint64(key, uint64(i))
binary.BigEndian.PutUint64(val, uint64(i))
key = crypto.Keccak256(key)
val = crypto.Keccak256(val)
trie.MustUpdate(key, val)
}
root, nodes, _ := trie.Commit(false)
triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil)
triedb.Commit(root, false)
// Return the generated trie
trie, _ = NewStateTrie(TrieID(root), triedb)
return triedb, trie, logDb
}
// Tests that the node iterator indeed walks over the entire database contents.
func TestNodeIteratorLargeTrie(t *testing.T) {
// Create some arbitrary test trie to iterate
db, trie, logDb := makeLargeTestTrie()
db.Cap(0) // flush everything
// Do a seek operation
trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885"))
// master: 24 get operations
// this pr: 6 get operations
if have, want := logDb.getCount, uint64(6); have != want {
t.Fatalf("Too many lookups during seek, have %d want %d", have, want)
}
}
func testIteratorNodeBlob(t *testing.T, scheme string) {
var (
db = rawdb.NewMemoryDatabase()
@ -700,7 +607,7 @@ func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) {
}
hash = common.BytesToHash(key)
} else {
ok, remain := rawdb.IsAccountTrieNode(key)
ok, remain := rawdb.ResolveAccountTrieNodeKey(key)
if !ok {
return false, nil, common.Hash{}
}

20
trie/proof_test.go
View File

@ -94,7 +94,7 @@ func TestProof(t *testing.T) {
}
func TestOneElementProof(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "k", "v")
for i, prover := range makeProvers(trie) {
proof := prover([]byte("k"))
@ -145,7 +145,7 @@ func TestBadProof(t *testing.T) {
// Tests that missing keys can also be proven. The test explicitly uses a single
// entry trie and checks for missing keys both before and after the single entry.
func TestMissingKeyProof(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "k", "v")
for i, key := range []string{"a", "j", "l", "z"} {
@ -395,7 +395,7 @@ func TestOneElementRangeProof(t *testing.T) {
}
// Test the mini trie with only a single element.
tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
entry := &kv{randBytes(32), randBytes(20), false}
tinyTrie.MustUpdate(entry.k, entry.v)
@ -467,7 +467,7 @@ func TestAllElementsProof(t *testing.T) {
// TestSingleSideRangeProof tests the range starts from zero.
func TestSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
@ -502,7 +502,7 @@ func TestSingleSideRangeProof(t *testing.T) {
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
func TestReverseSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
@ -609,7 +609,7 @@ func TestBadRangeProof(t *testing.T) {
// TestGappedRangeProof focuses on the small trie with embedded nodes.
// If the gapped node is embedded in the trie, it should be detected too.
func TestGappedRangeProof(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv // Sorted entries
for i := byte(0); i < 10; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -683,7 +683,7 @@ func TestSameSideProofs(t *testing.T) {
}
func TestHasRightElement(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
@ -1036,7 +1036,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) {
}
func randomTrie(n int) (*Trie, map[string]*kv) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -1055,7 +1055,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) {
}
func nonRandomTrie(n int) (*Trie, map[string]*kv) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv)
max := uint64(0xffffffffffffffff)
for i := uint64(0); i < uint64(n); i++ {
@ -1080,7 +1080,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) {
common.Hex2Bytes("02"),
common.Hex2Bytes("03"),
}
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i, key := range keys {
trie.MustUpdate(key, vals[i])
}

4
trie/secure_trie_test.go
View File

@ -31,14 +31,14 @@ import (
)
func newEmptySecure() *StateTrie {
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase()))
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase(), nil))
return trie
}
// makeTestStateTrie creates a large enough secure trie for testing.
func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(rawdb.NewMemoryDatabase())
triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data

12
trie/stacktrie_test.go
View File

@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) {
func TestSizeBug(t *testing.T) {
st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) {
func TestEmptyBug(t *testing.T) {
st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) {
func TestValLength56(t *testing.T) {
st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -254,7 +254,7 @@ func TestValLength56(t *testing.T) {
// which causes a lot of node-within-node. This case was found via fuzzing.
func TestUpdateSmallNodes(t *testing.T) {
st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
kvs := []struct {
K string
V string
@ -282,7 +282,7 @@ func TestUpdateSmallNodes(t *testing.T) {
func TestUpdateVariableKeys(t *testing.T) {
t.SkipNow()
st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
kvs := []struct {
K string
V string
@ -351,7 +351,7 @@ func TestStacktrieNotModifyValues(t *testing.T) {
func TestStacktrieSerialization(t *testing.T) {
var (
st = NewStackTrie(nil)
nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
keyB = big.NewInt(1)
keyDelta = big.NewInt(1)
vals [][]byte

4
trie/sync_test.go
View File

@ -109,8 +109,8 @@ type trieElement struct {
// Tests that an empty trie is not scheduled for syncing.
func TestEmptySync(t *testing.T) {
dbA := NewDatabase(rawdb.NewMemoryDatabase())
dbB := NewDatabase(rawdb.NewMemoryDatabase())
dbA := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
dbB := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)
dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)

10
trie/tracer_test.go
View File

@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) {
// Tests if the trie diffs are tracked correctly. Tracer should capture
// all non-leaf dirty nodes, no matter the node is embedded or not.
func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
db := NewDatabase(rawdb.NewMemoryDatabase())
db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db)
// Determine all new nodes are tracked
@ -104,7 +104,7 @@ func TestTrieTracerNoop(t *testing.T) {
}
func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for _, val := range vals {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
@ -128,7 +128,7 @@ func TestAccessList(t *testing.T) {
func testAccessList(t *testing.T, vals []struct{ k, v string }) {
var (
db = NewDatabase(rawdb.NewMemoryDatabase())
db = NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie = NewEmpty(db)
orig = trie.Copy()
)
@ -211,7 +211,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
// Tests origin values won't be tracked in Iterator or Prover
func TestAccessListLeak(t *testing.T) {
var (
db = NewDatabase(rawdb.NewMemoryDatabase())
db = NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie = NewEmpty(db)
)
// Create trie from scratch
@ -262,7 +262,7 @@ func TestAccessListLeak(t *testing.T) {
// in its parent due to the smaller size of the original tree node.
func TestTinyTree(t *testing.T) {
var (
db = NewDatabase(rawdb.NewMemoryDatabase())
db = NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie = NewEmpty(db)
)
for _, val := range tiny {

16
trie/trie_reader.go
View File

@ -20,6 +20,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/triestate"
)
// Reader wraps the Node method of a backing trie store.
@ -83,3 +84,18 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) {
}
return blob, nil
}
// trieLoader implements triestate.TrieLoader for constructing tries.
type trieLoader struct {
db *Database
}
// OpenTrie opens the main account trie.
func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) {
return New(TrieID(root), l.db)
}
// OpenStorageTrie opens the storage trie of an account.
func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) {
return New(StorageTrieID(stateRoot, addrHash, root), l.db)
}

63
trie/trie_test.go
View File

@ -45,7 +45,7 @@ func init() {
}
func TestEmptyTrie(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
res := trie.Hash()
exp := types.EmptyRootHash
if res != exp {
@ -54,7 +54,7 @@ func TestEmptyTrie(t *testing.T) {
}
func TestNull(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
key := make([]byte, 32)
value := []byte("test")
trie.MustUpdate(key, value)
@ -64,8 +64,13 @@ func TestNull(t *testing.T) {
}
func TestMissingRoot(t *testing.T) {
testMissingRoot(t, rawdb.HashScheme)
testMissingRoot(t, rawdb.PathScheme)
}
func testMissingRoot(t *testing.T, scheme string) {
root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
trie, err := New(TrieID(root), NewDatabase(rawdb.NewMemoryDatabase()))
trie, err := New(TrieID(root), newTestDatabase(rawdb.NewMemoryDatabase(), scheme))
if trie != nil {
t.Error("New returned non-nil trie for invalid root")
}
@ -161,7 +166,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) {
}
func TestInsert(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy")
@ -173,7 +178,7 @@ func TestInsert(t *testing.T) {
t.Errorf("case 1: exp %x got %x", exp, root)
}
trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
@ -184,7 +189,7 @@ func TestInsert(t *testing.T) {
}
func TestGet(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase())
db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db)
updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy")
@ -209,7 +214,7 @@ func TestGet(t *testing.T) {
}
func TestDelete(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@ -236,7 +241,7 @@ func TestDelete(t *testing.T) {
}
func TestEmptyValues(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := []struct{ k, v string }{
{"do", "verb"},
@ -260,7 +265,7 @@ func TestEmptyValues(t *testing.T) {
}
func TestReplication(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase())
db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db)
vals := []struct{ k, v string }{
{"do", "verb"},
@ -321,7 +326,7 @@ func TestReplication(t *testing.T) {
}
func TestLargeValue(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99})
trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32))
trie.Hash()
@ -604,7 +609,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000
func benchGet(b *testing.B) {
triedb := NewDatabase(rawdb.NewMemoryDatabase())
triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(triedb)
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
@ -621,7 +626,7 @@ func benchGet(b *testing.B) {
}
func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
k := make([]byte, 32)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
@ -651,7 +656,7 @@ func BenchmarkHash(b *testing.B) {
// entries, then adding N more.
addresses, accounts := makeAccounts(2 * b.N)
// Insert the accounts into the trie and hash it
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
i := 0
for ; i < len(addresses)/2; i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
@ -682,7 +687,7 @@ func BenchmarkCommitAfterHash(b *testing.B) {
func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) {
// Make the random benchmark deterministic
addresses, accounts := makeAccounts(b.N)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -696,7 +701,7 @@ func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) {
func TestTinyTrie(t *testing.T) {
// Create a realistic account trie to hash
_, accounts := makeAccounts(5)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3])
if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root {
t.Errorf("1: got %x, exp %x", root, exp)
@ -709,7 +714,7 @@ func TestTinyTrie(t *testing.T) {
if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
t.Errorf("3: got %x, exp %x", root, exp)
}
checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
it := NewIterator(trie.MustNodeIterator(nil))
for it.Next() {
checktr.MustUpdate(it.Key, it.Value)
@ -722,7 +727,7 @@ func TestTinyTrie(t *testing.T) {
func TestCommitAfterHash(t *testing.T) {
// Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -788,11 +793,17 @@ func (s *spongeDb) Stat(property string) (string, error) { panic("implement
func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") }
func (s *spongeDb) Close() error { return nil }
func (s *spongeDb) Put(key []byte, value []byte) error {
valbrief := value
var (
keybrief = key
valbrief = value
)
if len(keybrief) > 8 {
keybrief = keybrief[:8]
}
if len(valbrief) > 8 {
valbrief = valbrief[:8]
}
s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, key[:8], len(value), valbrief))
s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief))
s.sponge.Write(key)
s.sponge.Write(value)
return nil
@ -830,7 +841,7 @@ func TestCommitSequence(t *testing.T) {
addresses, accounts := makeAccounts(tc.count)
// This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(rawdb.NewDatabase(s))
db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db)
// Fill the trie with elements
for i := 0; i < tc.count; i++ {
@ -861,7 +872,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) {
prng := rand.New(rand.NewSource(int64(i)))
// This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(rawdb.NewDatabase(s))
db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db)
// Fill the trie with elements
for i := 0; i < tc.count; i++ {
@ -893,7 +904,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
prng := rand.New(rand.NewSource(int64(count)))
// This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(rawdb.NewDatabase(s))
db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
@ -952,7 +963,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
// not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do.
func TestCommitSequenceSmallRoot(t *testing.T) {
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(rawdb.NewDatabase(s))
db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
@ -1029,7 +1040,7 @@ func BenchmarkHashFixedSize(b *testing.B) {
func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -1080,7 +1091,7 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) {
func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -1132,7 +1143,7 @@ func BenchmarkDerefRootFixedSize(b *testing.B) {
func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
triedb := NewDatabase(rawdb.NewMemoryDatabase())
triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(triedb)
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])

33
trie/triedb/hashdb/database.go
View File

@ -65,6 +65,20 @@ type ChildResolver interface {
ForEach(node []byte, onChild func(common.Hash))
}
// Config contains the settings for database.
type Config struct {
CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes
}
// Defaults is the default setting for database if it's not specified.
// Notably, clean cache is disabled explicitly,
var Defaults = &Config{
// Explicitly set clean cache size to 0 to avoid creating fastcache,
// otherwise database must be closed when it's no longer needed to
// prevent memory leak.
CleanCacheSize: 0,
}
// Database is an intermediate write layer between the trie data structures and
// the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder.
@ -122,12 +136,13 @@ func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash commo
}
// New initializes the hash-based node database.
func New(diskdb ethdb.Database, size int, resolver ChildResolver) *Database {
// Initialize the clean cache if the specified cache allowance
// is non-zero. Note, the size is in bytes.
func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Database {
if config == nil {
config = Defaults
}
var cleans *fastcache.Cache
if size > 0 {
cleans = fastcache.New(size)
if config.CleanCacheSize > 0 {
cleans = fastcache.New(config.CleanCacheSize)
}
return &Database{
diskdb: diskdb,
@ -621,7 +636,13 @@ func (db *Database) Size() common.StorageSize {
}
// Close closes the trie database and releases all held resources.
func (db *Database) Close() error { return nil }
func (db *Database) Close() error {
if db.cleans != nil {
db.cleans.Reset()
db.cleans = nil
}
return nil
}
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string {

76
trie/triedb/pathdb/database.go
View File

@ -33,8 +33,26 @@ import (
"github.com/ethereum/go-ethereum/trie/triestate"
)
// maxDiffLayers is the maximum diff layers allowed in the layer tree.
const maxDiffLayers = 128
const (
// maxDiffLayers is the maximum diff layers allowed in the layer tree.
maxDiffLayers = 128
// defaultCleanSize is the default memory allowance of clean cache.
defaultCleanSize = 16 * 1024 * 1024
// maxBufferSize is the maximum memory allowance of node buffer.
// Too large nodebuffer will cause the system to pause for a long
// time when write happens. Also, the largest batch that pebble can
// support is 4GB, node will panic if batch size exceeds this limit.
maxBufferSize = 256 * 1024 * 1024
// DefaultBufferSize is the default memory allowance of node buffer
// that aggregates the writes from above until it's flushed into the
// disk. It's meant to be used once the initial sync is finished.
// Do not increase the buffer size arbitrarily, otherwise the system
// pause time will increase when the database writes happen.
DefaultBufferSize = 64 * 1024 * 1024
)
// layer is the interface implemented by all state layers which includes some
// public methods and some additional methods for internal usage.
@ -68,30 +86,33 @@ type layer interface {
// Config contains the settings for database.
type Config struct {
StateLimit uint64 // Number of recent blocks to maintain state history for
CleanSize int // Maximum memory allowance (in bytes) for caching clean nodes
DirtySize int // Maximum memory allowance (in bytes) for caching dirty nodes
ReadOnly bool // Flag whether the database is opened in read only mode.
StateHistory uint64 // Number of recent blocks to maintain state history for
CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes
DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes
ReadOnly bool // Flag whether the database is opened in read only mode.
}
var (
// defaultCleanSize is the default memory allowance of clean cache.
defaultCleanSize = 16 * 1024 * 1024
// defaultBufferSize is the default memory allowance of node buffer
// that aggregates the writes from above until it's flushed into the
// disk. Do not increase the buffer size arbitrarily, otherwise the
// system pause time will increase when the database writes happen.
defaultBufferSize = 128 * 1024 * 1024
)
// sanitize checks the provided user configurations and changes anything that's
// unreasonable or unworkable.
func (c *Config) sanitize() *Config {
conf := *c
if conf.DirtyCacheSize > maxBufferSize {
log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize))
conf.DirtyCacheSize = maxBufferSize
}
return &conf
}
// Defaults contains default settings for Ethereum mainnet.
var Defaults = &Config{
StateLimit: params.FullImmutabilityThreshold,
CleanSize: defaultCleanSize,
DirtySize: defaultBufferSize,
StateHistory: params.FullImmutabilityThreshold,
CleanCacheSize: defaultCleanSize,
DirtyCacheSize: DefaultBufferSize,
}
// ReadOnly is the config in order to open database in read only mode.
var ReadOnly = &Config{ReadOnly: true}
// Database is a multiple-layered structure for maintaining in-memory trie nodes.
// It consists of one persistent base layer backed by a key-value store, on top
// of which arbitrarily many in-memory diff layers are stacked. The memory diffs
@ -123,9 +144,11 @@ func New(diskdb ethdb.Database, config *Config) *Database {
if config == nil {
config = Defaults
}
config = config.sanitize()
db := &Database{
readOnly: config.ReadOnly,
bufferSize: config.DirtySize,
bufferSize: config.DirtyCacheSize,
config: config,
diskdb: diskdb,
}
@ -140,7 +163,7 @@ func New(diskdb ethdb.Database, config *Config) *Database {
// mechanism also ensures that at most one **non-readOnly** database
// is opened at the same time to prevent accidental mutation.
if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly {
freezer, err := rawdb.NewStateHistoryFreezer(ancient, false)
freezer, err := rawdb.NewStateFreezer(ancient, false)
if err != nil {
log.Crit("Failed to open state history freezer", "err", err)
}
@ -344,7 +367,14 @@ func (db *Database) Close() error {
db.lock.Lock()
defer db.lock.Unlock()
// Set the database to read-only mode to prevent all
// following mutations.
db.readOnly = true
// Release the memory held by clean cache.
db.tree.bottom().resetCache()
// Close the attached state history freezer.
if db.freezer == nil {
return nil
}
@ -382,6 +412,10 @@ func (db *Database) SetBufferSize(size int) error {
db.lock.Lock()
defer db.lock.Unlock()
if size > maxBufferSize {
log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize))
size = maxBufferSize
}
db.bufferSize = size
return db.tree.bottom().setBufferSize(db.bufferSize)
}

5
trie/triedb/pathdb/database_test.go
View File

@ -46,7 +46,8 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm
h.Update(key.Bytes(), val)
}
}
return h.Commit(false)
root, nodes, _ := h.Commit(false)
return root, nodes
}
func generateAccount(storageRoot common.Hash) types.StateAccount {
@ -98,7 +99,7 @@ type tester struct {
func newTester(t *testing.T) *tester {
var (
disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
db = New(disk, &Config{CleanSize: 256 * 1024, DirtySize: 256 * 1024})
db = New(disk, &Config{CleanCacheSize: 256 * 1024, DirtyCacheSize: 256 * 1024})
obj = &tester{
db: db,
preimages: make(map[common.Hash]common.Address),

2
trie/triedb/pathdb/difflayer_test.go
View File

@ -29,7 +29,7 @@ import (
func emptyLayer() *diskLayer {
return &diskLayer{
db: New(rawdb.NewMemoryDatabase(), nil),
buffer: newNodeBuffer(defaultBufferSize, nil, 0),
buffer: newNodeBuffer(DefaultBufferSize, nil, 0),
}
}

20
trie/triedb/pathdb/disklayer.go
View File

@ -47,8 +47,8 @@ func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.C
// Initialize a clean cache if the memory allowance is not zero
// or reuse the provided cache if it is not nil (inherited from
// the original disk layer).
if cleans == nil && db.config.CleanSize != 0 {
cleans = fastcache.New(db.config.CleanSize)
if cleans == nil && db.config.CleanCacheSize != 0 {
cleans = fastcache.New(db.config.CleanCacheSize)
}
return &diskLayer{
root: root,
@ -177,7 +177,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
// corresponding states(journal), the stored state history will
// be truncated in the next restart.
if dl.db.freezer != nil {
err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateLimit)
err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateHistory)
if err != nil {
return nil, err
}
@ -276,6 +276,20 @@ func (dl *diskLayer) size() common.StorageSize {
return common.StorageSize(dl.buffer.size)
}
// resetCache releases the memory held by clean cache to prevent memory leak.
func (dl *diskLayer) resetCache() {
dl.lock.RLock()
defer dl.lock.RUnlock()
// Stale disk layer loses the ownership of clean cache.
if dl.stale {
return
}
if dl.cleans != nil {
dl.cleans.Reset()
}
}
// hasher is used to compute the sha256 hash of the provided data.
type hasher struct{ sha crypto.KeccakState }

2
trie/triedb/pathdb/history_test.go
View File

@ -226,7 +226,7 @@ func TestTruncateTailHistories(t *testing.T) {
// openFreezer initializes the freezer instance for storing state histories.
func openFreezer(datadir string, readOnly bool) (*rawdb.ResettableFreezer, error) {
return rawdb.NewStateHistoryFreezer(datadir, readOnly)
return rawdb.NewStateFreezer(datadir, readOnly)
}
func compareSet[k comparable](a, b map[k][]byte) bool {

4
trie/triedb/pathdb/testutils.go
View File

@ -80,7 +80,7 @@ func (h *testHasher) Delete(key []byte) error {
// Commit computes the new hash of the states and returns the set with all
// state changes.
func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) {
var (
nodes = make(map[common.Hash][]byte)
set = trienode.NewNodeSet(h.owner)
@ -108,7 +108,7 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
if root == types.EmptyRootHash && h.root != types.EmptyRootHash {
set.AddNode(nil, trienode.NewDeleted())
}
return root, set
return root, set, nil
}
// hash performs the hash computation upon the provided states.

17
trie/triestate/state.go
View File

@ -43,7 +43,7 @@ type Trie interface {
// Commit the trie and returns a set of dirty nodes generated along with
// the new root hash.
Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet)
Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error)
}
// TrieLoader wraps functions to load tries.
@ -129,7 +129,10 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre
return nil, fmt.Errorf("failed to revert state, err: %w", err)
}
}
root, result := tr.Commit(false)
root, result, err := tr.Commit(false)
if err != nil {
return nil, err
}
if root != prevRoot {
return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root)
}
@ -181,7 +184,10 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error {
return err
}
}
root, result := st.Commit(false)
root, result, err := st.Commit(false)
if err != nil {
return err
}
if root != prev.Root {
return errors.New("failed to reset storage trie")
}
@ -232,7 +238,10 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error {
return err
}
}
root, result := st.Commit(false)
root, result, err := st.Commit(false)
if err != nil {
return err
}
if root != types.EmptyRootHash {
return errors.New("failed to clear storage trie")
}