all: activate pbss as experimental feature (#26274)

* all: activate pbss

* core/rawdb: fix compilation error

* cma, core, eth, les, trie: address comments

* cmd, core, eth, trie: polish code

* core, cmd, eth: address comments

* cmd, core, eth, les, light, tests: address comment

* cmd/utils: shorten log message

* trie/triedb/pathdb: limit node buffer size to 1gb

* cmd/utils: fix opening non-existing db

* cmd/utils: rename flag name

* cmd, core: group chain history flags and fix tests

* core, eth, trie: fix memory leak in snapshot generation

* cmd, eth, internal: deprecate flags

* all: enable state tests for pathdb, fixes

* cmd, core: polish code

* trie/triedb/pathdb: limit the node buffer size to 256mb

---------

Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
rjl493456442 2023-08-11 03:21:36 +08:00 committed by GitHub
parent 5e89ff4d6b
commit 503f1f7ada
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
96 changed files with 2372 additions and 1021 deletions

View File

@ -22,6 +22,7 @@ import (
"fmt" "fmt"
"os" "os"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
@ -64,7 +65,7 @@ func blockTestCmd(ctx *cli.Context) error {
return err return err
} }
for i, test := range tests { for i, test := range tests {
if err := test.Run(false, tracer); err != nil { if err := test.Run(false, rawdb.HashScheme, tracer); err != nil {
return fmt.Errorf("test %v: %w", i, err) return fmt.Errorf("test %v: %w", i, err)
} }
} }

View File

@ -42,6 +42,7 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@ -142,12 +143,23 @@ func runCmd(ctx *cli.Context) error {
gen := readGenesis(ctx.String(GenesisFlag.Name)) gen := readGenesis(ctx.String(GenesisFlag.Name))
genesisConfig = gen genesisConfig = gen
db := rawdb.NewMemoryDatabase() db := rawdb.NewMemoryDatabase()
genesis := gen.MustCommit(db) triedb := trie.NewDatabase(db, &trie.Config{
sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: preimages}) Preimages: preimages,
HashDB: hashdb.Defaults,
})
defer triedb.Close()
genesis := gen.MustCommit(db, triedb)
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ = state.New(genesis.Root(), sdb, nil) statedb, _ = state.New(genesis.Root(), sdb, nil)
chainConfig = gen.Config chainConfig = gen.Config
} else { } else {
sdb := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: preimages}) db := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(db, &trie.Config{
Preimages: preimages,
HashDB: hashdb.Defaults,
})
defer triedb.Close()
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ = state.New(types.EmptyRootHash, sdb, nil) statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
genesisConfig = new(core.Genesis) genesisConfig = new(core.Genesis)
} }

View File

@ -23,7 +23,9 @@ import (
"os" "os"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
@ -104,25 +106,22 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error {
for _, st := range test.Subtests() { for _, st := range test.Subtests() {
// Run the test and aggregate the result // Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
_, s, err := test.Run(st, cfg, false) test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
// print state root for evmlab tracing if err != nil {
if s != nil { // Test failed, mark as so and dump any state to aid debugging
root := s.IntermediateRoot(false) result.Pass, result.Error = false, err.Error()
if dump {
dump := state.RawDump(nil)
result.State = &dump
}
} else {
root := state.IntermediateRoot(false)
result.Root = &root result.Root = &root
if jsonOut { if jsonOut {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
} }
} }
if err != nil { })
// Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error()
if dump && s != nil {
s, _ = state.New(*result.Root, s.Database(), nil)
dump := s.RawDump(nil)
result.State = &dump
}
}
results = append(results, *result) results = append(results, *result)
} }
} }

View File

@ -39,7 +39,6 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/trie"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@ -49,7 +48,10 @@ var (
Name: "init", Name: "init",
Usage: "Bootstrap and initialize a new genesis block", Usage: "Bootstrap and initialize a new genesis block",
ArgsUsage: "<genesisPath>", ArgsUsage: "<genesisPath>",
Flags: flags.Merge([]cli.Flag{utils.CachePreimagesFlag}, utils.DatabasePathFlags), Flags: flags.Merge([]cli.Flag{
utils.CachePreimagesFlag,
utils.StateSchemeFlag,
}, utils.DatabasePathFlags),
Description: ` Description: `
The init command initializes a new genesis block and definition for the network. The init command initializes a new genesis block and definition for the network.
This is a destructive action and changes the network in which you will be This is a destructive action and changes the network in which you will be
@ -94,6 +96,9 @@ if one is set. Otherwise it prints the genesis from the datadir.`,
utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBBucketFlag,
utils.MetricsInfluxDBOrganizationFlag, utils.MetricsInfluxDBOrganizationFlag,
utils.TxLookupLimitFlag, utils.TxLookupLimitFlag,
utils.TransactionHistoryFlag,
utils.StateSchemeFlag,
utils.StateHistoryFlag,
}, utils.DatabasePathFlags), }, utils.DatabasePathFlags),
Description: ` Description: `
The import command imports blocks from an RLP-encoded form. The form can be one file The import command imports blocks from an RLP-encoded form. The form can be one file
@ -110,6 +115,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
Flags: flags.Merge([]cli.Flag{ Flags: flags.Merge([]cli.Flag{
utils.CacheFlag, utils.CacheFlag,
utils.SyncModeFlag, utils.SyncModeFlag,
utils.StateSchemeFlag,
}, utils.DatabasePathFlags), }, utils.DatabasePathFlags),
Description: ` Description: `
Requires a first argument of the file to write to. Requires a first argument of the file to write to.
@ -159,6 +165,7 @@ It's deprecated, please use "geth db export" instead.
utils.IncludeIncompletesFlag, utils.IncludeIncompletesFlag,
utils.StartKeyFlag, utils.StartKeyFlag,
utils.DumpLimitFlag, utils.DumpLimitFlag,
utils.StateSchemeFlag,
}, utils.DatabasePathFlags), }, utils.DatabasePathFlags),
Description: ` Description: `
This command dumps out the state for a given block (or latest, if none provided). This command dumps out the state for a given block (or latest, if none provided).
@ -195,14 +202,15 @@ func initGenesis(ctx *cli.Context) error {
if err != nil { if err != nil {
utils.Fatalf("Failed to open database: %v", err) utils.Fatalf("Failed to open database: %v", err)
} }
triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{ defer chaindb.Close()
Preimages: ctx.Bool(utils.CachePreimagesFlag.Name),
}) triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
defer triedb.Close()
_, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis) _, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis)
if err != nil { if err != nil {
utils.Fatalf("Failed to write genesis block: %v", err) utils.Fatalf("Failed to write genesis block: %v", err)
} }
chaindb.Close()
log.Info("Successfully wrote genesis state", "database", name, "hash", hash) log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
} }
return nil return nil
@ -241,7 +249,7 @@ func dumpGenesis(ctx *cli.Context) error {
if ctx.IsSet(utils.DataDirFlag.Name) { if ctx.IsSet(utils.DataDirFlag.Name) {
utils.Fatalf("no existing datadir at %s", stack.Config().DataDir) utils.Fatalf("no existing datadir at %s", stack.Config().DataDir)
} }
utils.Fatalf("no network preset provided. no exisiting genesis in the default datadir") utils.Fatalf("no network preset provided, no existing genesis in the default datadir")
return nil return nil
} }
@ -465,10 +473,10 @@ func dump(ctx *cli.Context) error {
if err != nil { if err != nil {
return err return err
} }
config := &trie.Config{ triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup
Preimages: true, // always enable preimage lookup defer triedb.Close()
}
state, err := state.New(root, state.NewDatabaseWithConfig(db, config), nil) state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
if err != nil { if err != nil {
return err return err
} }

View File

@ -151,6 +151,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>", ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
Flags: flags.Merge([]cli.Flag{ Flags: flags.Merge([]cli.Flag{
utils.SyncModeFlag, utils.SyncModeFlag,
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags), }, utils.NetworkFlags, utils.DatabasePathFlags),
Description: "This command looks up the specified database key from the database.", Description: "This command looks up the specified database key from the database.",
} }
@ -482,6 +483,9 @@ func dbDumpTrie(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true) db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close() defer db.Close()
triedb := utils.MakeTrieDatabase(ctx, db, false, true)
defer triedb.Close()
var ( var (
state []byte state []byte
storage []byte storage []byte
@ -515,7 +519,7 @@ func dbDumpTrie(ctx *cli.Context) error {
} }
} }
id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage)) id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage))
theTrie, err := trie.New(id, trie.NewDatabase(db)) theTrie, err := trie.New(id, triedb)
if err != nil { if err != nil {
return err return err
} }

View File

@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) {
{ // Can't start pebble on top of leveldb { // Can't start pebble on top of leveldb
initArgs: []string{"--db.engine", "leveldb"}, initArgs: []string{"--db.engine", "leveldb"},
execArgs: []string{"--db.engine", "pebble"}, execArgs: []string{"--db.engine", "pebble"},
execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
}, },
{ // Can't start leveldb on top of pebble { // Can't start leveldb on top of pebble
initArgs: []string{"--db.engine", "pebble"}, initArgs: []string{"--db.engine", "pebble"},
execArgs: []string{"--db.engine", "leveldb"}, execArgs: []string{"--db.engine", "leveldb"},
execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
}, },
{ // Reject invalid backend choice { // Reject invalid backend choice
initArgs: []string{"--db.engine", "mssql"}, initArgs: []string{"--db.engine", "mssql"},

View File

@ -88,6 +88,9 @@ var (
utils.GCModeFlag, utils.GCModeFlag,
utils.SnapshotFlag, utils.SnapshotFlag,
utils.TxLookupLimitFlag, utils.TxLookupLimitFlag,
utils.TransactionHistoryFlag,
utils.StateSchemeFlag,
utils.StateHistoryFlag,
utils.LightServeFlag, utils.LightServeFlag,
utils.LightIngressFlag, utils.LightIngressFlag,
utils.LightEgressFlag, utils.LightEgressFlag,

View File

@ -61,10 +61,7 @@ two version states are available: genesis and the specific one.
The default pruning target is the HEAD-127 state. The default pruning target is the HEAD-127 state.
WARNING: It's necessary to delete the trie clean cache after the pruning. WARNING: it's only supported in hash mode(--state.scheme=hash)".
If you specify another directory for the trie clean cache via "--cache.trie.journal"
during the use of Geth, please also specify it here for correct deletion. Otherwise
the trie clean cache with default directory will be deleted.
`, `,
}, },
{ {
@ -72,7 +69,9 @@ the trie clean cache with default directory will be deleted.
Usage: "Recalculate state hash based on the snapshot for verification", Usage: "Recalculate state hash based on the snapshot for verification",
ArgsUsage: "<root>", ArgsUsage: "<root>",
Action: verifyState, Action: verifyState,
Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Flags: flags.Merge([]cli.Flag{
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: ` Description: `
geth snapshot verify-state <state-root> geth snapshot verify-state <state-root>
will traverse the whole accounts and storages set based on the specified will traverse the whole accounts and storages set based on the specified
@ -107,7 +106,9 @@ information about the specified address.
Usage: "Traverse the state with given root hash and perform quick verification", Usage: "Traverse the state with given root hash and perform quick verification",
ArgsUsage: "<root>", ArgsUsage: "<root>",
Action: traverseState, Action: traverseState,
Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Flags: flags.Merge([]cli.Flag{
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: ` Description: `
geth snapshot traverse-state <state-root> geth snapshot traverse-state <state-root>
will traverse the whole state from the given state root and will abort if any will traverse the whole state from the given state root and will abort if any
@ -122,7 +123,9 @@ It's also usable without snapshot enabled.
Usage: "Traverse the state with given root hash and perform detailed verification", Usage: "Traverse the state with given root hash and perform detailed verification",
ArgsUsage: "<root>", ArgsUsage: "<root>",
Action: traverseRawState, Action: traverseRawState,
Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Flags: flags.Merge([]cli.Flag{
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: ` Description: `
geth snapshot traverse-rawstate <state-root> geth snapshot traverse-rawstate <state-root>
will traverse the whole state from the given root and will abort if any referenced will traverse the whole state from the given root and will abort if any referenced
@ -143,6 +146,7 @@ It's also usable without snapshot enabled.
utils.ExcludeStorageFlag, utils.ExcludeStorageFlag,
utils.StartKeyFlag, utils.StartKeyFlag,
utils.DumpLimitFlag, utils.DumpLimitFlag,
utils.StateSchemeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags), }, utils.NetworkFlags, utils.DatabasePathFlags),
Description: ` Description: `
This command is semantically equivalent to 'geth dump', but uses the snapshots This command is semantically equivalent to 'geth dump', but uses the snapshots
@ -165,6 +169,9 @@ func pruneState(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, false) chaindb := utils.MakeChainDatabase(ctx, stack, false)
defer chaindb.Close() defer chaindb.Close()
if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
log.Crit("Offline pruning is not required for path scheme")
}
prunerconfig := pruner.Config{ prunerconfig := pruner.Config{
Datadir: stack.ResolvePath(""), Datadir: stack.ResolvePath(""),
BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name), BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name),
@ -205,13 +212,16 @@ func verifyState(ctx *cli.Context) error {
log.Error("Failed to load head block") log.Error("Failed to load head block")
return errors.New("no head block") return errors.New("no head block")
} }
snapconfig := snapshot.Config{ triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
snapConfig := snapshot.Config{
CacheSize: 256, CacheSize: 256,
Recovery: false, Recovery: false,
NoBuild: true, NoBuild: true,
AsyncBuild: false, AsyncBuild: false,
} }
snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root()) snaptree, err := snapshot.New(snapConfig, chaindb, triedb, headBlock.Root())
if err != nil { if err != nil {
log.Error("Failed to open snapshot tree", "err", err) log.Error("Failed to open snapshot tree", "err", err)
return err return err
@ -253,6 +263,11 @@ func traverseState(ctx *cli.Context) error {
defer stack.Close() defer stack.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true) chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
headBlock := rawdb.ReadHeadBlock(chaindb) headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil { if headBlock == nil {
log.Error("Failed to load head block") log.Error("Failed to load head block")
@ -277,7 +292,6 @@ func traverseState(ctx *cli.Context) error {
root = headBlock.Root() root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
} }
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb)
if err != nil { if err != nil {
log.Error("Failed to open trie", "root", root, "err", err) log.Error("Failed to open trie", "root", root, "err", err)
@ -353,6 +367,11 @@ func traverseRawState(ctx *cli.Context) error {
defer stack.Close() defer stack.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true) chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
headBlock := rawdb.ReadHeadBlock(chaindb) headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil { if headBlock == nil {
log.Error("Failed to load head block") log.Error("Failed to load head block")
@ -377,7 +396,6 @@ func traverseRawState(ctx *cli.Context) error {
root = headBlock.Root() root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
} }
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb)
if err != nil { if err != nil {
log.Error("Failed to open trie", "root", root, "err", err) log.Error("Failed to open trie", "root", root, "err", err)
@ -398,6 +416,11 @@ func traverseRawState(ctx *cli.Context) error {
log.Error("Failed to open iterator", "root", root, "err", err) log.Error("Failed to open iterator", "root", root, "err", err)
return err return err
} }
reader, err := triedb.Reader(root)
if err != nil {
log.Error("State is non-existent", "root", root)
return nil
}
for accIter.Next(true) { for accIter.Next(true) {
nodes += 1 nodes += 1
node := accIter.Hash() node := accIter.Hash()
@ -405,7 +428,7 @@ func traverseRawState(ctx *cli.Context) error {
// Check the present for non-empty hash node(embedded node doesn't // Check the present for non-empty hash node(embedded node doesn't
// have their own hash). // have their own hash).
if node != (common.Hash{}) { if node != (common.Hash{}) {
blob := rawdb.ReadLegacyTrieNode(chaindb, node) blob, _ := reader.Node(common.Hash{}, accIter.Path(), node)
if len(blob) == 0 { if len(blob) == 0 {
log.Error("Missing trie node(account)", "hash", node) log.Error("Missing trie node(account)", "hash", node)
return errors.New("missing account") return errors.New("missing account")
@ -446,7 +469,7 @@ func traverseRawState(ctx *cli.Context) error {
// Check the presence for non-empty hash node(embedded node doesn't // Check the presence for non-empty hash node(embedded node doesn't
// have their own hash). // have their own hash).
if node != (common.Hash{}) { if node != (common.Hash{}) {
blob := rawdb.ReadLegacyTrieNode(chaindb, node) blob, _ := reader.Node(common.BytesToHash(accIter.LeafKey()), storageIter.Path(), node)
if len(blob) == 0 { if len(blob) == 0 {
log.Error("Missing trie node(storage)", "hash", node) log.Error("Missing trie node(storage)", "hash", node)
return errors.New("missing storage") return errors.New("missing storage")
@ -506,13 +529,16 @@ func dumpState(ctx *cli.Context) error {
if err != nil { if err != nil {
return err return err
} }
triedb := utils.MakeTrieDatabase(ctx, db, false, true)
defer triedb.Close()
snapConfig := snapshot.Config{ snapConfig := snapshot.Config{
CacheSize: 256, CacheSize: 256,
Recovery: false, Recovery: false,
NoBuild: true, NoBuild: true,
AsyncBuild: false, AsyncBuild: false,
} }
snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db), root) snaptree, err := snapshot.New(snapConfig, db, triedb, root)
if err != nil { if err != nil {
return err return err
} }

View File

@ -48,7 +48,7 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth"
ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/catalyst"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/filters"
@ -74,6 +74,9 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
pcsclite "github.com/gballet/go-libpcsclite" pcsclite "github.com/gballet/go-libpcsclite"
gopsutil "github.com/shirou/gopsutil/mem" gopsutil "github.com/shirou/gopsutil/mem"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
@ -218,30 +221,12 @@ var (
} }
defaultSyncMode = ethconfig.Defaults.SyncMode defaultSyncMode = ethconfig.Defaults.SyncMode
SyncModeFlag = &flags.TextMarshalerFlag{
Name: "syncmode",
Usage: `Blockchain sync mode ("snap", "full" or "light")`,
Value: &defaultSyncMode,
Category: flags.EthCategory,
}
GCModeFlag = &cli.StringFlag{
Name: "gcmode",
Usage: `Blockchain garbage collection mode ("full", "archive")`,
Value: "full",
Category: flags.EthCategory,
}
SnapshotFlag = &cli.BoolFlag{ SnapshotFlag = &cli.BoolFlag{
Name: "snapshot", Name: "snapshot",
Usage: `Enables snapshot-database mode (default = enable)`, Usage: `Enables snapshot-database mode (default = enable)`,
Value: true, Value: true,
Category: flags.EthCategory, Category: flags.EthCategory,
} }
TxLookupLimitFlag = &cli.Uint64Flag{
Name: "txlookuplimit",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
Value: ethconfig.Defaults.TxLookupLimit,
Category: flags.EthCategory,
}
LightKDFFlag = &cli.BoolFlag{ LightKDFFlag = &cli.BoolFlag{
Name: "lightkdf", Name: "lightkdf",
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@ -268,6 +253,36 @@ var (
Usage: "Manually specify the Verkle fork timestamp, overriding the bundled setting", Usage: "Manually specify the Verkle fork timestamp, overriding the bundled setting",
Category: flags.EthCategory, Category: flags.EthCategory,
} }
SyncModeFlag = &flags.TextMarshalerFlag{
Name: "syncmode",
Usage: `Blockchain sync mode ("snap", "full" or "light")`,
Value: &defaultSyncMode,
Category: flags.StateCategory,
}
GCModeFlag = &cli.StringFlag{
Name: "gcmode",
Usage: `Blockchain garbage collection mode, only relevant in state.scheme=hash ("full", "archive")`,
Value: "full",
Category: flags.StateCategory,
}
StateSchemeFlag = &cli.StringFlag{
Name: "state.scheme",
Usage: "Scheme to use for storing ethereum state ('hash' or 'path')",
Value: rawdb.HashScheme,
Category: flags.StateCategory,
}
StateHistoryFlag = &cli.Uint64Flag{
Name: "history.state",
Usage: "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)",
Value: ethconfig.Defaults.StateHistory,
Category: flags.StateCategory,
}
TransactionHistoryFlag = &cli.Uint64Flag{
Name: "history.transactions",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.StateCategory,
}
// Light server and client settings // Light server and client settings
LightServeFlag = &cli.IntFlag{ LightServeFlag = &cli.IntFlag{
Name: "light.serve", Name: "light.serve",
@ -1624,13 +1639,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
CheckExclusive(ctx, MainnetFlag, DeveloperFlag, GoerliFlag, SepoliaFlag) CheckExclusive(ctx, MainnetFlag, DeveloperFlag, GoerliFlag, SepoliaFlag)
CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light")
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 {
ctx.Set(TxLookupLimitFlag.Name, "0") // Set configurations from CLI flags
log.Warn("Disable transaction unindexing for archive node")
}
if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 {
log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited")
}
setEtherbase(ctx, cfg) setEtherbase(ctx, cfg)
setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light") setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light")
setTxPool(ctx, &cfg.TxPool) setTxPool(ctx, &cfg.TxPool)
@ -1687,8 +1697,36 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
cfg.Preimages = true cfg.Preimages = true
log.Info("Enabling recording of key preimages since archive mode is used") log.Info("Enabling recording of key preimages since archive mode is used")
} }
if ctx.IsSet(TxLookupLimitFlag.Name) { if ctx.IsSet(StateHistoryFlag.Name) {
cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name) cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
}
// Parse state scheme, abort the process if it's not compatible.
chaindb := tryMakeReadOnlyDatabase(ctx, stack)
scheme, err := ParseStateScheme(ctx, chaindb)
chaindb.Close()
if err != nil {
Fatalf("%v", err)
}
cfg.StateScheme = scheme
// Parse transaction history flag, if user is still using legacy config
// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
log.Warn("The config option 'TxLookupLimit' is deprecated and will be removed, please use 'TransactionHistory'")
cfg.TransactionHistory = cfg.TxLookupLimit
}
if ctx.IsSet(TransactionHistoryFlag.Name) {
cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name)
} else if ctx.IsSet(TxLookupLimitFlag.Name) {
log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions")
cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name)
}
if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
cfg.TransactionHistory = 0
log.Warn("Disabled transaction unindexing for archive node")
}
if ctx.IsSet(LightServeFlag.Name) && cfg.TransactionHistory != 0 {
log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited")
} }
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
@ -1814,15 +1852,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
// Create a new developer genesis block or reuse existing one // Create a new developer genesis block or reuse existing one
cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address) cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address)
if ctx.IsSet(DataDirFlag.Name) { if ctx.IsSet(DataDirFlag.Name) {
// If datadir doesn't exist we need to open db in write-mode chaindb := tryMakeReadOnlyDatabase(ctx, stack)
// so leveldb can create files.
readonly := true
if !common.FileExist(stack.ResolvePath("chaindata")) {
readonly = false
}
// Check if we have an already initialized chain and fall back to
// that if so. Otherwise we need to generate a new genesis spec.
chaindb := MakeChainDatabase(ctx, stack, readonly)
if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
cfg.Genesis = nil // fallback to db content cfg.Genesis = nil // fallback to db content
} }
@ -1930,7 +1960,7 @@ func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, path string) {
if err := rlp.DecodeBytes(rlpBlob, &block); err != nil { if err := rlp.DecodeBytes(rlpBlob, &block); err != nil {
Fatalf("Failed to decode block: %v", err) Fatalf("Failed to decode block: %v", err)
} }
ethcatalyst.RegisterFullSyncTester(stack, eth, &block) catalyst.RegisterFullSyncTester(stack, eth, &block)
log.Info("Registered full-sync tester", "number", block.NumberU64(), "hash", block.Hash()) log.Info("Registered full-sync tester", "number", block.NumberU64(), "hash", block.Hash())
} }
@ -2040,6 +2070,18 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
return chainDb return chainDb
} }
// tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
// or fallback to write mode if the database is not initialized.
func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
// If datadir doesn't exist we need to open db in write-mode
// so database engine can create files.
readonly := true
if !common.FileExist(stack.ResolvePath("chaindata")) {
readonly = false
}
return MakeChainDatabase(ctx, stack, readonly)
}
func IsNetworkPreset(ctx *cli.Context) bool { func IsNetworkPreset(ctx *cli.Context) bool {
for _, flag := range NetworkFlags { for _, flag := range NetworkFlags {
bFlag, _ := flag.(*cli.BoolFlag) bFlag, _ := flag.(*cli.BoolFlag)
@ -2106,6 +2148,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
} }
scheme, err := ParseStateScheme(ctx, chainDb)
if err != nil {
Fatalf("%v", err)
}
cache := &core.CacheConfig{ cache := &core.CacheConfig{
TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, TrieCleanLimit: ethconfig.Defaults.TrieCleanCache,
TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name),
@ -2114,6 +2160,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
TrieTimeLimit: ethconfig.Defaults.TrieTimeout, TrieTimeLimit: ethconfig.Defaults.TrieTimeout,
SnapshotLimit: ethconfig.Defaults.SnapshotCache, SnapshotLimit: ethconfig.Defaults.SnapshotCache,
Preimages: ctx.Bool(CachePreimagesFlag.Name), Preimages: ctx.Bool(CachePreimagesFlag.Name),
StateScheme: scheme,
StateHistory: ctx.Uint64(StateHistoryFlag.Name),
} }
if cache.TrieDirtyDisabled && !cache.Preimages { if cache.TrieDirtyDisabled && !cache.Preimages {
cache.Preimages = true cache.Preimages = true
@ -2158,3 +2206,62 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
} }
return preloads return preloads
} }
// ParseStateScheme resolves scheme identifier from CLI flag. If the provided
// state scheme is not compatible with the one of persistent scheme, an error
// will be returned.
//
// - none: use the scheme consistent with persistent state, or fallback
// to hash-based scheme if state is empty.
// - hash: use hash-based scheme or error out if not compatible with
// persistent state scheme.
// - path: use path-based scheme or error out if not compatible with
// persistent state scheme.
func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) {
// If state scheme is not specified, use the scheme consistent
// with persistent state, or fallback to hash mode if database
// is empty.
stored := rawdb.ReadStateScheme(disk)
if !ctx.IsSet(StateSchemeFlag.Name) {
if stored == "" {
// use default scheme for empty database, flip it when
// path mode is chosen as default
log.Info("State schema set to default", "scheme", "hash")
return rawdb.HashScheme, nil
}
log.Info("State scheme set to already existing", "scheme", stored)
return stored, nil // reuse scheme of persistent scheme
}
// If state scheme is specified, ensure it's compatible with
// persistent state.
scheme := ctx.String(StateSchemeFlag.Name)
if stored == "" || scheme == stored {
log.Info("State scheme set by user", "scheme", scheme)
return scheme, nil
}
return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme)
}
// MakeTrieDatabase constructs a trie database based on the configured scheme.
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database {
config := &trie.Config{
Preimages: preimage,
}
scheme, err := ParseStateScheme(ctx, disk)
if err != nil {
Fatalf("%v", err)
}
if scheme == rawdb.HashScheme {
// Read-only mode is not implemented in hash mode,
// ignore the parameter silently. TODO(rjl493456442)
// please config it if read mode is implemented.
config.HashDB = hashdb.Defaults
return trie.NewDatabase(disk, config)
}
if readOnly {
config.PathDB = pathdb.ReadOnly
} else {
config.PathDB = pathdb.Defaults
}
return trie.NewDatabase(disk, config)
}

View File

@ -19,6 +19,7 @@ package utils
import ( import (
"fmt" "fmt"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/internal/flags"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@ -37,6 +38,7 @@ var DeprecatedFlags = []cli.Flag{
CacheTrieJournalFlag, CacheTrieJournalFlag,
CacheTrieRejournalFlag, CacheTrieRejournalFlag,
LegacyDiscoveryV5Flag, LegacyDiscoveryV5Flag,
TxLookupLimitFlag,
} }
var ( var (
@ -68,6 +70,13 @@ var (
Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)", Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)",
Category: flags.DeprecatedCategory, Category: flags.DeprecatedCategory,
} }
// Deprecated August 2023
TxLookupLimitFlag = &cli.Uint64Flag{
Name: "txlookuplimit",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain) (deprecated, use history.transactions instead)",
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.DeprecatedCategory,
}
) )
// showDeprecated displays deprecated flags that will be soon removed from the codebase. // showDeprecated displays deprecated flags that will be soon removed from the codebase.

View File

@ -35,6 +35,11 @@ import (
// Tests that simple header verification works, for both good and bad blocks. // Tests that simple header verification works, for both good and bad blocks.
func TestHeaderVerification(t *testing.T) { func TestHeaderVerification(t *testing.T) {
testHeaderVerification(t, rawdb.HashScheme)
testHeaderVerification(t, rawdb.PathScheme)
}
func testHeaderVerification(t *testing.T, scheme string) {
// Create a simple chain to verify // Create a simple chain to verify
var ( var (
gspec = &Genesis{Config: params.TestChainConfig} gspec = &Genesis{Config: params.TestChainConfig}
@ -45,7 +50,7 @@ func TestHeaderVerification(t *testing.T) {
headers[i] = block.Header() headers[i] = block.Header()
} }
// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
defer chain.Stop() defer chain.Stop()
for i := 0; i < len(blocks); i++ { for i := 0; i < len(blocks); i++ {

View File

@ -48,6 +48,8 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
) )
@ -128,7 +130,7 @@ const (
) )
// CacheConfig contains the configuration values for the trie database // CacheConfig contains the configuration values for the trie database
// that's resident in a blockchain. // and state snapshot these are resident in a blockchain.
type CacheConfig struct { type CacheConfig struct {
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks
@ -137,11 +139,31 @@ type CacheConfig struct {
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory
Preimages bool // Whether to store preimage of trie key to the disk Preimages bool // Whether to store preimage of trie key to the disk
StateHistory uint64 // Number of blocks from head whose state histories are reserved.
StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top
SnapshotNoBuild bool // Whether the background generation is allowed SnapshotNoBuild bool // Whether the background generation is allowed
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
} }
// triedbConfig derives the configures for trie database.
func (c *CacheConfig) triedbConfig() *trie.Config {
config := &trie.Config{Preimages: c.Preimages}
if c.StateScheme == rawdb.HashScheme {
config.HashDB = &hashdb.Config{
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
}
}
if c.StateScheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{
StateHistory: c.StateHistory,
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024,
}
}
return config
}
// defaultCacheConfig are the default caching values if none are specified by the // defaultCacheConfig are the default caching values if none are specified by the
// user (also used during testing). // user (also used during testing).
var defaultCacheConfig = &CacheConfig{ var defaultCacheConfig = &CacheConfig{
@ -150,6 +172,15 @@ var defaultCacheConfig = &CacheConfig{
TrieTimeLimit: 5 * time.Minute, TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256, SnapshotLimit: 256,
SnapshotWait: true, SnapshotWait: true,
StateScheme: rawdb.HashScheme,
}
// DefaultCacheConfigWithScheme returns a deep copied default cache config with
// a provided trie node scheme.
func DefaultCacheConfigWithScheme(scheme string) *CacheConfig {
config := *defaultCacheConfig
config.StateScheme = scheme
return &config
} }
// BlockChain represents the canonical chain given a database with a genesis // BlockChain represents the canonical chain given a database with a genesis
@ -235,10 +266,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
cacheConfig = defaultCacheConfig cacheConfig = defaultCacheConfig
} }
// Open trie database with provided config // Open trie database with provided config
triedb := trie.NewDatabaseWithConfig(db, &trie.Config{ triedb := trie.NewDatabase(db, cacheConfig.triedbConfig())
Cache: cacheConfig.TrieCleanLimit,
Preimages: cacheConfig.Preimages,
})
// Setup the genesis block, commit the provided genesis specification // Setup the genesis block, commit the provided genesis specification
// to database if the genesis block is not present yet, or load the // to database if the genesis block is not present yet, or load the
// stored one from database. // stored one from database.
@ -612,6 +641,25 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
pivot := rawdb.ReadLastPivotNumber(bc.db) pivot := rawdb.ReadLastPivotNumber(bc.db)
frozen, _ := bc.db.Ancients() frozen, _ := bc.db.Ancients()
// resetState resets the persistent state to genesis if it's not available.
resetState := func() {
// Short circuit if the genesis state is already present.
if bc.HasState(bc.genesisBlock.Root()) {
return
}
// Reset the state database to empty for committing genesis state.
// Note, it should only happen in path-based scheme and Reset function
// is also only call-able in this mode.
if bc.triedb.Scheme() == rawdb.PathScheme {
if err := bc.triedb.Reset(types.EmptyRootHash); err != nil {
log.Crit("Failed to clean state", "err", err) // Shouldn't happen
}
}
// Write genesis state into database.
if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil {
log.Crit("Failed to commit genesis state", "err", err)
}
}
updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) {
// Rewind the blockchain, ensuring we don't end up with a stateless head // Rewind the blockchain, ensuring we don't end up with a stateless head
// block. Note, depth equality is permitted to allow using SetHead as a // block. Note, depth equality is permitted to allow using SetHead as a
@ -621,6 +669,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
if newHeadBlock == nil { if newHeadBlock == nil {
log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash())
newHeadBlock = bc.genesisBlock newHeadBlock = bc.genesisBlock
resetState()
} else { } else {
// Block exists, keep rewinding until we find one with state, // Block exists, keep rewinding until we find one with state,
// keeping rewinding until we exceed the optional threshold // keeping rewinding until we exceed the optional threshold
@ -632,7 +681,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root {
beyondRoot, rootNumber = true, newHeadBlock.NumberU64() beyondRoot, rootNumber = true, newHeadBlock.NumberU64()
} }
if !bc.HasState(newHeadBlock.Root()) { if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) {
log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
if pivot == nil || newHeadBlock.NumberU64() > *pivot { if pivot == nil || newHeadBlock.NumberU64() > *pivot {
parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1)
@ -649,16 +698,12 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
} }
if beyondRoot || newHeadBlock.NumberU64() == 0 { if beyondRoot || newHeadBlock.NumberU64() == 0 {
if newHeadBlock.NumberU64() == 0 { if newHeadBlock.NumberU64() == 0 {
// Recommit the genesis state into disk in case the rewinding destination resetState()
// is genesis block and the relevant state is gone. In the future this } else if !bc.HasState(newHeadBlock.Root()) {
// rewinding destination can be the earliest block stored in the chain // Rewind to a block with recoverable state. If the state is
// if the historical chain pruning is enabled. In that case the logic // missing, run the state recovery here.
// needs to be improved here. if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil {
if !bc.HasState(bc.genesisBlock.Root()) { log.Crit("Failed to rollback state", "err", err) // Shouldn't happen
if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil {
log.Crit("Failed to commit genesis state", "err", err)
}
log.Debug("Recommitted genesis state to disk")
} }
} }
log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
@ -772,7 +817,13 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
if block == nil { if block == nil {
return fmt.Errorf("non existent block [%x..]", hash[:4]) return fmt.Errorf("non existent block [%x..]", hash[:4])
} }
// Reset the trie database with the fresh snap synced state.
root := block.Root() root := block.Root()
if bc.triedb.Scheme() == rawdb.PathScheme {
if err := bc.triedb.Reset(root); err != nil {
return err
}
}
if !bc.HasState(root) { if !bc.HasState(root) {
return fmt.Errorf("non existent state [%x..]", root[:4]) return fmt.Errorf("non existent state [%x..]", root[:4])
} }
@ -937,7 +988,12 @@ func (bc *BlockChain) Stop() {
log.Error("Failed to journal state snapshot", "err", err) log.Error("Failed to journal state snapshot", "err", err)
} }
} }
if bc.triedb.Scheme() == rawdb.PathScheme {
// Ensure that the in-memory trie nodes are journaled to disk properly.
if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
log.Info("Failed to journal in-memory trie nodes", "err", err)
}
} else {
// Ensure the state of a recent block is also stored to disk before exiting. // Ensure the state of a recent block is also stored to disk before exiting.
// We're writing three different states to catch different restart scenarios: // We're writing three different states to catch different restart scenarios:
// - HEAD: So we don't need to reprocess any blocks in the general case // - HEAD: So we don't need to reprocess any blocks in the general case
@ -969,9 +1025,10 @@ func (bc *BlockChain) Stop() {
log.Error("Dangling trie nodes after full cleanup") log.Error("Dangling trie nodes after full cleanup")
} }
} }
// Flush the collected preimages to disk }
if err := bc.stateCache.TrieDB().Close(); err != nil { // Close the trie database, release all the held resources as the last step.
log.Error("Failed to close trie db", "err", err) if err := bc.triedb.Close(); err != nil {
log.Error("Failed to close trie database", "err", err)
} }
log.Info("Blockchain stopped") log.Info("Blockchain stopped")
} }
@ -1341,6 +1398,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
if err != nil { if err != nil {
return err return err
} }
// If node is running in path mode, skip explicit gc operation
// which is unnecessary in this mode.
if bc.triedb.Scheme() == rawdb.PathScheme {
return nil
}
// If we're running an archive node, always flush // If we're running an archive node, always flush
if bc.cacheConfig.TrieDirtyDisabled { if bc.cacheConfig.TrieDirtyDisabled {
return bc.triedb.Commit(root, false) return bc.triedb.Commit(root, false)
@ -1349,8 +1411,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -int64(block.NumberU64())) bc.triegc.Push(root, -int64(block.NumberU64()))
current := block.NumberU64()
// Flush limits are not considered for the first TriesInMemory blocks. // Flush limits are not considered for the first TriesInMemory blocks.
current := block.NumberU64()
if current <= TriesInMemory { if current <= TriesInMemory {
return nil return nil
} }
@ -1936,6 +1998,12 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
) )
parent := it.previous() parent := it.previous()
for parent != nil && !bc.HasState(parent.Root) { for parent != nil && !bc.HasState(parent.Root) {
if bc.stateRecoverable(parent.Root) {
if err := bc.triedb.Recover(parent.Root); err != nil {
return 0, err
}
break
}
hashes = append(hashes, parent.Hash()) hashes = append(hashes, parent.Hash())
numbers = append(numbers, parent.Number.Uint64()) numbers = append(numbers, parent.Number.Uint64())
@ -1992,6 +2060,12 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
parent = block parent = block
) )
for parent != nil && !bc.HasState(parent.Root()) { for parent != nil && !bc.HasState(parent.Root()) {
if bc.stateRecoverable(parent.Root()) {
if err := bc.triedb.Recover(parent.Root()); err != nil {
return common.Hash{}, err
}
break
}
hashes = append(hashes, parent.Hash()) hashes = append(hashes, parent.Hash())
numbers = append(numbers, parent.NumberU64()) numbers = append(numbers, parent.NumberU64())
parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
@ -2393,6 +2467,7 @@ func (bc *BlockChain) maintainTxIndex() {
return return
} }
defer sub.Unsubscribe() defer sub.Unsubscribe()
log.Info("Initialized transaction indexer", "limit", bc.TxLookupLimit())
for { for {
select { select {

View File

@ -293,10 +293,16 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
return bc.HasState(block.Root()) return bc.HasState(block.Root())
} }
// TrieNode retrieves a blob of data associated with a trie node // stateRecoverable checks if the specified state is recoverable.
// either from ephemeral in-memory cache, or from persistent storage. // Note, this function assumes the state is not present, because
func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { // state is not treated as recoverable if it's available, thus
return bc.stateCache.TrieDB().Node(hash) // false will be returned in this case.
func (bc *BlockChain) stateRecoverable(root common.Hash) bool {
if bc.triedb.Scheme() == rawdb.HashScheme {
return false
}
result, _ := bc.triedb.Recoverable(root)
return result
} }
// ContractCodeWithPrefix retrieves a blob of data associated with a contract // ContractCodeWithPrefix retrieves a blob of data associated with a contract

View File

@ -22,6 +22,7 @@ package core
import ( import (
"math/big" "math/big"
"path"
"testing" "testing"
"time" "time"
@ -1749,16 +1750,23 @@ func testLongReorgedSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
} }
func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
testRepairWithScheme(t, tt, snapshots, scheme)
}
}
func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
// It's hard to follow the test case, visualize the input // It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump(true)) // fmt.Println(tt.dump(true))
// Create a temporary persistent database // Create a temporary persistent database
datadir := t.TempDir() datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{ db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir, Directory: datadir,
AncientsDirectory: datadir, AncientsDirectory: ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to create persistent database: %v", err) t.Fatalf("Failed to create persistent database: %v", err)
@ -1777,6 +1785,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
TrieDirtyLimit: 256, TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute, TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, // Disable snapshot by default SnapshotLimit: 0, // Disable snapshot by default
StateScheme: scheme,
} }
) )
defer engine.Close() defer engine.Close()
@ -1806,7 +1815,9 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
t.Fatalf("Failed to import canonical chain start: %v", err) t.Fatalf("Failed to import canonical chain start: %v", err)
} }
if tt.commitBlock > 0 { if tt.commitBlock > 0 {
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false) if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil {
t.Fatalf("Failed to flush trie state: %v", err)
}
if snapshots { if snapshots {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err) t.Fatalf("Failed to flatten snapshots: %v", err)
@ -1828,21 +1839,21 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
rawdb.WriteLastPivotNumber(db, *tt.pivotBlock) rawdb.WriteLastPivotNumber(db, *tt.pivotBlock)
} }
// Pull the plug on the database, simulating a hard crash // Pull the plug on the database, simulating a hard crash
chain.triedb.Close()
db.Close() db.Close()
chain.stopWithoutSaving() chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us // Start a new blockchain back up and see where the repair leads us
db, err = rawdb.Open(rawdb.OpenOptions{ db, err = rawdb.Open(rawdb.OpenOptions{
Directory: datadir, Directory: datadir,
AncientsDirectory: datadir, AncientsDirectory: ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err) t.Fatalf("Failed to reopen persistent database: %v", err)
} }
defer db.Close() defer db.Close()
newChain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) newChain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
@ -1885,17 +1896,22 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
// In this case the snapshot layer of B3 is not created because of existent // In this case the snapshot layer of B3 is not created because of existent
// state. // state.
func TestIssue23496(t *testing.T) { func TestIssue23496(t *testing.T) {
testIssue23496(t, rawdb.HashScheme)
testIssue23496(t, rawdb.PathScheme)
}
func testIssue23496(t *testing.T, scheme string) {
// It's hard to follow the test case, visualize the input // It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// Create a temporary persistent database // Create a temporary persistent database
datadir := t.TempDir() datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{ db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir, Directory: datadir,
AncientsDirectory: datadir, AncientsDirectory: ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to create persistent database: %v", err) t.Fatalf("Failed to create persistent database: %v", err)
} }
@ -1908,15 +1924,8 @@ func TestIssue23496(t *testing.T) {
BaseFee: big.NewInt(params.InitialBaseFee), BaseFee: big.NewInt(params.InitialBaseFee),
} }
engine = ethash.NewFullFaker() engine = ethash.NewFullFaker()
config = &CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256,
SnapshotWait: true,
}
) )
chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to create chain: %v", err) t.Fatalf("Failed to create chain: %v", err)
} }
@ -1929,7 +1938,7 @@ func TestIssue23496(t *testing.T) {
if _, err := chain.InsertChain(blocks[:1]); err != nil { if _, err := chain.InsertChain(blocks[:1]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err) t.Fatalf("Failed to import canonical chain start: %v", err)
} }
chain.stateCache.TrieDB().Commit(blocks[0].Root(), false) chain.triedb.Commit(blocks[0].Root(), false)
// Insert block B2 and commit the snapshot into disk // Insert block B2 and commit the snapshot into disk
if _, err := chain.InsertChain(blocks[1:2]); err != nil { if _, err := chain.InsertChain(blocks[1:2]); err != nil {
@ -1943,7 +1952,7 @@ func TestIssue23496(t *testing.T) {
if _, err := chain.InsertChain(blocks[2:3]); err != nil { if _, err := chain.InsertChain(blocks[2:3]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err) t.Fatalf("Failed to import canonical chain start: %v", err)
} }
chain.stateCache.TrieDB().Commit(blocks[2].Root(), false) chain.triedb.Commit(blocks[2].Root(), false)
// Insert the remaining blocks // Insert the remaining blocks
if _, err := chain.InsertChain(blocks[3:]); err != nil { if _, err := chain.InsertChain(blocks[3:]); err != nil {
@ -1951,20 +1960,21 @@ func TestIssue23496(t *testing.T) {
} }
// Pull the plug on the database, simulating a hard crash // Pull the plug on the database, simulating a hard crash
chain.triedb.Close()
db.Close() db.Close()
chain.stopWithoutSaving() chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us // Start a new blockchain back up and see where the repair leads us
db, err = rawdb.Open(rawdb.OpenOptions{ db, err = rawdb.Open(rawdb.OpenOptions{
Directory: datadir, Directory: datadir,
AncientsDirectory: datadir, AncientsDirectory: ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err) t.Fatalf("Failed to reopen persistent database: %v", err)
} }
defer db.Close() defer db.Close()
chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
@ -1976,8 +1986,12 @@ func TestIssue23496(t *testing.T) {
if head := chain.CurrentSnapBlock(); head.Number.Uint64() != uint64(4) { if head := chain.CurrentSnapBlock(); head.Number.Uint64() != uint64(4) {
t.Errorf("Head fast block mismatch: have %d, want %d", head.Number, uint64(4)) t.Errorf("Head fast block mismatch: have %d, want %d", head.Number, uint64(4))
} }
if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(1) { expHead := uint64(1)
t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(1)) if scheme == rawdb.PathScheme {
expHead = uint64(2)
}
if head := chain.CurrentBlock(); head.Number.Uint64() != expHead {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead)
} }
// Reinsert B2-B4 // Reinsert B2-B4

View File

@ -22,6 +22,7 @@ package core
import ( import (
"fmt" "fmt"
"math/big" "math/big"
"path"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -29,9 +30,13 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
) )
// rewindTest is a test case for chain rollback upon user request. // rewindTest is a test case for chain rollback upon user request.
@ -1949,16 +1954,23 @@ func testLongReorgedSnapSyncingDeepSetHead(t *testing.T, snapshots bool) {
} }
func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
testSetHeadWithScheme(t, tt, snapshots, scheme)
}
}
func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
// It's hard to follow the test case, visualize the input // It's hard to follow the test case, visualize the input
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump(false)) // fmt.Println(tt.dump(false))
// Create a temporary persistent database // Create a temporary persistent database
datadir := t.TempDir() datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{ db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir, Directory: datadir,
AncientsDirectory: datadir, AncientsDirectory: ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to create persistent database: %v", err) t.Fatalf("Failed to create persistent database: %v", err)
@ -1977,6 +1989,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
TrieDirtyLimit: 256, TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute, TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, // Disable snapshot SnapshotLimit: 0, // Disable snapshot
StateScheme: scheme,
} }
) )
if snapshots { if snapshots {
@ -2007,7 +2020,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
t.Fatalf("Failed to import canonical chain start: %v", err) t.Fatalf("Failed to import canonical chain start: %v", err)
} }
if tt.commitBlock > 0 { if tt.commitBlock > 0 {
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false) chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false)
if snapshots { if snapshots {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err) t.Fatalf("Failed to flatten snapshots: %v", err)
@ -2017,13 +2030,17 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil { if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err) t.Fatalf("Failed to import canonical chain tail: %v", err)
} }
// Manually dereference anything not committed to not have to work with 128+ tries // Reopen the trie database without persisting in-memory dirty nodes.
for _, block := range sideblocks { chain.triedb.Close()
chain.stateCache.TrieDB().Dereference(block.Root()) dbconfig := &trie.Config{}
} if scheme == rawdb.PathScheme {
for _, block := range canonblocks { dbconfig.PathDB = pathdb.Defaults
chain.stateCache.TrieDB().Dereference(block.Root()) } else {
dbconfig.HashDB = hashdb.Defaults
} }
chain.triedb = trie.NewDatabase(chain.db, dbconfig)
chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb)
// Force run a freeze cycle // Force run a freeze cycle
type freezer interface { type freezer interface {
Freeze(threshold uint64) error Freeze(threshold uint64) error

View File

@ -24,6 +24,7 @@ import (
"fmt" "fmt"
"math/big" "math/big"
"os" "os"
"path"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -39,6 +40,7 @@ import (
// snapshotTestBasic wraps the common testing fields in the snapshot tests. // snapshotTestBasic wraps the common testing fields in the snapshot tests.
type snapshotTestBasic struct { type snapshotTestBasic struct {
scheme string // Disk scheme used for storing trie nodes
chainBlocks int // Number of blocks to generate for the canonical chain chainBlocks int // Number of blocks to generate for the canonical chain
snapshotBlock uint64 // Block number of the relevant snapshot disk layer snapshotBlock uint64 // Block number of the relevant snapshot disk layer
commitBlock uint64 // Block number for which to commit the state to disk commitBlock uint64 // Block number for which to commit the state to disk
@ -51,6 +53,7 @@ type snapshotTestBasic struct {
// share fields, set in runtime // share fields, set in runtime
datadir string datadir string
ancient string
db ethdb.Database db ethdb.Database
genDb ethdb.Database genDb ethdb.Database
engine consensus.Engine engine consensus.Engine
@ -60,10 +63,11 @@ type snapshotTestBasic struct {
func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) {
// Create a temporary persistent database // Create a temporary persistent database
datadir := t.TempDir() datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{ db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir, Directory: datadir,
AncientsDirectory: datadir, AncientsDirectory: ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to create persistent database: %v", err) t.Fatalf("Failed to create persistent database: %v", err)
@ -75,13 +79,8 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
Config: params.AllEthashProtocolChanges, Config: params.AllEthashProtocolChanges,
} }
engine = ethash.NewFullFaker() engine = ethash.NewFullFaker()
// Snapshot is enabled, the first snapshot is created from the Genesis.
// The snapshot memory allowance is 256MB, it means no snapshot flush
// will happen during the block insertion.
cacheConfig = defaultCacheConfig
) )
chain, err := NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil) chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, nil, engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to create chain: %v", err) t.Fatalf("Failed to create chain: %v", err)
} }
@ -102,7 +101,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
startPoint = point startPoint = point
if basic.commitBlock > 0 && basic.commitBlock == point { if basic.commitBlock > 0 && basic.commitBlock == point {
chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false) chain.TrieDB().Commit(blocks[point-1].Root(), false)
} }
if basic.snapshotBlock > 0 && basic.snapshotBlock == point { if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
// Flushing the entire snap tree into the disk, the // Flushing the entire snap tree into the disk, the
@ -121,6 +120,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
// Set runtime fields // Set runtime fields
basic.datadir = datadir basic.datadir = datadir
basic.ancient = ancient
basic.db = db basic.db = db
basic.genDb = genDb basic.genDb = genDb
basic.engine = engine basic.engine = engine
@ -210,6 +210,7 @@ func (basic *snapshotTestBasic) teardown() {
basic.db.Close() basic.db.Close()
basic.genDb.Close() basic.genDb.Close()
os.RemoveAll(basic.datadir) os.RemoveAll(basic.datadir)
os.RemoveAll(basic.ancient)
} }
// snapshotTest is a test case type for normal snapshot recovery. // snapshotTest is a test case type for normal snapshot recovery.
@ -226,7 +227,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
// Restart the chain normally // Restart the chain normally
chain.Stop() chain.Stop()
newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
@ -235,7 +236,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
snaptest.verify(t, newchain, blocks) snaptest.verify(t, newchain, blocks)
} }
// crashSnapshotTest is a test case type for innormal snapshot recovery. // crashSnapshotTest is a test case type for irregular snapshot recovery.
// It can be used for testing that restart Geth after the crash. // It can be used for testing that restart Geth after the crash.
type crashSnapshotTest struct { type crashSnapshotTest struct {
snapshotTestBasic snapshotTestBasic
@ -251,13 +252,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
db := chain.db db := chain.db
db.Close() db.Close()
chain.stopWithoutSaving() chain.stopWithoutSaving()
chain.triedb.Close()
// Start a new blockchain back up and see where the repair leads us // Start a new blockchain back up and see where the repair leads us
newdb, err := rawdb.Open(rawdb.OpenOptions{ newdb, err := rawdb.Open(rawdb.OpenOptions{
Directory: snaptest.datadir, Directory: snaptest.datadir,
AncientsDirectory: snaptest.datadir, AncientsDirectory: snaptest.ancient,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err) t.Fatalf("Failed to reopen persistent database: %v", err)
} }
@ -267,13 +268,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
// the crash, we do restart twice here: one after the crash and one // the crash, we do restart twice here: one after the crash and one
// after the normal stop. It's used to ensure the broken snapshot // after the normal stop. It's used to ensure the broken snapshot
// can be detected all the time. // can be detected all the time.
newchain, err := NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
newchain.Stop() newchain.Stop()
newchain, err = NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
@ -300,7 +301,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
// Insert blocks without enabling snapshot if gapping is required. // Insert blocks without enabling snapshot if gapping is required.
chain.Stop() chain.Stop()
gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {}) gappedBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {})
// Insert a few more blocks without enabling snapshot // Insert a few more blocks without enabling snapshot
var cacheConfig = &CacheConfig{ var cacheConfig = &CacheConfig{
@ -308,6 +309,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
TrieDirtyLimit: 256, TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute, TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, SnapshotLimit: 0,
StateScheme: snaptest.scheme,
} }
newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
@ -317,7 +319,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
newchain.Stop() newchain.Stop()
// Restart the chain with enabling the snapshot // Restart the chain with enabling the snapshot
newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
@ -345,7 +347,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) {
chain.SetHead(snaptest.setHead) chain.SetHead(snaptest.setHead)
chain.Stop() chain.Stop()
newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
@ -379,22 +381,24 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
TrieDirtyLimit: 256, TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute, TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, SnapshotLimit: 0,
StateScheme: snaptest.scheme,
} }
newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {}) newBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {})
newchain.InsertChain(newBlocks) newchain.InsertChain(newBlocks)
newchain.Stop() newchain.Stop()
// Restart the chain, the wiper should starts working // Restart the chain, the wiper should start working
config = &CacheConfig{ config = &CacheConfig{
TrieCleanLimit: 256, TrieCleanLimit: 256,
TrieDirtyLimit: 256, TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute, TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256, SnapshotLimit: 256,
SnapshotWait: false, // Don't wait rebuild SnapshotWait: false, // Don't wait rebuild
StateScheme: snaptest.scheme,
} }
tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
@ -402,14 +406,15 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
} }
// Simulate the blockchain crash. // Simulate the blockchain crash.
tmp.triedb.Close()
tmp.stopWithoutSaving() tmp.stopWithoutSaving()
newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
if err != nil { if err != nil {
t.Fatalf("Failed to recreate chain: %v", err) t.Fatalf("Failed to recreate chain: %v", err)
} }
defer newchain.Stop()
snaptest.verify(t, newchain, blocks) snaptest.verify(t, newchain, blocks)
newchain.Stop()
} }
// Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
@ -433,8 +438,10 @@ func TestRestartWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8 // Expected head fast block: C8
// Expected head block : C8 // Expected head block : C8
// Expected snapshot disk : G // Expected snapshot disk : G
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &snapshotTest{ test := &snapshotTest{
snapshotTestBasic{ snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 0, snapshotBlock: 0,
commitBlock: 0, commitBlock: 0,
@ -448,6 +455,7 @@ func TestRestartWithNewSnapshot(t *testing.T) {
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case the // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
// chain head should be rewound to the point with available state. And also the // chain head should be rewound to the point with available state. And also the
@ -472,8 +480,10 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8 // Expected head fast block: C8
// Expected head block : G // Expected head block : G
// Expected snapshot disk : C4 // Expected snapshot disk : C4
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &crashSnapshotTest{ test := &crashSnapshotTest{
snapshotTestBasic{ snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 4, snapshotBlock: 4,
commitBlock: 0, commitBlock: 0,
@ -487,6 +497,7 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case the // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
// chain head should be rewound to the point with available state. And also the // chain head should be rewound to the point with available state. And also the
@ -511,8 +522,10 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8 // Expected head fast block: C8
// Expected head block : C2 // Expected head block : C2
// Expected snapshot disk : C4 // Expected snapshot disk : C4
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &crashSnapshotTest{ test := &crashSnapshotTest{
snapshotTestBasic{ snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 4, snapshotBlock: 4,
commitBlock: 2, commitBlock: 2,
@ -526,6 +539,7 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case // Tests a Geth was crashed and restarts with a broken snapshot. In this case
// the chain head should be rewound to the point with available state. And also // the chain head should be rewound to the point with available state. And also
@ -550,21 +564,28 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
// Expected head fast block: C8 // Expected head fast block: C8
// Expected head block : G // Expected head block : G
// Expected snapshot disk : C4 // Expected snapshot disk : C4
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
expHead := uint64(0)
if scheme == rawdb.PathScheme {
expHead = uint64(4)
}
test := &crashSnapshotTest{ test := &crashSnapshotTest{
snapshotTestBasic{ snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 4, snapshotBlock: 4,
commitBlock: 6, commitBlock: 6,
expCanonicalBlocks: 8, expCanonicalBlocks: 8,
expHeadHeader: 8, expHeadHeader: 8,
expHeadFastBlock: 8, expHeadFastBlock: 8,
expHeadBlock: 0, expHeadBlock: expHead,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery expSnapshotBottom: 4, // Last committed disk layer, wait recovery
}, },
} }
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}
// Tests a Geth was running with snapshot enabled. Then restarts without // Tests a Geth was running with snapshot enabled. Then restarts without
// enabling snapshot and after that re-enable the snapshot again. In this // enabling snapshot and after that re-enable the snapshot again. In this
@ -587,8 +608,10 @@ func TestGappedNewSnapshot(t *testing.T) {
// Expected head fast block: C10 // Expected head fast block: C10
// Expected head block : C10 // Expected head block : C10
// Expected snapshot disk : C10 // Expected snapshot disk : C10
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &gappedSnapshotTest{ test := &gappedSnapshotTest{
snapshotTestBasic: snapshotTestBasic{ snapshotTestBasic: snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 0, snapshotBlock: 0,
commitBlock: 0, commitBlock: 0,
@ -603,6 +626,7 @@ func TestGappedNewSnapshot(t *testing.T) {
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}
// Tests the Geth was running with snapshot enabled and resetHead is applied. // Tests the Geth was running with snapshot enabled and resetHead is applied.
// In this case the head is rewound to the target(with state available). After // In this case the head is rewound to the target(with state available). After
@ -625,8 +649,10 @@ func TestSetHeadWithNewSnapshot(t *testing.T) {
// Expected head fast block: C4 // Expected head fast block: C4
// Expected head block : C4 // Expected head block : C4
// Expected snapshot disk : G // Expected snapshot disk : G
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &setHeadSnapshotTest{ test := &setHeadSnapshotTest{
snapshotTestBasic: snapshotTestBasic{ snapshotTestBasic: snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 0, snapshotBlock: 0,
commitBlock: 0, commitBlock: 0,
@ -641,6 +667,7 @@ func TestSetHeadWithNewSnapshot(t *testing.T) {
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}
// Tests the Geth was running with a complete snapshot and then imports a few // Tests the Geth was running with a complete snapshot and then imports a few
// more new blocks on top without enabling the snapshot. After the restart, // more new blocks on top without enabling the snapshot. After the restart,
@ -663,8 +690,10 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
// Expected head fast block: C10 // Expected head fast block: C10
// Expected head block : C8 // Expected head block : C8
// Expected snapshot disk : C10 // Expected snapshot disk : C10
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
test := &wipeCrashSnapshotTest{ test := &wipeCrashSnapshotTest{
snapshotTestBasic: snapshotTestBasic{ snapshotTestBasic: snapshotTestBasic{
scheme: scheme,
chainBlocks: 8, chainBlocks: 8,
snapshotBlock: 4, snapshotBlock: 4,
commitBlock: 0, commitBlock: 0,
@ -679,3 +708,4 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
test.test(t) test.test(t)
test.teardown() test.teardown()
} }
}

File diff suppressed because it is too large Load Diff

View File

@ -283,7 +283,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
} }
blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
chainreader := &fakeChainReader{config: config} chainreader := &fakeChainReader{config: config}
genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) {
b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
b.header = makeHeader(chainreader, parent, statedb, b.engine) b.header = makeHeader(chainreader, parent, statedb, b.engine)
@ -326,19 +326,23 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
if err != nil { if err != nil {
panic(fmt.Sprintf("state write error: %v", err)) panic(fmt.Sprintf("state write error: %v", err))
} }
if err := statedb.Database().TrieDB().Commit(root, false); err != nil { if err = triedb.Commit(root, false); err != nil {
panic(fmt.Sprintf("trie write error: %v", err)) panic(fmt.Sprintf("trie write error: %v", err))
} }
return block, b.receipts return block, b.receipts
} }
return nil, nil return nil, nil
} }
// Forcibly use hash-based state scheme for retaining all nodes in disk.
triedb := trie.NewDatabase(db, trie.HashDefaults)
defer triedb.Close()
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil) statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil)
if err != nil { if err != nil {
panic(err) panic(err)
} }
block, receipt := genblock(i, parent, statedb) block, receipt := genblock(i, parent, triedb, statedb)
blocks[i] = block blocks[i] = block
receipts[i] = receipt receipts[i] = receipt
parent = block parent = block
@ -351,7 +355,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
// then generate chain on top. // then generate chain on top.
func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) {
db := rawdb.NewMemoryDatabase() db := rawdb.NewMemoryDatabase()
_, err := genesis.Commit(db, trie.NewDatabase(db)) triedb := trie.NewDatabase(db, trie.HashDefaults)
defer triedb.Close()
_, err := genesis.Commit(db, triedb)
if err != nil { if err != nil {
panic(err) panic(err)
} }

View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
) )
func TestGenerateWithdrawalChain(t *testing.T) { func TestGenerateWithdrawalChain(t *testing.T) {
@ -74,8 +75,7 @@ func TestGenerateWithdrawalChain(t *testing.T) {
Storage: storage, Storage: storage,
Code: common.Hex2Bytes("600154600354"), Code: common.Hex2Bytes("600154600354"),
} }
genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults))
genesis := gspec.MustCommit(gendb)
chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) { chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) {
tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key) tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key)
@ -146,6 +146,7 @@ func ExampleGenerateChain() {
addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey)
addr3 = crypto.PubkeyToAddress(key3.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey)
db = rawdb.NewMemoryDatabase() db = rawdb.NewMemoryDatabase()
genDb = rawdb.NewMemoryDatabase()
) )
// Ensure that key1 has some funds in the genesis block. // Ensure that key1 has some funds in the genesis block.
@ -153,13 +154,13 @@ func ExampleGenerateChain() {
Config: &params.ChainConfig{HomesteadBlock: new(big.Int)}, Config: &params.ChainConfig{HomesteadBlock: new(big.Int)},
Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}},
} }
genesis := gspec.MustCommit(db) genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults))
// This call generates a chain of 5 blocks. The function runs for // This call generates a chain of 5 blocks. The function runs for
// each block and adds different features to gen based on the // each block and adds different features to gen based on the
// block index. // block index.
signer := types.HomesteadSigner{} signer := types.HomesteadSigner{}
chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 5, func(i int, gen *BlockGen) { chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), genDb, 5, func(i int, gen *BlockGen) {
switch i { switch i {
case 0: case 0:
// In block 1, addr1 sends addr2 some ether. // In block 1, addr1 sends addr2 some ether.
@ -188,7 +189,7 @@ func ExampleGenerateChain() {
}) })
// Import the chain. This runs all block validation rules. // Import the chain. This runs all block validation rules.
blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
defer blockchain.Stop() defer blockchain.Stop()
if i, err := blockchain.InsertChain(chain); err != nil { if i, err := blockchain.InsertChain(chain); err != nil {

View File

@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil { if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err) t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
} }
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err) t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
} }
bc.Stop() bc.Stop()
@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil { if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err) t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
} }
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err) t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
} }
bc.Stop() bc.Stop()
@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil { if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err) t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
} }
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err) t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
} }
blocks, _ = GenerateChain(&proConf, conBc.GetBlockByHash(conBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) blocks, _ = GenerateChain(&proConf, conBc.GetBlockByHash(conBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})
@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil { if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err) t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
} }
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err) t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
} }
blocks, _ = GenerateChain(&conConf, proBc.GetBlockByHash(proBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) blocks, _ = GenerateChain(&conConf, proBc.GetBlockByHash(proBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})

View File

@ -323,10 +323,12 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen
applyOverrides(genesis.Config) applyOverrides(genesis.Config)
return genesis.Config, block.Hash(), nil return genesis.Config, block.Hash(), nil
} }
// We have the genesis block in database(perhaps in ancient database) // The genesis block is present(perhaps in ancient database) while the
// but the corresponding state is missing. // state database is not initialized yet. It can happen that the node
// is initialized with an external ancient store. Commit genesis state
// in this case.
header := rawdb.ReadHeader(db, stored, 0) header := rawdb.ReadHeader(db, stored, 0)
if header.Root != types.EmptyRootHash && !rawdb.HasLegacyTrieNode(db, header.Root) { if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) {
if genesis == nil { if genesis == nil {
genesis = DefaultGenesisBlock() genesis = DefaultGenesisBlock()
} }
@ -526,10 +528,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block
// MustCommit writes the genesis block and state to db, panicking on error. // MustCommit writes the genesis block and state to db, panicking on error.
// The block is committed as the canonical head block. // The block is committed as the canonical head block.
// Note the state changes will be committed in hash-based scheme, use Commit func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block {
// if path-scheme is preferred. block, err := g.Commit(db, triedb)
func (g *Genesis) MustCommit(db ethdb.Database) *types.Block {
block, err := g.Commit(db, trie.NewDatabase(db))
if err != nil { if err != nil {
panic(err) panic(err)
} }

View File

@ -30,18 +30,24 @@ import (
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
) )
func TestInvalidCliqueConfig(t *testing.T) { func TestInvalidCliqueConfig(t *testing.T) {
block := DefaultGoerliGenesisBlock() block := DefaultGoerliGenesisBlock()
block.ExtraData = []byte{} block.ExtraData = []byte{}
db := rawdb.NewMemoryDatabase() db := rawdb.NewMemoryDatabase()
if _, err := block.Commit(db, trie.NewDatabase(db)); err == nil { if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil {
t.Fatal("Expected error on invalid clique config") t.Fatal("Expected error on invalid clique config")
} }
} }
func TestSetupGenesis(t *testing.T) { func TestSetupGenesis(t *testing.T) {
testSetupGenesis(t, rawdb.HashScheme)
testSetupGenesis(t, rawdb.PathScheme)
}
func testSetupGenesis(t *testing.T, scheme string) {
var ( var (
customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50") customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50")
customg = Genesis{ customg = Genesis{
@ -53,6 +59,7 @@ func TestSetupGenesis(t *testing.T) {
oldcustomg = customg oldcustomg = customg
) )
oldcustomg.Config = &params.ChainConfig{HomesteadBlock: big.NewInt(2)} oldcustomg.Config = &params.ChainConfig{HomesteadBlock: big.NewInt(2)}
tests := []struct { tests := []struct {
name string name string
fn func(ethdb.Database) (*params.ChainConfig, common.Hash, error) fn func(ethdb.Database) (*params.ChainConfig, common.Hash, error)
@ -63,7 +70,7 @@ func TestSetupGenesis(t *testing.T) {
{ {
name: "genesis without ChainConfig", name: "genesis without ChainConfig",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
return SetupGenesisBlock(db, trie.NewDatabase(db), new(Genesis)) return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis))
}, },
wantErr: errGenesisNoConfig, wantErr: errGenesisNoConfig,
wantConfig: params.AllEthashProtocolChanges, wantConfig: params.AllEthashProtocolChanges,
@ -71,7 +78,7 @@ func TestSetupGenesis(t *testing.T) {
{ {
name: "no block in DB, genesis == nil", name: "no block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
return SetupGenesisBlock(db, trie.NewDatabase(db), nil) return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil)
}, },
wantHash: params.MainnetGenesisHash, wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig, wantConfig: params.MainnetChainConfig,
@ -79,8 +86,8 @@ func TestSetupGenesis(t *testing.T) {
{ {
name: "mainnet block in DB, genesis == nil", name: "mainnet block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
DefaultGenesisBlock().MustCommit(db) DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, newDbConfig(scheme)))
return SetupGenesisBlock(db, trie.NewDatabase(db), nil) return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil)
}, },
wantHash: params.MainnetGenesisHash, wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig, wantConfig: params.MainnetChainConfig,
@ -88,8 +95,9 @@ func TestSetupGenesis(t *testing.T) {
{ {
name: "custom block in DB, genesis == nil", name: "custom block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
customg.MustCommit(db) tdb := trie.NewDatabase(db, newDbConfig(scheme))
return SetupGenesisBlock(db, trie.NewDatabase(db), nil) customg.Commit(db, tdb)
return SetupGenesisBlock(db, tdb, nil)
}, },
wantHash: customghash, wantHash: customghash,
wantConfig: customg.Config, wantConfig: customg.Config,
@ -97,8 +105,9 @@ func TestSetupGenesis(t *testing.T) {
{ {
name: "custom block in DB, genesis == goerli", name: "custom block in DB, genesis == goerli",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
customg.MustCommit(db) tdb := trie.NewDatabase(db, newDbConfig(scheme))
return SetupGenesisBlock(db, trie.NewDatabase(db), DefaultGoerliGenesisBlock()) customg.Commit(db, tdb)
return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock())
}, },
wantErr: &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash}, wantErr: &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash},
wantHash: params.GoerliGenesisHash, wantHash: params.GoerliGenesisHash,
@ -107,8 +116,9 @@ func TestSetupGenesis(t *testing.T) {
{ {
name: "compatible config in DB", name: "compatible config in DB",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
oldcustomg.MustCommit(db) tdb := trie.NewDatabase(db, newDbConfig(scheme))
return SetupGenesisBlock(db, trie.NewDatabase(db), &customg) oldcustomg.Commit(db, tdb)
return SetupGenesisBlock(db, tdb, &customg)
}, },
wantHash: customghash, wantHash: customghash,
wantConfig: customg.Config, wantConfig: customg.Config,
@ -118,16 +128,17 @@ func TestSetupGenesis(t *testing.T) {
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
// Commit the 'old' genesis block with Homestead transition at #2. // Commit the 'old' genesis block with Homestead transition at #2.
// Advance to block #4, past the homestead transition block of customg. // Advance to block #4, past the homestead transition block of customg.
genesis := oldcustomg.MustCommit(db) tdb := trie.NewDatabase(db, newDbConfig(scheme))
oldcustomg.Commit(db, tdb)
bc, _ := NewBlockChain(db, nil, &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil)
defer bc.Stop() defer bc.Stop()
blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) _, blocks, _ := GenerateChainWithGenesis(&oldcustomg, ethash.NewFaker(), 4, nil)
bc.InsertChain(blocks) bc.InsertChain(blocks)
// This should return a compatibility error. // This should return a compatibility error.
return SetupGenesisBlock(db, trie.NewDatabase(db), &customg) return SetupGenesisBlock(db, tdb, &customg)
}, },
wantHash: customghash, wantHash: customghash,
wantConfig: customg.Config, wantConfig: customg.Config,
@ -175,7 +186,8 @@ func TestGenesisHashes(t *testing.T) {
{DefaultSepoliaGenesisBlock(), params.SepoliaGenesisHash}, {DefaultSepoliaGenesisBlock(), params.SepoliaGenesisHash},
} { } {
// Test via MustCommit // Test via MustCommit
if have := c.genesis.MustCommit(rawdb.NewMemoryDatabase()).Hash(); have != c.want { db := rawdb.NewMemoryDatabase()
if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want {
t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex())
} }
// Test via ToBlock // Test via ToBlock
@ -193,7 +205,7 @@ func TestGenesis_Commit(t *testing.T) {
} }
db := rawdb.NewMemoryDatabase() db := rawdb.NewMemoryDatabase()
genesisBlock := genesis.MustCommit(db) genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
if genesis.Difficulty != nil { if genesis.Difficulty != nil {
t.Fatalf("assumption wrong") t.Fatalf("assumption wrong")
@ -242,3 +254,10 @@ func TestReadWriteGenesisAlloc(t *testing.T) {
} }
} }
} }
func newDbConfig(scheme string) *trie.Config {
if scheme == rawdb.HashScheme {
return trie.HashDefaults
}
return &trie.Config{PathDB: pathdb.Defaults}
}

View File

@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) {
db = rawdb.NewMemoryDatabase() db = rawdb.NewMemoryDatabase()
gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges}
) )
gspec.Commit(db, trie.NewDatabase(db)) gspec.Commit(db, trie.NewDatabase(db, nil))
hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View File

@ -435,12 +435,12 @@ func checkReceiptsRLP(have, want types.Receipts) error {
func TestAncientStorage(t *testing.T) { func TestAncientStorage(t *testing.T) {
// Freezer style fast import the chain. // Freezer style fast import the chain.
frdir := t.TempDir() frdir := t.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
if err != nil { if err != nil {
t.Fatalf("failed to create database with ancient backend") t.Fatalf("failed to create database with ancient backend")
} }
defer db.Close() defer db.Close()
// Create a test block // Create a test block
block := types.NewBlockWithHeader(&types.Header{ block := types.NewBlockWithHeader(&types.Header{
Number: big.NewInt(0), Number: big.NewInt(0),

View File

@ -36,7 +36,7 @@ import (
// //
// Now this scheme is still kept for backward compatibility, and it will be used // Now this scheme is still kept for backward compatibility, and it will be used
// for archive node and some other tries(e.g. light trie). // for archive node and some other tries(e.g. light trie).
const HashScheme = "hashScheme" const HashScheme = "hash"
// PathScheme is the new path-based state scheme with which trie nodes are stored // PathScheme is the new path-based state scheme with which trie nodes are stored
// in the disk with node path as the database key. This scheme will only store one // in the disk with node path as the database key. This scheme will only store one
@ -44,7 +44,7 @@ const HashScheme = "hashScheme"
// is native. At the same time, this scheme will put adjacent trie nodes in the same // is native. At the same time, this scheme will put adjacent trie nodes in the same
// area of the disk with good data locality property. But this scheme needs to rely // area of the disk with good data locality property. But this scheme needs to rely
// on extra state diffs to survive deep reorg. // on extra state diffs to survive deep reorg.
const PathScheme = "pathScheme" const PathScheme = "path"
// hasher is used to compute the sha256 hash of the provided data. // hasher is used to compute the sha256 hash of the provided data.
type hasher struct{ sha crypto.KeccakState } type hasher struct{ sha crypto.KeccakState }
@ -263,3 +263,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
panic(fmt.Sprintf("Unknown scheme %v", scheme)) panic(fmt.Sprintf("Unknown scheme %v", scheme))
} }
} }
// ReadStateScheme reads the state scheme of persistent state, or none
// if the state is not present in database.
func ReadStateScheme(db ethdb.Reader) string {
// Check if state in path-based scheme is present
blob, _ := ReadAccountTrieNode(db, nil)
if len(blob) != 0 {
return PathScheme
}
// In a hash-based scheme, the genesis state is consistently stored
// on the disk. To assess the scheme of the persistent state, it
// suffices to inspect the scheme of the genesis state.
header := ReadHeader(db, ReadCanonicalHash(db, 0), 0)
if header == nil {
return "" // empty datadir
}
blob = ReadLegacyTrieNode(db, header.Root)
if len(blob) == 0 {
return "" // no state in disk
}
return HashScheme
}

View File

@ -58,7 +58,7 @@ const (
stateHistoryStorageData = "storage.data" stateHistoryStorageData = "storage.data"
) )
var stateHistoryFreezerNoSnappy = map[string]bool{ var stateFreezerNoSnappy = map[string]bool{
stateHistoryMeta: true, stateHistoryMeta: true,
stateHistoryAccountIndex: false, stateHistoryAccountIndex: false,
stateHistoryStorageIndex: false, stateHistoryStorageIndex: false,
@ -75,7 +75,7 @@ var (
// freezers the collections of all builtin freezers. // freezers the collections of all builtin freezers.
var freezers = []string{chainFreezerName, stateFreezerName} var freezers = []string{chainFreezerName, stateFreezerName}
// NewStateHistoryFreezer initializes the freezer for state history. // NewStateFreezer initializes the freezer for state history.
func NewStateHistoryFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) { func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateHistoryFreezerNoSnappy) return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
} }

View File

@ -50,36 +50,58 @@ func (info *freezerInfo) size() common.StorageSize {
return total return total
} }
func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
info := freezerInfo{name: name}
for t := range order {
size, err := reader.AncientSize(t)
if err != nil {
return freezerInfo{}, err
}
info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
}
// Retrieve the number of last stored item
ancients, err := reader.Ancients()
if err != nil {
return freezerInfo{}, err
}
info.head = ancients - 1
// Retrieve the number of first stored item
tail, err := reader.Tail()
if err != nil {
return freezerInfo{}, err
}
info.tail = tail
return info, nil
}
// inspectFreezers inspects all freezers registered in the system. // inspectFreezers inspects all freezers registered in the system.
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
var infos []freezerInfo var infos []freezerInfo
for _, freezer := range freezers { for _, freezer := range freezers {
switch freezer { switch freezer {
case chainFreezerName: case chainFreezerName:
// Chain ancient store is a bit special. It's always opened along info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db)
// with the key-value store, inspect the chain store directly.
info := freezerInfo{name: freezer}
// Retrieve storage size of every contained table.
for table := range chainFreezerNoSnappy {
size, err := db.AncientSize(table)
if err != nil { if err != nil {
return nil, err return nil, err
} }
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)}) infos = append(infos, info)
}
// Retrieve the number of last stored item
ancients, err := db.Ancients()
if err != nil {
return nil, err
}
info.head = ancients - 1
// Retrieve the number of first stored item case stateFreezerName:
tail, err := db.Tail() datadir, err := db.AncientDatadir()
if err != nil {
return nil, err
}
f, err := NewStateFreezer(datadir, true)
if err != nil {
return nil, err
}
defer f.Close()
info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f)
if err != nil { if err != nil {
return nil, err return nil, err
} }
info.tail = tail
infos = append(infos, info) infos = append(infos, info)
default: default:

View File

@ -463,7 +463,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
tds stat tds stat
numHashPairings stat numHashPairings stat
hashNumPairings stat hashNumPairings stat
tries stat legacyTries stat
stateLookups stat
accountTries stat
storageTries stat
codes stat codes stat
txLookups stat txLookups stat
accountSnaps stat accountSnaps stat
@ -504,8 +507,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
numHashPairings.Add(size) numHashPairings.Add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
hashNumPairings.Add(size) hashNumPairings.Add(size)
case len(key) == common.HashLength: case IsLegacyTrieNode(key, it.Value()):
tries.Add(size) legacyTries.Add(size)
case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
stateLookups.Add(size)
case IsAccountTrieNode(key):
accountTries.Add(size)
case IsStorageTrieNode(key):
storageTries.Add(size)
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
codes.Add(size) codes.Add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
@ -543,6 +552,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey,
} { } {
if bytes.Equal(key, meta) { if bytes.Equal(key, meta) {
metadata.Add(size) metadata.Add(size)
@ -571,7 +581,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},

View File

@ -273,9 +273,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
return bytes.Equal(key, crypto.Keccak256(val)) return bytes.Equal(key, crypto.Keccak256(val))
} }
// IsAccountTrieNode reports whether a provided database entry is an account // ResolveAccountTrieNodeKey reports whether a provided database entry is an
// trie node in path-based state scheme. // account trie node in path-based state scheme, and returns the resolved
func IsAccountTrieNode(key []byte) (bool, []byte) { // node path if so.
func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
if !bytes.HasPrefix(key, trieNodeAccountPrefix) { if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
return false, nil return false, nil
} }
@ -288,9 +289,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) {
return true, key[len(trieNodeAccountPrefix):] return true, key[len(trieNodeAccountPrefix):]
} }
// IsStorageTrieNode reports whether a provided database entry is a storage // IsAccountTrieNode reports whether a provided database entry is an account
// trie node in path-based state scheme. // trie node in path-based state scheme.
func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { func IsAccountTrieNode(key []byte) bool {
ok, _ := ResolveAccountTrieNodeKey(key)
return ok
}
// ResolveStorageTrieNode reports whether a provided database entry is a storage
// trie node in path-based state scheme, and returns the resolved account hash
// and node path if so.
func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
if !bytes.HasPrefix(key, trieNodeStoragePrefix) { if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
return false, common.Hash{}, nil return false, common.Hash{}, nil
} }
@ -306,3 +315,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
} }
// IsStorageTrieNode reports whether a provided database entry is a storage
// trie node in path-based state scheme.
func IsStorageTrieNode(key []byte) bool {
ok, _, _ := ResolveStorageTrieNode(key)
return ok
}

View File

@ -58,7 +58,7 @@ type Database interface {
// DiskDB returns the underlying key-value disk database. // DiskDB returns the underlying key-value disk database.
DiskDB() ethdb.KeyValueStore DiskDB() ethdb.KeyValueStore
// TrieDB retrieves the low level trie database used for data storage. // TrieDB returns the underlying trie database for managing trie nodes.
TrieDB() *trie.Database TrieDB() *trie.Database
} }
@ -147,7 +147,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
disk: db, disk: db,
codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
triedb: trie.NewDatabaseWithConfig(db, config), triedb: trie.NewDatabase(db, config),
} }
} }

View File

@ -26,9 +26,14 @@ import (
// Tests that the node iterator indeed walks over the entire database contents. // Tests that the node iterator indeed walks over the entire database contents.
func TestNodeIteratorCoverage(t *testing.T) { func TestNodeIteratorCoverage(t *testing.T) {
testNodeIteratorCoverage(t, rawdb.HashScheme)
testNodeIteratorCoverage(t, rawdb.PathScheme)
}
func testNodeIteratorCoverage(t *testing.T, scheme string) {
// Create some arbitrary test state to iterate // Create some arbitrary test state to iterate
db, sdb, root, _ := makeTestState() db, sdb, ndb, root, _ := makeTestState(scheme)
sdb.TrieDB().Commit(root, false) ndb.Commit(root, false)
state, err := New(root, sdb, nil) state, err := New(root, sdb, nil)
if err != nil { if err != nil {
@ -48,7 +53,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
) )
it := db.NewIterator(nil, nil) it := db.NewIterator(nil, nil)
for it.Next() { for it.Next() {
ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) ok, hash := isTrieNode(scheme, it.Key(), it.Value())
if !ok { if !ok {
continue continue
} }
@ -90,11 +95,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) {
return true, common.BytesToHash(key) return true, common.BytesToHash(key)
} }
} else { } else {
ok, _ := rawdb.IsAccountTrieNode(key) ok := rawdb.IsAccountTrieNode(key)
if ok { if ok {
return true, crypto.Keccak256Hash(val) return true, crypto.Keccak256Hash(val)
} }
ok, _, _ = rawdb.IsStorageTrieNode(key) ok = rawdb.IsStorageTrieNode(key)
if ok { if ok {
return true, crypto.Keccak256Hash(val) return true, crypto.Keccak256Hash(val)
} }

View File

@ -85,13 +85,16 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
if headBlock == nil { if headBlock == nil {
return nil, errors.New("failed to load head block") return nil, errors.New("failed to load head block")
} }
// Offline pruning is only supported in legacy hash based scheme.
triedb := trie.NewDatabase(db, trie.HashDefaults)
snapconfig := snapshot.Config{ snapconfig := snapshot.Config{
CacheSize: 256, CacheSize: 256,
Recovery: false, Recovery: false,
NoBuild: true, NoBuild: true,
AsyncBuild: false, AsyncBuild: false,
} }
snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root()) snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
if err != nil { if err != nil {
return nil, err // The relevant snapshot(s) might not exist return nil, err // The relevant snapshot(s) might not exist
} }
@ -361,7 +364,9 @@ func RecoverPruning(datadir string, db ethdb.Database) error {
NoBuild: true, NoBuild: true,
AsyncBuild: false, AsyncBuild: false,
} }
snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root()) // Offline pruning is only supported in legacy hash based scheme.
triedb := trie.NewDatabase(db, trie.HashDefaults)
snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
if err != nil { if err != nil {
return err // The relevant snapshot(s) might not exist return err // The relevant snapshot(s) might not exist
} }
@ -403,7 +408,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil { if genesis == nil {
return errors.New("missing genesis block") return errors.New("missing genesis block")
} }
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db)) t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults))
if err != nil { if err != nil {
return err return err
} }
@ -427,7 +432,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
} }
if acc.Root != types.EmptyRootHash { if acc.Root != types.EmptyRootHash {
id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db)) storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults))
if err != nil { if err != nil {
return err return err
} }

View File

@ -356,7 +356,8 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
var resolver trie.NodeResolver var resolver trie.NodeResolver
if len(result.keys) > 0 { if len(result.keys) > 0 {
mdb := rawdb.NewMemoryDatabase() mdb := rawdb.NewMemoryDatabase()
tdb := trie.NewDatabase(mdb) tdb := trie.NewDatabase(mdb, trie.HashDefaults)
defer tdb.Close()
snapTrie := trie.NewEmpty(tdb) snapTrie := trie.NewEmpty(tdb)
for i, key := range result.keys { for i, key := range result.keys {
snapTrie.Update(key, result.vals[i]) snapTrie.Update(key, result.vals[i])

View File

@ -30,6 +30,8 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/trienode"
"golang.org/x/crypto/sha3" "golang.org/x/crypto/sha3"
) )
@ -45,10 +47,15 @@ func hashData(input []byte) common.Hash {
// Tests that snapshot generation from an empty database. // Tests that snapshot generation from an empty database.
func TestGeneration(t *testing.T) { func TestGeneration(t *testing.T) {
testGeneration(t, rawdb.HashScheme)
testGeneration(t, rawdb.PathScheme)
}
func testGeneration(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
var helper = newHelper() var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@ -79,10 +86,15 @@ func TestGeneration(t *testing.T) {
// Tests that snapshot generation with existent flat state. // Tests that snapshot generation with existent flat state.
func TestGenerateExistentState(t *testing.T) { func TestGenerateExistentState(t *testing.T) {
testGenerateExistentState(t, rawdb.HashScheme)
testGenerateExistentState(t, rawdb.PathScheme)
}
func testGenerateExistentState(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
var helper = newHelper() var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@ -148,9 +160,15 @@ type testHelper struct {
nodes *trienode.MergedNodeSet nodes *trienode.MergedNodeSet
} }
func newHelper() *testHelper { func newHelper(scheme string) *testHelper {
diskdb := rawdb.NewMemoryDatabase() diskdb := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(diskdb) config := &trie.Config{}
if scheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{} // disable caching
} else {
config.HashDB = &hashdb.Config{} // disable caching
}
triedb := trie.NewDatabase(diskdb, config)
accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb) accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb)
return &testHelper{ return &testHelper{
diskdb: diskdb, diskdb: diskdb,
@ -233,7 +251,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
// - extra slots in the middle // - extra slots in the middle
// - extra slots in the end // - extra slots in the end
func TestGenerateExistentStateWithWrongStorage(t *testing.T) { func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
helper := newHelper() testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme)
testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme)
}
func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) {
helper := newHelper(scheme)
// Account one, empty root but non-empty database // Account one, empty root but non-empty database
helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
@ -325,7 +348,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
// - wrong accounts // - wrong accounts
// - extra accounts // - extra accounts
func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper := newHelper() testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme)
testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme)
}
func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) {
helper := newHelper(scheme)
helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@ -380,10 +408,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
// Tests that snapshot generation errors out correctly in case of a missing trie // Tests that snapshot generation errors out correctly in case of a missing trie
// node in the account trie. // node in the account trie.
func TestGenerateCorruptAccountTrie(t *testing.T) { func TestGenerateCorruptAccountTrie(t *testing.T) {
testGenerateCorruptAccountTrie(t, rawdb.HashScheme)
testGenerateCorruptAccountTrie(t, rawdb.PathScheme)
}
func testGenerateCorruptAccountTrie(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// without any storage slots to keep the test smaller. // without any storage slots to keep the test smaller.
helper := newHelper() helper := newHelper(scheme)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@ -391,9 +424,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
// Delete an account trie leaf and ensure the generator chokes // Delete an account trie node and ensure the generator chokes
helper.triedb.Commit(root, false) targetPath := []byte{0xc}
helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7")
rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select { select {
@ -414,11 +449,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
// trie node for a storage trie. It's similar to internal corruption but it is // trie node for a storage trie. It's similar to internal corruption but it is
// handled differently inside the generator. // handled differently inside the generator.
func TestGenerateMissingStorageTrie(t *testing.T) { func TestGenerateMissingStorageTrie(t *testing.T) {
testGenerateMissingStorageTrie(t, rawdb.HashScheme)
testGenerateMissingStorageTrie(t, rawdb.PathScheme)
}
func testGenerateMissingStorageTrie(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
helper := newHelper() var (
acc1 = hashData([]byte("acc-1"))
acc3 = hashData([]byte("acc-3"))
helper = newHelper(scheme)
)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@ -427,8 +470,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
root := helper.Commit() root := helper.Commit()
// Delete a storage trie root and ensure the generator chokes // Delete storage trie root of account one and three.
helper.diskdb.Delete(stRoot.Bytes()) rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme)
rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select { select {
@ -448,10 +492,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
// Tests that snapshot generation errors out correctly in case of a missing trie // Tests that snapshot generation errors out correctly in case of a missing trie
// node in a storage trie. // node in a storage trie.
func TestGenerateCorruptStorageTrie(t *testing.T) { func TestGenerateCorruptStorageTrie(t *testing.T) {
testGenerateCorruptStorageTrie(t, rawdb.HashScheme)
testGenerateCorruptStorageTrie(t, rawdb.PathScheme)
}
func testGenerateCorruptStorageTrie(t *testing.T, scheme string) {
// We can't use statedb to make a test trie (circular dependency), so make // We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts, // a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached. // two of which also has the same 3-slot storage trie attached.
helper := newHelper() helper := newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
@ -461,8 +510,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
root := helper.Commit() root := helper.Commit()
// Delete a storage trie leaf and ensure the generator chokes // Delete a node in the storage trie.
helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) targetPath := []byte{0x4}
targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371")
rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme)
rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme)
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select { select {
@ -481,7 +533,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
// Tests that snapshot generation when an extra account with storage exists in the snap state. // Tests that snapshot generation when an extra account with storage exists in the snap state.
func TestGenerateWithExtraAccounts(t *testing.T) { func TestGenerateWithExtraAccounts(t *testing.T) {
helper := newHelper() testGenerateWithExtraAccounts(t, rawdb.HashScheme)
testGenerateWithExtraAccounts(t, rawdb.PathScheme)
}
func testGenerateWithExtraAccounts(t *testing.T, scheme string) {
helper := newHelper(scheme)
{ {
// Account one in the trie // Account one in the trie
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@ -549,10 +606,15 @@ func enableLogging() {
// Tests that snapshot generation when an extra account with storage exists in the snap state. // Tests that snapshot generation when an extra account with storage exists in the snap state.
func TestGenerateWithManyExtraAccounts(t *testing.T) { func TestGenerateWithManyExtraAccounts(t *testing.T) {
testGenerateWithManyExtraAccounts(t, rawdb.HashScheme)
testGenerateWithManyExtraAccounts(t, rawdb.PathScheme)
}
func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
if false { if false {
enableLogging() enableLogging()
} }
helper := newHelper() helper := newHelper(scheme)
{ {
// Account one in the trie // Account one in the trie
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@ -605,11 +667,16 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
// So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted.
// But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished.
func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme)
testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme)
}
func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) {
accountCheckRange = 3 accountCheckRange = 3
if false { if false {
enableLogging() enableLogging()
} }
helper := newHelper() helper := newHelper(scheme)
{ {
acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
@ -642,11 +709,16 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
// TestGenerateWithMalformedSnapdata tests what happes if we have some junk // TestGenerateWithMalformedSnapdata tests what happes if we have some junk
// in the snapshot database, which cannot be parsed back to an account // in the snapshot database, which cannot be parsed back to an account
func TestGenerateWithMalformedSnapdata(t *testing.T) { func TestGenerateWithMalformedSnapdata(t *testing.T) {
testGenerateWithMalformedSnapdata(t, rawdb.HashScheme)
testGenerateWithMalformedSnapdata(t, rawdb.PathScheme)
}
func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) {
accountCheckRange = 3 accountCheckRange = 3
if false { if false {
enableLogging() enableLogging()
} }
helper := newHelper() helper := newHelper(scheme)
{ {
acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
@ -679,10 +751,15 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
} }
func TestGenerateFromEmptySnap(t *testing.T) { func TestGenerateFromEmptySnap(t *testing.T) {
testGenerateFromEmptySnap(t, rawdb.HashScheme)
testGenerateFromEmptySnap(t, rawdb.PathScheme)
}
func testGenerateFromEmptySnap(t *testing.T, scheme string) {
//enableLogging() //enableLogging()
accountCheckRange = 10 accountCheckRange = 10
storageCheckRange = 20 storageCheckRange = 20
helper := newHelper() helper := newHelper(scheme)
// Add 1K accounts to the trie // Add 1K accounts to the trie
for i := 0; i < 400; i++ { for i := 0; i < 400; i++ {
stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@ -714,8 +791,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
// This hits a case where the snap verification passes, but there are more elements in the trie // This hits a case where the snap verification passes, but there are more elements in the trie
// which we must also add. // which we must also add.
func TestGenerateWithIncompleteStorage(t *testing.T) { func TestGenerateWithIncompleteStorage(t *testing.T) {
testGenerateWithIncompleteStorage(t, rawdb.HashScheme)
testGenerateWithIncompleteStorage(t, rawdb.PathScheme)
}
func testGenerateWithIncompleteStorage(t *testing.T, scheme string) {
storageCheckRange = 4 storageCheckRange = 4
helper := newHelper() helper := newHelper(scheme)
stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
// We add 8 accounts, each one is missing exactly one of the storage slots. This means // We add 8 accounts, each one is missing exactly one of the storage slots. This means
@ -813,7 +895,12 @@ func populateDangling(disk ethdb.KeyValueStore) {
// //
// This test will populate some dangling storages to see if they can be cleaned up. // This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper() testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme)
testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme)
}
func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) {
var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@ -848,7 +935,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
// //
// This test will populate some dangling storages to see if they can be cleaned up. // This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper() testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme)
testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme)
}
func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) {
var helper = newHelper(scheme)
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})

View File

@ -179,7 +179,7 @@ func (test *stateTest) run() bool {
storageList = append(storageList, copy2DSet(states.Storages)) storageList = append(storageList, copy2DSet(states.Storages))
} }
disk = rawdb.NewMemoryDatabase() disk = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit}) tdb = trie.NewDatabase(disk, &trie.Config{OnCommit: onCommit})
sdb = NewDatabaseWithNodeDB(disk, tdb) sdb = NewDatabaseWithNodeDB(disk, tdb)
byzantium = rand.Intn(2) == 0 byzantium = rand.Intn(2) == 0
) )

View File

@ -36,14 +36,19 @@ import (
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
) )
// Tests that updating a state trie does not leak any database writes prior to // Tests that updating a state trie does not leak any database writes prior to
// actually committing the state. // actually committing the state.
func TestUpdateLeaks(t *testing.T) { func TestUpdateLeaks(t *testing.T) {
// Create an empty state database // Create an empty state database
db := rawdb.NewMemoryDatabase() var (
state, _ := New(types.EmptyRootHash, NewDatabase(db), nil) db = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabase(db, nil)
)
state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil)
// Update it with some accounts // Update it with some accounts
for i := byte(0); i < 255; i++ { for i := byte(0); i < 255; i++ {
@ -59,7 +64,7 @@ func TestUpdateLeaks(t *testing.T) {
} }
root := state.IntermediateRoot(false) root := state.IntermediateRoot(false)
if err := state.Database().TrieDB().Commit(root, false); err != nil { if err := tdb.Commit(root, false); err != nil {
t.Errorf("can not commit trie %v to persistent database", root.Hex()) t.Errorf("can not commit trie %v to persistent database", root.Hex())
} }
@ -77,8 +82,10 @@ func TestIntermediateLeaks(t *testing.T) {
// Create two state databases, one transitioning to the final state, the other final from the beginning // Create two state databases, one transitioning to the final state, the other final from the beginning
transDb := rawdb.NewMemoryDatabase() transDb := rawdb.NewMemoryDatabase()
finalDb := rawdb.NewMemoryDatabase() finalDb := rawdb.NewMemoryDatabase()
transState, _ := New(types.EmptyRootHash, NewDatabase(transDb), nil) transNdb := trie.NewDatabase(transDb, nil)
finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb), nil) finalNdb := trie.NewDatabase(finalDb, nil)
transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil)
finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil)
modify := func(state *StateDB, addr common.Address, i, tweak byte) { modify := func(state *StateDB, addr common.Address, i, tweak byte) {
state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
@ -110,7 +117,7 @@ func TestIntermediateLeaks(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to commit transition state: %v", err) t.Fatalf("failed to commit transition state: %v", err)
} }
if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil { if err = transNdb.Commit(transRoot, false); err != nil {
t.Errorf("can not commit trie %v to persistent database", transRoot.Hex()) t.Errorf("can not commit trie %v to persistent database", transRoot.Hex())
} }
@ -118,7 +125,7 @@ func TestIntermediateLeaks(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to commit final state: %v", err) t.Fatalf("failed to commit final state: %v", err)
} }
if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil { if err = finalNdb.Commit(finalRoot, false); err != nil {
t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex()) t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex())
} }
@ -747,9 +754,28 @@ func TestDeleteCreateRevert(t *testing.T) {
// the Commit operation fails with an error // the Commit operation fails with an error
// If we are missing trie nodes, we should not continue writing to the trie // If we are missing trie nodes, we should not continue writing to the trie
func TestMissingTrieNodes(t *testing.T) { func TestMissingTrieNodes(t *testing.T) {
testMissingTrieNodes(t, rawdb.HashScheme)
testMissingTrieNodes(t, rawdb.PathScheme)
}
func testMissingTrieNodes(t *testing.T, scheme string) {
// Create an initial state with a few accounts // Create an initial state with a few accounts
memDb := rawdb.NewMemoryDatabase() var (
db := NewDatabase(memDb) triedb *trie.Database
memDb = rawdb.NewMemoryDatabase()
)
if scheme == rawdb.PathScheme {
triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{
CleanCacheSize: 0,
DirtyCacheSize: 0,
}}) // disable caching
} else {
triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{
CleanCacheSize: 0,
}}) // disable caching
}
db := NewDatabaseWithNodeDB(memDb, triedb)
var root common.Hash var root common.Hash
state, _ := New(types.EmptyRootHash, db, nil) state, _ := New(types.EmptyRootHash, db, nil)
addr := common.BytesToAddress([]byte("so")) addr := common.BytesToAddress([]byte("so"))
@ -762,7 +788,7 @@ func TestMissingTrieNodes(t *testing.T) {
root, _ = state.Commit(0, false) root, _ = state.Commit(0, false)
t.Logf("root: %x", root) t.Logf("root: %x", root)
// force-flush // force-flush
state.Database().TrieDB().Cap(0) triedb.Commit(root, false)
} }
// Create a new state on the old root // Create a new state on the old root
state, _ = New(root, db, nil) state, _ = New(root, db, nil)
@ -969,7 +995,8 @@ func TestFlushOrderDataLoss(t *testing.T) {
// Create a state trie with many accounts and slots // Create a state trie with many accounts and slots
var ( var (
memdb = rawdb.NewMemoryDatabase() memdb = rawdb.NewMemoryDatabase()
statedb = NewDatabase(memdb) triedb = trie.NewDatabase(memdb, nil)
statedb = NewDatabaseWithNodeDB(memdb, triedb)
state, _ = New(types.EmptyRootHash, statedb, nil) state, _ = New(types.EmptyRootHash, statedb, nil)
) )
for a := byte(0); a < 10; a++ { for a := byte(0); a < 10; a++ {
@ -982,11 +1009,11 @@ func TestFlushOrderDataLoss(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to commit state trie: %v", err) t.Fatalf("failed to commit state trie: %v", err)
} }
statedb.TrieDB().Reference(root, common.Hash{}) triedb.Reference(root, common.Hash{})
if err := statedb.TrieDB().Cap(1024); err != nil { if err := triedb.Cap(1024); err != nil {
t.Fatalf("failed to cap trie dirty cache: %v", err) t.Fatalf("failed to cap trie dirty cache: %v", err)
} }
if err := statedb.TrieDB().Commit(root, false); err != nil { if err := triedb.Commit(root, false); err != nil {
t.Fatalf("failed to commit state trie: %v", err) t.Fatalf("failed to commit state trie: %v", err)
} }
// Reopen the state trie from flushed disk and verify it // Reopen the state trie from flushed disk and verify it
@ -1040,7 +1067,7 @@ func TestStateDBTransientStorage(t *testing.T) {
func TestResetObject(t *testing.T) { func TestResetObject(t *testing.T) {
var ( var (
disk = rawdb.NewMemoryDatabase() disk = rawdb.NewMemoryDatabase()
tdb = trie.NewDatabase(disk) tdb = trie.NewDatabase(disk, nil)
db = NewDatabaseWithNodeDB(disk, tdb) db = NewDatabaseWithNodeDB(disk, tdb)
snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash)
state, _ = New(types.EmptyRootHash, db, snaps) state, _ = New(types.EmptyRootHash, db, snaps)

View File

@ -28,6 +28,8 @@ import (
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
) )
// testAccount is the data associated with an account used by the state tests. // testAccount is the data associated with an account used by the state tests.
@ -39,10 +41,17 @@ type testAccount struct {
} }
// makeTestState create a sample test state to test node-wise reconstruction. // makeTestState create a sample test state to test node-wise reconstruction.
func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) {
// Create an empty state // Create an empty state
config := &trie.Config{Preimages: true}
if scheme == rawdb.PathScheme {
config.PathDB = pathdb.Defaults
} else {
config.HashDB = hashdb.Defaults
}
db := rawdb.NewMemoryDatabase() db := rawdb.NewMemoryDatabase()
sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) nodeDb := trie.NewDatabase(db, config)
sdb := NewDatabaseWithNodeDB(db, nodeDb)
state, _ := New(types.EmptyRootHash, sdb, nil) state, _ := New(types.EmptyRootHash, sdb, nil)
// Fill it with some arbitrary data // Fill it with some arbitrary data
@ -67,24 +76,27 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
obj.SetState(hash, hash) obj.SetState(hash, hash)
} }
} }
state.updateStateObject(obj)
accounts = append(accounts, acc) accounts = append(accounts, acc)
} }
root, _ := state.Commit(0, false) root, _ := state.Commit(0, false)
// Return the generated state // Return the generated state
return db, sdb, root, accounts return db, sdb, nodeDb, root, accounts
} }
// checkStateAccounts cross references a reconstructed state with an expected // checkStateAccounts cross references a reconstructed state with an expected
// account array. // account array.
func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) {
var config trie.Config
if scheme == rawdb.PathScheme {
config.PathDB = pathdb.Defaults
}
// Check root availability and state contents // Check root availability and state contents
state, err := New(root, NewDatabase(db), nil) state, err := New(root, NewDatabaseWithConfig(db, &config), nil)
if err != nil { if err != nil {
t.Fatalf("failed to create state trie at %x: %v", root, err) t.Fatalf("failed to create state trie at %x: %v", root, err)
} }
if err := checkStateConsistency(db, root); err != nil { if err := checkStateConsistency(db, scheme, root); err != nil {
t.Fatalf("inconsistent state trie at %x: %v", root, err) t.Fatalf("inconsistent state trie at %x: %v", root, err)
} }
for i, acc := range accounts { for i, acc := range accounts {
@ -101,8 +113,12 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou
} }
// checkStateConsistency checks that all data of a state root is present. // checkStateConsistency checks that all data of a state root is present.
func checkStateConsistency(db ethdb.Database, root common.Hash) error { func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error {
state, err := New(root, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) config := &trie.Config{Preimages: true}
if scheme == rawdb.PathScheme {
config.PathDB = pathdb.Defaults
}
state, err := New(root, NewDatabaseWithConfig(db, config), nil)
if err != nil { if err != nil {
return err return err
} }
@ -114,8 +130,14 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
// Tests that an empty state is not scheduled for syncing. // Tests that an empty state is not scheduled for syncing.
func TestEmptyStateSync(t *testing.T) { func TestEmptyStateSync(t *testing.T) {
db := trie.NewDatabase(rawdb.NewMemoryDatabase()) dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)
sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, db.Scheme()) dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults})
sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbA.Scheme())
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
}
sync = NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbB.Scheme())
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
} }
@ -124,22 +146,28 @@ func TestEmptyStateSync(t *testing.T) {
// Tests that given a root hash, a state can sync iteratively on a single thread, // Tests that given a root hash, a state can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go. // requesting retrieval tasks and returning all of them in one go.
func TestIterativeStateSyncIndividual(t *testing.T) { func TestIterativeStateSyncIndividual(t *testing.T) {
testIterativeStateSync(t, 1, false, false) testIterativeStateSync(t, 1, false, false, rawdb.HashScheme)
testIterativeStateSync(t, 1, false, false, rawdb.PathScheme)
} }
func TestIterativeStateSyncBatched(t *testing.T) { func TestIterativeStateSyncBatched(t *testing.T) {
testIterativeStateSync(t, 100, false, false) testIterativeStateSync(t, 100, false, false, rawdb.HashScheme)
testIterativeStateSync(t, 100, false, false, rawdb.PathScheme)
} }
func TestIterativeStateSyncIndividualFromDisk(t *testing.T) { func TestIterativeStateSyncIndividualFromDisk(t *testing.T) {
testIterativeStateSync(t, 1, true, false) testIterativeStateSync(t, 1, true, false, rawdb.HashScheme)
testIterativeStateSync(t, 1, true, false, rawdb.PathScheme)
} }
func TestIterativeStateSyncBatchedFromDisk(t *testing.T) { func TestIterativeStateSyncBatchedFromDisk(t *testing.T) {
testIterativeStateSync(t, 100, true, false) testIterativeStateSync(t, 100, true, false, rawdb.HashScheme)
testIterativeStateSync(t, 100, true, false, rawdb.PathScheme)
} }
func TestIterativeStateSyncIndividualByPath(t *testing.T) { func TestIterativeStateSyncIndividualByPath(t *testing.T) {
testIterativeStateSync(t, 1, false, true) testIterativeStateSync(t, 1, false, true, rawdb.HashScheme)
testIterativeStateSync(t, 1, false, true, rawdb.PathScheme)
} }
func TestIterativeStateSyncBatchedByPath(t *testing.T) { func TestIterativeStateSyncBatchedByPath(t *testing.T) {
testIterativeStateSync(t, 100, false, true) testIterativeStateSync(t, 100, false, true, rawdb.HashScheme)
testIterativeStateSync(t, 100, false, true, rawdb.PathScheme)
} }
// stateElement represents the element in the state trie(bytecode or trie node). // stateElement represents the element in the state trie(bytecode or trie node).
@ -150,17 +178,17 @@ type stateElement struct {
syncPath trie.SyncPath syncPath trie.SyncPath
} }
func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, scheme string) {
// Create a random state to copy // Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
if commit { if commit {
srcDb.TrieDB().Commit(srcRoot, false) ndb.Commit(srcRoot, false)
} }
srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB()) srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), ndb)
// Create a destination state and sync with the scheduler // Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase() dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
var ( var (
nodeElements []stateElement nodeElements []stateElement
@ -175,9 +203,11 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
}) })
} }
for i := 0; i < len(codes); i++ { for i := 0; i < len(codes); i++ {
codeElements = append(codeElements, stateElement{ codeElements = append(codeElements, stateElement{code: codes[i]})
code: codes[i], }
}) reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
} }
for len(nodeElements)+len(codeElements) > 0 { for len(nodeElements)+len(codeElements) > 0 {
var ( var (
@ -205,7 +235,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err)
} }
id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root)
stTrie, err := trie.New(id, srcDb.TrieDB()) stTrie, err := trie.New(id, ndb)
if err != nil { if err != nil {
t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err)
} }
@ -216,7 +246,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data}
} }
} else { } else {
data, err := srcDb.TrieDB().Node(node.hash) owner, inner := trie.ResolvePath([]byte(node.path))
data, err := reader.Node(owner, inner, node.hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for key %v", []byte(node.path)) t.Fatalf("failed to retrieve node data for key %v", []byte(node.path))
} }
@ -260,18 +291,23 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
copyPreimages(srcDisk, dstDb) copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync // Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts) checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
} }
// Tests that the trie scheduler can correctly reconstruct the state even if only // Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned, and the others sent only later. // partial results are returned, and the others sent only later.
func TestIterativeDelayedStateSync(t *testing.T) { func TestIterativeDelayedStateSync(t *testing.T) {
testIterativeDelayedStateSync(t, rawdb.HashScheme)
testIterativeDelayedStateSync(t, rawdb.PathScheme)
}
func testIterativeDelayedStateSync(t *testing.T, scheme string) {
// Create a random state to copy // Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// Create a destination state and sync with the scheduler // Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase() dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
var ( var (
nodeElements []stateElement nodeElements []stateElement
@ -286,9 +322,11 @@ func TestIterativeDelayedStateSync(t *testing.T) {
}) })
} }
for i := 0; i < len(codes); i++ { for i := 0; i < len(codes); i++ {
codeElements = append(codeElements, stateElement{ codeElements = append(codeElements, stateElement{code: codes[i]})
code: codes[i], }
}) reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
} }
for len(nodeElements)+len(codeElements) > 0 { for len(nodeElements)+len(codeElements) > 0 {
// Sync only half of the scheduled nodes // Sync only half of the scheduled nodes
@ -313,7 +351,8 @@ func TestIterativeDelayedStateSync(t *testing.T) {
if len(nodeElements) > 0 { if len(nodeElements) > 0 {
nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1)
for i, element := range nodeElements[:len(nodeResults)] { for i, element := range nodeElements[:len(nodeResults)] {
data, err := srcDb.TrieDB().Node(element.hash) owner, inner := trie.ResolvePath([]byte(element.path))
data, err := reader.Node(owner, inner, element.hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve contract bytecode for %x", element.code) t.Fatalf("failed to retrieve contract bytecode for %x", element.code)
} }
@ -353,22 +392,28 @@ func TestIterativeDelayedStateSync(t *testing.T) {
copyPreimages(srcDisk, dstDb) copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync // Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts) checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
} }
// Tests that given a root hash, a trie can sync iteratively on a single thread, // Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go, however in a // requesting retrieval tasks and returning all of them in one go, however in a
// random order. // random order.
func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) } func TestIterativeRandomStateSyncIndividual(t *testing.T) {
func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) } testIterativeRandomStateSync(t, 1, rawdb.HashScheme)
testIterativeRandomStateSync(t, 1, rawdb.PathScheme)
}
func TestIterativeRandomStateSyncBatched(t *testing.T) {
testIterativeRandomStateSync(t, 100, rawdb.HashScheme)
testIterativeRandomStateSync(t, 100, rawdb.PathScheme)
}
func testIterativeRandomStateSync(t *testing.T, count int) { func testIterativeRandomStateSync(t *testing.T, count int, scheme string) {
// Create a random state to copy // Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// Create a destination state and sync with the scheduler // Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase() dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
nodeQueue := make(map[string]stateElement) nodeQueue := make(map[string]stateElement)
codeQueue := make(map[common.Hash]struct{}) codeQueue := make(map[common.Hash]struct{})
@ -383,6 +428,10 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
for _, hash := range codes { for _, hash := range codes {
codeQueue[hash] = struct{}{} codeQueue[hash] = struct{}{}
} }
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
}
for len(nodeQueue)+len(codeQueue) > 0 { for len(nodeQueue)+len(codeQueue) > 0 {
// Fetch all the queued nodes in a random order // Fetch all the queued nodes in a random order
if len(codeQueue) > 0 { if len(codeQueue) > 0 {
@ -403,7 +452,8 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
if len(nodeQueue) > 0 { if len(nodeQueue) > 0 {
results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) results := make([]trie.NodeSyncResult, 0, len(nodeQueue))
for path, element := range nodeQueue { for path, element := range nodeQueue {
data, err := srcDb.TrieDB().Node(element.hash) owner, inner := trie.ResolvePath([]byte(element.path))
data, err := reader.Node(owner, inner, element.hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path) t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path)
} }
@ -415,7 +465,6 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
} }
} }
} }
// Feed the retrieved results back and queue new tasks
batch := dstDb.NewBatch() batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil { if err := sched.Commit(batch); err != nil {
t.Fatalf("failed to commit data: %v", err) t.Fatalf("failed to commit data: %v", err)
@ -441,18 +490,23 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
copyPreimages(srcDisk, dstDb) copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync // Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts) checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
} }
// Tests that the trie scheduler can correctly reconstruct the state even if only // Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned (Even those randomly), others sent only later. // partial results are returned (Even those randomly), others sent only later.
func TestIterativeRandomDelayedStateSync(t *testing.T) { func TestIterativeRandomDelayedStateSync(t *testing.T) {
testIterativeRandomDelayedStateSync(t, rawdb.HashScheme)
testIterativeRandomDelayedStateSync(t, rawdb.PathScheme)
}
func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) {
// Create a random state to copy // Create a random state to copy
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// Create a destination state and sync with the scheduler // Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase() dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
nodeQueue := make(map[string]stateElement) nodeQueue := make(map[string]stateElement)
codeQueue := make(map[common.Hash]struct{}) codeQueue := make(map[common.Hash]struct{})
@ -467,6 +521,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
for _, hash := range codes { for _, hash := range codes {
codeQueue[hash] = struct{}{} codeQueue[hash] = struct{}{}
} }
reader, err := ndb.Reader(srcRoot)
if err != nil {
t.Fatalf("state is not existent, %#x", srcRoot)
}
for len(nodeQueue)+len(codeQueue) > 0 { for len(nodeQueue)+len(codeQueue) > 0 {
// Sync only half of the scheduled nodes, even those in random order // Sync only half of the scheduled nodes, even those in random order
if len(codeQueue) > 0 { if len(codeQueue) > 0 {
@ -495,7 +553,8 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
for path, element := range nodeQueue { for path, element := range nodeQueue {
delete(nodeQueue, path) delete(nodeQueue, path)
data, err := srcDb.TrieDB().Node(element.hash) owner, inner := trie.ResolvePath([]byte(element.path))
data, err := reader.Node(owner, inner, element.hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x", element.hash) t.Fatalf("failed to retrieve node data for %x", element.hash)
} }
@ -535,14 +594,19 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
copyPreimages(srcDisk, dstDb) copyPreimages(srcDisk, dstDb)
// Cross check that the two states are in sync // Cross check that the two states are in sync
checkStateAccounts(t, dstDb, srcRoot, srcAccounts) checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
} }
// Tests that at any point in time during a sync, only complete sub-tries are in // Tests that at any point in time during a sync, only complete sub-tries are in
// the database. // the database.
func TestIncompleteStateSync(t *testing.T) { func TestIncompleteStateSync(t *testing.T) {
testIncompleteStateSync(t, rawdb.HashScheme)
testIncompleteStateSync(t, rawdb.PathScheme)
}
func testIncompleteStateSync(t *testing.T, scheme string) {
// Create a random state to copy // Create a random state to copy
db, srcDb, srcRoot, srcAccounts := makeTestState() db, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
// isCodeLookup to save some hashing // isCodeLookup to save some hashing
var isCode = make(map[common.Hash]struct{}) var isCode = make(map[common.Hash]struct{})
@ -555,14 +619,14 @@ func TestIncompleteStateSync(t *testing.T) {
// Create a destination state and sync with the scheduler // Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase() dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
var ( var (
addedCodes []common.Hash addedCodes []common.Hash
addedPaths []string addedPaths []string
addedHashes []common.Hash addedHashes []common.Hash
) )
reader, err := srcDb.TrieDB().Reader(srcRoot) reader, err := ndb.Reader(srcRoot)
if err != nil { if err != nil {
t.Fatalf("state is not available %x", srcRoot) t.Fatalf("state is not available %x", srcRoot)
} }
@ -649,12 +713,11 @@ func TestIncompleteStateSync(t *testing.T) {
for _, node := range addedCodes { for _, node := range addedCodes {
val := rawdb.ReadCode(dstDb, node) val := rawdb.ReadCode(dstDb, node)
rawdb.DeleteCode(dstDb, node) rawdb.DeleteCode(dstDb, node)
if err := checkStateConsistency(dstDb, srcRoot); err == nil { if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil {
t.Errorf("trie inconsistency not caught, missing: %x", node) t.Errorf("trie inconsistency not caught, missing: %x", node)
} }
rawdb.WriteCode(dstDb, node, val) rawdb.WriteCode(dstDb, node, val)
} }
scheme := srcDb.TrieDB().Scheme()
for i, path := range addedPaths { for i, path := range addedPaths {
owner, inner := trie.ResolvePath([]byte(path)) owner, inner := trie.ResolvePath([]byte(path))
hash := addedHashes[i] hash := addedHashes[i]
@ -663,7 +726,7 @@ func TestIncompleteStateSync(t *testing.T) {
t.Error("missing trie node") t.Error("missing trie node")
} }
rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme) rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme)
if err := checkStateConsistency(dstDb, srcRoot); err == nil { if err := checkStateConsistency(dstDb, scheme, srcRoot); err == nil {
t.Errorf("trie inconsistency not caught, missing: %v", path) t.Errorf("trie inconsistency not caught, missing: %v", path)
} }
rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme) rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme)

View File

@ -39,7 +39,7 @@ func TestDeriveSha(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
for len(txs) < 1000 { for len(txs) < 1000 {
exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
got := types.DeriveSha(txs, trie.NewStackTrie(nil)) got := types.DeriveSha(txs, trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) { if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@ -86,7 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) {
b.ResetTimer() b.ResetTimer()
b.ReportAllocs() b.ReportAllocs()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
} }
}) })
@ -107,7 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) {
rndSeed := mrand.Int() rndSeed := mrand.Int()
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
seed := rndSeed + i seed := rndSeed + i
exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) { if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed)) printList(newDummy(seed))
@ -135,7 +135,7 @@ func TestDerivableList(t *testing.T) {
}, },
} }
for i, tc := range tcs[1:] { for i, tc := range tcs[1:] {
exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) { if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp) t.Fatalf("case %d: got %x exp %x", i, got, exp)

View File

@ -418,7 +418,7 @@ func (b *EthAPIBackend) StartMining() error {
} }
func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
return b.eth.StateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
} }
func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) {

View File

@ -322,7 +322,7 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c
if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
} }
triedb := api.eth.BlockChain().StateCache().TrieDB() triedb := api.eth.BlockChain().TrieDB()
oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb) oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb)
if err != nil { if err != nil {

View File

@ -133,9 +133,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Try to recover offline state pruning only in hash-based.
if config.StateScheme == rawdb.HashScheme {
if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil { if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil {
log.Error("Failed to recover state", "error", err) log.Error("Failed to recover state", "error", err)
} }
}
// Transfer mining-related config to the ethash config. // Transfer mining-related config to the ethash config.
chainConfig, err := core.LoadChainConfig(chainDb, config.Genesis) chainConfig, err := core.LoadChainConfig(chainDb, config.Genesis)
if err != nil { if err != nil {
@ -161,7 +164,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
p2pServer: stack.Server(), p2pServer: stack.Server(),
shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb), shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb),
} }
bcVersion := rawdb.ReadDatabaseVersion(chainDb) bcVersion := rawdb.ReadDatabaseVersion(chainDb)
var dbVer = "<nil>" var dbVer = "<nil>"
if bcVersion != nil { if bcVersion != nil {
@ -191,6 +193,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
TrieTimeLimit: config.TrieTimeout, TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache, SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages, Preimages: config.Preimages,
StateHistory: config.StateHistory,
StateScheme: config.StateScheme,
} }
) )
// Override the chain config with provided settings. // Override the chain config with provided settings.
@ -201,7 +205,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if config.OverrideVerkle != nil { if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle overrides.OverrideVerkle = config.OverrideVerkle
} }
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -438,7 +442,7 @@ func (s *Ethereum) StartMining() error {
} }
// If mining is started, we can disable the transaction rejection mechanism // If mining is started, we can disable the transaction rejection mechanism
// introduced to speed sync times. // introduced to speed sync times.
s.handler.acceptTxs.Store(true) s.handler.enableSyncedFeatures()
go s.miner.Start() go s.miner.Start()
} }
@ -471,7 +475,7 @@ func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }
func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) IsListening() bool { return true } // Always listening
func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader }
func (s *Ethereum) Synced() bool { return s.handler.acceptTxs.Load() } func (s *Ethereum) Synced() bool { return s.handler.acceptTxs.Load() }
func (s *Ethereum) SetSynced() { s.handler.acceptTxs.Store(true) } func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() }
func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning }
func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer }
func (s *Ethereum) Merger() *consensus.Merger { return s.merger } func (s *Ethereum) Merger() *consensus.Merger { return s.merger }

View File

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
) )
// Test chain parameters. // Test chain parameters.
@ -43,7 +44,7 @@ var (
Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}},
BaseFee: big.NewInt(params.InitialBaseFee), BaseFee: big.NewInt(params.InitialBaseFee),
} }
testGenesis = testGspec.MustCommit(testDB) testGenesis = testGspec.MustCommit(testDB, trie.NewDatabase(testDB, trie.HashDefaults))
) )
// The common prefix of all test chains: // The common prefix of all test chains:

View File

@ -27,6 +27,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/txpool/legacypool"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
@ -61,6 +62,9 @@ var Defaults = Config{
SyncMode: downloader.SnapSync, SyncMode: downloader.SnapSync,
NetworkId: 1, NetworkId: 1,
TxLookupLimit: 2350000, TxLookupLimit: 2350000,
TransactionHistory: 2350000,
StateHistory: params.FullImmutabilityThreshold,
StateScheme: rawdb.HashScheme,
LightPeers: 100, LightPeers: 100,
DatabaseCache: 512, DatabaseCache: 512,
TrieCleanCache: 154, TrieCleanCache: 154,
@ -97,7 +101,11 @@ type Config struct {
NoPruning bool // Whether to disable pruning and flush everything to disk NoPruning bool // Whether to disable pruning and flush everything to disk
NoPrefetch bool // Whether to disable prefetching and only load state on demand NoPrefetch bool // Whether to disable prefetching and only load state on demand
// Deprecated, use 'TransactionHistory' instead.
TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
// RequiredBlocks is a set of block number -> hash mappings which must be in the // RequiredBlocks is a set of block number -> hash mappings which must be in the
// canonical chain of all remote peers. Setting the option makes geth verify the // canonical chain of all remote peers. Setting the option makes geth verify the

View File

@ -25,6 +25,9 @@ func (c Config) MarshalTOML() (interface{}, error) {
NoPruning bool NoPruning bool
NoPrefetch bool NoPrefetch bool
TxLookupLimit uint64 `toml:",omitempty"` TxLookupLimit uint64 `toml:",omitempty"`
TransactionHistory uint64 `toml:",omitempty"`
StateHistory uint64 `toml:",omitempty"`
StateScheme string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"` RequiredBlocks map[uint64]common.Hash `toml:"-"`
LightServ int `toml:",omitempty"` LightServ int `toml:",omitempty"`
LightIngress int `toml:",omitempty"` LightIngress int `toml:",omitempty"`
@ -63,6 +66,9 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.NoPruning = c.NoPruning enc.NoPruning = c.NoPruning
enc.NoPrefetch = c.NoPrefetch enc.NoPrefetch = c.NoPrefetch
enc.TxLookupLimit = c.TxLookupLimit enc.TxLookupLimit = c.TxLookupLimit
enc.TransactionHistory = c.TransactionHistory
enc.StateHistory = c.StateHistory
enc.StateScheme = c.StateScheme
enc.RequiredBlocks = c.RequiredBlocks enc.RequiredBlocks = c.RequiredBlocks
enc.LightServ = c.LightServ enc.LightServ = c.LightServ
enc.LightIngress = c.LightIngress enc.LightIngress = c.LightIngress
@ -105,6 +111,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
NoPruning *bool NoPruning *bool
NoPrefetch *bool NoPrefetch *bool
TxLookupLimit *uint64 `toml:",omitempty"` TxLookupLimit *uint64 `toml:",omitempty"`
TransactionHistory *uint64 `toml:",omitempty"`
StateHistory *uint64 `toml:",omitempty"`
StateScheme *string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"` RequiredBlocks map[uint64]common.Hash `toml:"-"`
LightServ *int `toml:",omitempty"` LightServ *int `toml:",omitempty"`
LightIngress *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"`
@ -162,6 +171,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.TxLookupLimit != nil { if dec.TxLookupLimit != nil {
c.TxLookupLimit = *dec.TxLookupLimit c.TxLookupLimit = *dec.TxLookupLimit
} }
if dec.TransactionHistory != nil {
c.TransactionHistory = *dec.TransactionHistory
}
if dec.StateHistory != nil {
c.StateHistory = *dec.StateHistory
}
if dec.StateScheme != nil {
c.StateScheme = *dec.StateScheme
}
if dec.RequiredBlocks != nil { if dec.RequiredBlocks != nil {
c.RequiredBlocks = dec.RequiredBlocks c.RequiredBlocks = dec.RequiredBlocks
} }

View File

@ -44,7 +44,7 @@ var (
Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}},
BaseFee: big.NewInt(params.InitialBaseFee), BaseFee: big.NewInt(params.InitialBaseFee),
} }
genesis = gspec.MustCommit(testdb) genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults))
unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil))
) )

View File

@ -86,7 +86,7 @@ func BenchmarkFilters(b *testing.B) {
// The test txs are not properly signed, can't simply create a chain // The test txs are not properly signed, can't simply create a chain
// and then import blocks. TODO(rjl493456442) try to get rid of the // and then import blocks. TODO(rjl493456442) try to get rid of the
// manual database writes. // manual database writes.
gspec.MustCommit(db) gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
for i, block := range chain { for i, block := range chain {
rawdb.WriteBlock(db, block) rawdb.WriteBlock(db, block)
@ -180,7 +180,7 @@ func TestFilters(t *testing.T) {
// Hack: GenerateChainWithGenesis creates a new db. // Hack: GenerateChainWithGenesis creates a new db.
// Commit the genesis manually and use GenerateChain. // Commit the genesis manually and use GenerateChain.
_, err = gspec.Commit(db, trie.NewDatabase(db)) _, err = gspec.Commit(db, trie.NewDatabase(db, nil))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
@ -40,6 +41,7 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
) )
const ( const (
@ -183,7 +185,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
} }
// If we've successfully finished a sync cycle, accept transactions from // If we've successfully finished a sync cycle, accept transactions from
// the network // the network
h.acceptTxs.Store(true) h.enableSyncedFeatures()
} }
// Construct the downloader (long sync) // Construct the downloader (long sync)
h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, success) h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, success)
@ -272,7 +274,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
} }
n, err := h.chain.InsertChain(blocks) n, err := h.chain.InsertChain(blocks)
if err == nil { if err == nil {
h.acceptTxs.Store(true) // Mark initial sync done on any fetcher import h.enableSyncedFeatures() // Mark initial sync done on any fetcher import
} }
return n, err return n, err
} }
@ -674,3 +676,12 @@ func (h *handler) txBroadcastLoop() {
} }
} }
} }
// enableSyncedFeatures enables the post-sync functionalities when the initial
// sync is finished.
func (h *handler) enableSyncedFeatures() {
h.acceptTxs.Store(true)
if h.chain.TrieDB().Scheme() == rawdb.PathScheme {
h.chain.TrieDB().SetBufferSize(pathdb.DefaultBufferSize)
}
}

View File

@ -112,7 +112,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int,
panic(err) panic(err)
} }
for _, block := range bs { for _, block := range bs {
chain.StateCache().TrieDB().Commit(block.Root(), false) chain.TrieDB().Commit(block.Root(), false)
} }
txconfig := legacypool.DefaultConfig txconfig := legacypool.DefaultConfig
txconfig.Journal = "" // Don't litter the disk with test journals txconfig.Journal = "" // Don't litter the disk with test journals

View File

@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
@ -246,6 +247,10 @@ func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error {
// ServiceGetNodeDataQuery assembles the response to a node data query. It is // ServiceGetNodeDataQuery assembles the response to a node data query. It is
// exposed to allow external packages to test protocol behavior. // exposed to allow external packages to test protocol behavior.
func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) [][]byte { func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) [][]byte {
// Request nodes by hash is not supported in path-based scheme.
if chain.TrieDB().Scheme() == rawdb.PathScheme {
return nil
}
// Gather state data until the fetch or network limits is reached // Gather state data until the fetch or network limits is reached
var ( var (
bytes int bytes int
@ -257,7 +262,7 @@ func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) []
break break
} }
// Retrieve the requested state entry // Retrieve the requested state entry
entry, err := chain.TrieNode(hash) entry, err := chain.TrieDB().Node(hash)
if len(entry) == 0 || err != nil { if len(entry) == 0 || err != nil {
// Read the contract code with prefix only to save unnecessary lookups. // Read the contract code with prefix only to save unnecessary lookups.
entry, err = chain.ContractCodeWithPrefix(hash) entry, err = chain.ContractCodeWithPrefix(hash)

View File

@ -284,7 +284,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac
req.Bytes = softResponseLimit req.Bytes = softResponseLimit
} }
// Retrieve the requested state and bail out if non existent // Retrieve the requested state and bail out if non existent
tr, err := trie.New(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) tr, err := trie.New(trie.StateTrieID(req.Root), chain.TrieDB())
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
@ -414,7 +414,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
if origin != (common.Hash{}) || (abort && len(storage) > 0) { if origin != (common.Hash{}) || (abort && len(storage) > 0) {
// Request started at a non-zero hash or was capped prematurely, add // Request started at a non-zero hash or was capped prematurely, add
// the endpoint Merkle proofs // the endpoint Merkle proofs
accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.TrieDB())
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
@ -423,7 +423,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
return nil, nil return nil, nil
} }
id := trie.StorageTrieID(req.Root, account, acc.Root) id := trie.StorageTrieID(req.Root, account, acc.Root)
stTrie, err := trie.NewStateTrie(id, chain.StateCache().TrieDB()) stTrie, err := trie.NewStateTrie(id, chain.TrieDB())
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
@ -487,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
req.Bytes = softResponseLimit req.Bytes = softResponseLimit
} }
// Make sure we have the state associated with the request // Make sure we have the state associated with the request
triedb := chain.StateCache().TrieDB() triedb := chain.TrieDB()
accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), triedb) accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), triedb)
if err != nil { if err != nil {

View File

@ -35,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/trienode"
"golang.org/x/crypto/sha3" "golang.org/x/crypto/sha3"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
@ -561,6 +562,11 @@ func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Has
func TestSyncBloatedProof(t *testing.T) { func TestSyncBloatedProof(t *testing.T) {
t.Parallel() t.Parallel()
testSyncBloatedProof(t, rawdb.HashScheme)
testSyncBloatedProof(t, rawdb.PathScheme)
}
func testSyncBloatedProof(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -570,7 +576,7 @@ func TestSyncBloatedProof(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
source := newTestPeer("source", t, term) source := newTestPeer("source", t, term)
source.accountTrie = sourceAccountTrie.Copy() source.accountTrie = sourceAccountTrie.Copy()
source.accountValues = elems source.accountValues = elems
@ -638,6 +644,11 @@ func setupSyncer(scheme string, peers ...*testPeer) *Syncer {
func TestSync(t *testing.T) { func TestSync(t *testing.T) {
t.Parallel() t.Parallel()
testSync(t, rawdb.HashScheme)
testSync(t, rawdb.PathScheme)
}
func testSync(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -647,7 +658,7 @@ func TestSync(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -659,7 +670,7 @@ func TestSync(t *testing.T) {
if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a // TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a
@ -667,6 +678,11 @@ func TestSync(t *testing.T) {
func TestSyncTinyTriePanic(t *testing.T) { func TestSyncTinyTriePanic(t *testing.T) {
t.Parallel() t.Parallel()
testSyncTinyTriePanic(t, rawdb.HashScheme)
testSyncTinyTriePanic(t, rawdb.PathScheme)
}
func testSyncTinyTriePanic(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -676,7 +692,7 @@ func TestSyncTinyTriePanic(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1, scheme)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -690,13 +706,18 @@ func TestSyncTinyTriePanic(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestMultiSync tests a basic sync with multiple peers // TestMultiSync tests a basic sync with multiple peers
func TestMultiSync(t *testing.T) { func TestMultiSync(t *testing.T) {
t.Parallel() t.Parallel()
testMultiSync(t, rawdb.HashScheme)
testMultiSync(t, rawdb.PathScheme)
}
func testMultiSync(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -706,7 +727,7 @@ func TestMultiSync(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -720,13 +741,18 @@ func TestMultiSync(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncWithStorage tests basic sync using accounts + storage + code // TestSyncWithStorage tests basic sync using accounts + storage + code
func TestSyncWithStorage(t *testing.T) { func TestSyncWithStorage(t *testing.T) {
t.Parallel() t.Parallel()
testSyncWithStorage(t, rawdb.HashScheme)
testSyncWithStorage(t, rawdb.PathScheme)
}
func testSyncWithStorage(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -736,7 +762,7 @@ func TestSyncWithStorage(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 3000, true, false)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -752,13 +778,18 @@ func TestSyncWithStorage(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all
func TestMultiSyncManyUseless(t *testing.T) { func TestMultiSyncManyUseless(t *testing.T) {
t.Parallel() t.Parallel()
testMultiSyncManyUseless(t, rawdb.HashScheme)
testMultiSyncManyUseless(t, rawdb.PathScheme)
}
func testMultiSyncManyUseless(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -768,7 +799,7 @@ func TestMultiSyncManyUseless(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -801,11 +832,18 @@ func TestMultiSyncManyUseless(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all
func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) {
t.Parallel()
testMultiSyncManyUselessWithLowTimeout(t, rawdb.HashScheme)
testMultiSyncManyUselessWithLowTimeout(t, rawdb.PathScheme)
}
func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -815,7 +853,7 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -853,11 +891,18 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all // TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all
func TestMultiSyncManyUnresponsive(t *testing.T) { func TestMultiSyncManyUnresponsive(t *testing.T) {
t.Parallel()
testMultiSyncManyUnresponsive(t, rawdb.HashScheme)
testMultiSyncManyUnresponsive(t, rawdb.PathScheme)
}
func testMultiSyncManyUnresponsive(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -867,7 +912,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -903,7 +948,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
func checkStall(t *testing.T, term func()) chan struct{} { func checkStall(t *testing.T, term func()) chan struct{} {
@ -925,6 +970,11 @@ func checkStall(t *testing.T, term func()) chan struct{} {
func TestSyncBoundaryAccountTrie(t *testing.T) { func TestSyncBoundaryAccountTrie(t *testing.T) {
t.Parallel() t.Parallel()
testSyncBoundaryAccountTrie(t, rawdb.HashScheme)
testSyncBoundaryAccountTrie(t, rawdb.PathScheme)
}
func testSyncBoundaryAccountTrie(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -934,7 +984,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(3000) nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(scheme, 3000)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -952,7 +1002,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is // TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is
@ -960,6 +1010,11 @@ func TestSyncBoundaryAccountTrie(t *testing.T) {
func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncNoStorageAndOneCappedPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneCappedPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneCappedPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -969,7 +1024,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, slow bool) *testPeer { mkSource := func(name string, slow bool) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -994,7 +1049,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver // TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver
@ -1002,6 +1057,11 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) {
func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneCodeCorruptPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1011,7 +1071,7 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { mkSource := func(name string, codeFn codeHandlerFunc) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1034,12 +1094,17 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneAccountCorruptPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1049,7 +1114,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, accFn accountHandlerFunc) *testPeer { mkSource := func(name string, accFn accountHandlerFunc) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1072,7 +1137,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes // TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes
@ -1080,6 +1145,11 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) {
func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.HashScheme)
testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.PathScheme)
}
func testSyncNoStorageAndOneCodeCappedPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1089,7 +1159,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme)
mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { mkSource := func(name string, codeFn codeHandlerFunc) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1123,7 +1193,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
if threshold := 100; counter > threshold { if threshold := 100; counter > threshold {
t.Logf("Error, expected < %d invocations, got %d", threshold, counter) t.Logf("Error, expected < %d invocations, got %d", threshold, counter)
} }
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncBoundaryStorageTrie tests sync against a few normal peers, but the // TestSyncBoundaryStorageTrie tests sync against a few normal peers, but the
@ -1131,6 +1201,11 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) {
func TestSyncBoundaryStorageTrie(t *testing.T) { func TestSyncBoundaryStorageTrie(t *testing.T) {
t.Parallel() t.Parallel()
testSyncBoundaryStorageTrie(t, rawdb.HashScheme)
testSyncBoundaryStorageTrie(t, rawdb.PathScheme)
}
func testSyncBoundaryStorageTrie(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1140,7 +1215,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(10, 1000, false, true) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 10, 1000, false, true)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1160,7 +1235,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is // TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is
@ -1168,6 +1243,11 @@ func TestSyncBoundaryStorageTrie(t *testing.T) {
func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncWithStorageAndOneCappedPeer(t, rawdb.HashScheme)
testSyncWithStorageAndOneCappedPeer(t, rawdb.PathScheme)
}
func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1177,7 +1257,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 300, 1000, false, false)
mkSource := func(name string, slow bool) *testPeer { mkSource := func(name string, slow bool) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1202,7 +1282,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is // TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is
@ -1210,6 +1290,11 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) {
func TestSyncWithStorageAndCorruptPeer(t *testing.T) { func TestSyncWithStorageAndCorruptPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncWithStorageAndCorruptPeer(t, rawdb.HashScheme)
testSyncWithStorageAndCorruptPeer(t, rawdb.PathScheme)
}
func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1219,7 +1304,7 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, handler storageHandlerFunc) *testPeer { mkSource := func(name string, handler storageHandlerFunc) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1243,12 +1328,17 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
t.Parallel() t.Parallel()
testSyncWithStorageAndNonProvingPeer(t, rawdb.HashScheme)
testSyncWithStorageAndNonProvingPeer(t, rawdb.PathScheme)
}
func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1258,7 +1348,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false)
mkSource := func(name string, handler storageHandlerFunc) *testPeer { mkSource := func(name string, handler storageHandlerFunc) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1281,7 +1371,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
close(done) close(done)
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
// TestSyncWithStorage tests basic sync using accounts + storage + code, against // TestSyncWithStorage tests basic sync using accounts + storage + code, against
@ -1290,6 +1380,12 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) {
// did not mark the account for healing. // did not mark the account for healing.
func TestSyncWithStorageMisbehavingProve(t *testing.T) { func TestSyncWithStorageMisbehavingProve(t *testing.T) {
t.Parallel() t.Parallel()
testSyncWithStorageMisbehavingProve(t, rawdb.HashScheme)
testSyncWithStorageMisbehavingProve(t, rawdb.PathScheme)
}
func testSyncWithStorageMisbehavingProve(t *testing.T, scheme string) {
var ( var (
once sync.Once once sync.Once
cancel = make(chan struct{}) cancel = make(chan struct{})
@ -1299,7 +1395,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(10, 30, false) nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(scheme, 10, 30, false)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1314,7 +1410,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) {
if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
} }
type kv struct { type kv struct {
@ -1364,9 +1460,9 @@ func getCodeByHash(hash common.Hash) []byte {
} }
// makeAccountTrieNoStorage spits out a trie, along with the leafs // makeAccountTrieNoStorage spits out a trie, along with the leafs
func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) {
var ( var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db) accTrie = trie.NewEmpty(db)
entries []*kv entries []*kv
) )
@ -1396,12 +1492,12 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) {
// makeBoundaryAccountTrie constructs an account trie. Instead of filling // makeBoundaryAccountTrie constructs an account trie. Instead of filling
// accounts normally, this function will fill a few accounts which have // accounts normally, this function will fill a few accounts which have
// boundary hash. // boundary hash.
func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) {
var ( var (
entries []*kv entries []*kv
boundaries []common.Hash boundaries []common.Hash
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db) accTrie = trie.NewEmpty(db)
) )
// Initialize boundaries // Initialize boundaries
@ -1457,9 +1553,9 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) {
// makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts
// has a unique storage set. // has a unique storage set.
func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) {
var ( var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db) accTrie = trie.NewEmpty(db)
entries []*kv entries []*kv
storageRoots = make(map[common.Hash]common.Hash) storageRoots = make(map[common.Hash]common.Hash)
@ -1512,9 +1608,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
} }
// makeAccountTrieWithStorage spits out a trie, along with the leafs // makeAccountTrieWithStorage spits out a trie, along with the leafs
func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) {
var ( var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase()) db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme))
accTrie = trie.NewEmpty(db) accTrie = trie.NewEmpty(db)
entries []*kv entries []*kv
storageRoots = make(map[common.Hash]common.Hash) storageRoots = make(map[common.Hash]common.Hash)
@ -1656,9 +1752,9 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo
return root, nodes, entries return root, nodes, entries
} }
func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
t.Helper() t.Helper()
triedb := trie.NewDatabase(rawdb.NewDatabase(db)) triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme))
accTrie, err := trie.New(trie.StateTrieID(root), triedb) accTrie, err := trie.New(trie.StateTrieID(root), triedb)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -1700,6 +1796,13 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
// TestSyncAccountPerformance tests how efficient the snap algo is at minimizing // TestSyncAccountPerformance tests how efficient the snap algo is at minimizing
// state healing // state healing
func TestSyncAccountPerformance(t *testing.T) { func TestSyncAccountPerformance(t *testing.T) {
t.Parallel()
testSyncAccountPerformance(t, rawdb.HashScheme)
testSyncAccountPerformance(t, rawdb.PathScheme)
}
func testSyncAccountPerformance(t *testing.T, scheme string) {
// Set the account concurrency to 1. This _should_ result in the // Set the account concurrency to 1. This _should_ result in the
// range root to become correct, and there should be no healing needed // range root to become correct, and there should be no healing needed
defer func(old int) { accountConcurrency = old }(accountConcurrency) defer func(old int) { accountConcurrency = old }(accountConcurrency)
@ -1714,7 +1817,7 @@ func TestSyncAccountPerformance(t *testing.T) {
}) })
} }
) )
nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme)
mkSource := func(name string) *testPeer { mkSource := func(name string) *testPeer {
source := newTestPeer(name, t, term) source := newTestPeer(name, t, term)
@ -1727,7 +1830,7 @@ func TestSyncAccountPerformance(t *testing.T) {
if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil {
t.Fatalf("sync failed: %v", err) t.Fatalf("sync failed: %v", err)
} }
verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t)
// The trie root will always be requested, since it is added when the snap // The trie root will always be requested, since it is added when the snap
// sync cycle starts. When popping the queue, we do not look it up again. // sync cycle starts. When popping the queue, we do not look it up again.
// Doing so would bring this number down to zero in this artificial testcase, // Doing so would bring this number down to zero in this artificial testcase,
@ -1787,3 +1890,10 @@ func TestSlotEstimation(t *testing.T) {
} }
} }
} }
func newDbConfig(scheme string) *trie.Config {
if scheme == rawdb.HashScheme {
return &trie.Config{}
}
return &trie.Config{PathDB: pathdb.Defaults}
}

View File

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
@ -36,31 +37,11 @@ import (
// for releasing state. // for releasing state.
var noopReleaser = tracers.StateReleaseFunc(func() {}) var noopReleaser = tracers.StateReleaseFunc(func() {})
// StateAtBlock retrieves the state database associated with a certain block. func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
// If no state is locally available for the given block, a number of blocks
// are attempted to be reexecuted to generate the desired state. The optional
// base layer statedb can be provided which is regarded as the statedb of the
// parent block.
//
// An additional release function will be returned if the requested state is
// available. Release is expected to be invoked when the returned state is no longer needed.
// Its purpose is to prevent resource leaking. Though it can be noop in some cases.
//
// Parameters:
// - block: The block for which we want the state(state = block.Root)
// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state
// - base: If the caller is tracing multiple blocks, the caller can provide the parent
// state continuously from the callsite.
// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should
// be made from caller, e.g. perform Commit or other 'save-to-disk' changes.
// Otherwise, the trash generated by caller may be persisted permanently.
// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is
// provided, it would be preferable to start from a fresh state, if we have it
// on disk.
func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
var ( var (
current *types.Block current *types.Block
database state.Database database state.Database
triedb *trie.Database
report = true report = true
origin = block.NumberU64() origin = block.NumberU64()
) )
@ -71,9 +52,9 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
// on top to prevent garbage collection and return a release // on top to prevent garbage collection and return a release
// function to deref it. // function to deref it.
if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil { if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil {
statedb.Database().TrieDB().Reference(block.Root(), common.Hash{}) eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{})
return statedb, func() { return statedb, func() {
statedb.Database().TrieDB().Dereference(block.Root()) eth.blockchain.TrieDB().Dereference(block.Root())
}, nil }, nil
} }
} }
@ -84,14 +65,16 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
if preferDisk { if preferDisk {
// Create an ephemeral trie.Database for isolating the live one. Otherwise // Create an ephemeral trie.Database for isolating the live one. Otherwise
// the internal junks created by tracing will be persisted into the disk. // the internal junks created by tracing will be persisted into the disk.
database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) // TODO(rjl493456442), clean cache is disabled to prevent memory leak,
// please re-enable it for better performance.
database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults)
if statedb, err = state.New(block.Root(), database, nil); err == nil { if statedb, err = state.New(block.Root(), database, nil); err == nil {
log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number())
return statedb, noopReleaser, nil return statedb, noopReleaser, nil
} }
} }
// The optional base statedb is given, mark the start point as parent block // The optional base statedb is given, mark the start point as parent block
statedb, database, report = base, base.Database(), false statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false
current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
} else { } else {
// Otherwise, try to reexec blocks until we find a state or reach our limit // Otherwise, try to reexec blocks until we find a state or reach our limit
@ -99,7 +82,10 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
// Create an ephemeral trie.Database for isolating the live one. Otherwise // Create an ephemeral trie.Database for isolating the live one. Otherwise
// the internal junks created by tracing will be persisted into the disk. // the internal junks created by tracing will be persisted into the disk.
database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) // TODO(rjl493456442), clean cache is disabled to prevent memory leak,
// please re-enable it for better performance.
triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults)
database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb)
// If we didn't check the live database, do check state over ephemeral database, // If we didn't check the live database, do check state over ephemeral database,
// otherwise we would rewind past a persisted block (specific corner case is // otherwise we would rewind past a persisted block (specific corner case is
@ -175,17 +161,58 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
} }
// Hold the state reference and also drop the parent state // Hold the state reference and also drop the parent state
// to prevent accumulating too many nodes in memory. // to prevent accumulating too many nodes in memory.
database.TrieDB().Reference(root, common.Hash{}) triedb.Reference(root, common.Hash{})
if parent != (common.Hash{}) { if parent != (common.Hash{}) {
database.TrieDB().Dereference(parent) triedb.Dereference(parent)
} }
parent = root parent = root
} }
if report { if report {
nodes, imgs := database.TrieDB().Size() nodes, imgs := triedb.Size()
log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs)
} }
return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil return statedb, func() { triedb.Dereference(block.Root()) }, nil
}
func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) {
// Check if the requested state is available in the live chain.
statedb, err := eth.blockchain.StateAt(block.Root())
if err == nil {
return statedb, noopReleaser, nil
}
// TODO historic state is not supported in path-based scheme.
// Fully archive node in pbss will be implemented by relying
// on state history, but needs more work on top.
return nil, nil, errors.New("historical state not available in path scheme yet")
}
// stateAtBlock retrieves the state database associated with a certain block.
// If no state is locally available for the given block, a number of blocks
// are attempted to be reexecuted to generate the desired state. The optional
// base layer statedb can be provided which is regarded as the statedb of the
// parent block.
//
// An additional release function will be returned if the requested state is
// available. Release is expected to be invoked when the returned state is no
// longer needed. Its purpose is to prevent resource leaking. Though it can be
// noop in some cases.
//
// Parameters:
// - block: The block for which we want the state(state = block.Root)
// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state
// - base: If the caller is tracing multiple blocks, the caller can provide the parent
// state continuously from the callsite.
// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should
// be made from caller, e.g. perform Commit or other 'save-to-disk' changes.
// Otherwise, the trash generated by caller may be persisted permanently.
// - preferDisk: This arg can be used by the caller to signal that even though the 'base' is
// provided, it would be preferable to start from a fresh state, if we have it
// on disk.
func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme {
return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk)
}
return eth.pathState(block)
} }
// stateAtTransaction returns the execution environment of a certain transaction. // stateAtTransaction returns the execution environment of a certain transaction.
@ -201,7 +228,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block,
} }
// Lookup the statedb of parent block from the live database, // Lookup the statedb of parent block from the live database,
// otherwise regenerate it on the flight. // otherwise regenerate it on the flight.
statedb, release, err := eth.StateAtBlock(ctx, parent, reexec, nil, true, false) statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, nil, true, false)
if err != nil { if err != nil {
return nil, vm.BlockContext{}, nil, nil, err return nil, vm.BlockContext{}, nil, nil, err
} }

View File

@ -137,8 +137,10 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) {
GasLimit: uint64(test.Context.GasLimit), GasLimit: uint64(test.Context.GasLimit),
BaseFee: test.Genesis.BaseFee, BaseFee: test.Genesis.BaseFee,
} }
_, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
) )
triedb.Close()
tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)
if err != nil { if err != nil {
t.Fatalf("failed to create call tracer: %v", err) t.Fatalf("failed to create call tracer: %v", err)
@ -237,7 +239,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) {
Difficulty: (*big.Int)(test.Context.Difficulty), Difficulty: (*big.Int)(test.Context.Difficulty),
GasLimit: uint64(test.Context.GasLimit), GasLimit: uint64(test.Context.GasLimit),
} }
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
defer triedb.Close()
b.ReportAllocs() b.ReportAllocs()
b.ResetTimer() b.ResetTimer()
@ -363,7 +366,7 @@ func TestInternals(t *testing.T) {
}, },
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(),
core.GenesisAlloc{ core.GenesisAlloc{
to: core.GenesisAccount{ to: core.GenesisAccount{
Code: tc.code, Code: tc.code,
@ -371,7 +374,9 @@ func TestInternals(t *testing.T) {
origin: core.GenesisAccount{ origin: core.GenesisAccount{
Balance: big.NewInt(500000000000000), Balance: big.NewInt(500000000000000),
}, },
}, false) }, false, rawdb.HashScheme)
defer triedb.Close()
evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer})
msg := &core.Message{ msg := &core.Message{
To: &to, To: &to,

View File

@ -100,7 +100,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string
Difficulty: (*big.Int)(test.Context.Difficulty), Difficulty: (*big.Int)(test.Context.Difficulty),
GasLimit: uint64(test.Context.GasLimit), GasLimit: uint64(test.Context.GasLimit),
} }
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
defer triedb.Close()
// Create the tracer, the EVM environment and run it // Create the tracer, the EVM environment and run it
tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)

View File

@ -108,8 +108,10 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) {
GasLimit: uint64(test.Context.GasLimit), GasLimit: uint64(test.Context.GasLimit),
BaseFee: test.Genesis.BaseFee, BaseFee: test.Genesis.BaseFee,
} }
_, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
) )
defer triedb.Close()
tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)
if err != nil { if err != nil {
t.Fatalf("failed to create call tracer: %v", err) t.Fatalf("failed to create call tracer: %v", err)

View File

@ -79,7 +79,9 @@ func BenchmarkTransactionTrace(b *testing.B) {
Code: []byte{}, Code: []byte{},
Balance: big.NewInt(500000000000000), Balance: big.NewInt(500000000000000),
} }
_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme)
defer triedb.Close()
// Create the tracer, the EVM environment and run it // Create the tracer, the EVM environment and run it
tracer := logger.NewStructLogger(&logger.Config{ tracer := logger.NewStructLogger(&logger.Config{
Debug: false, Debug: false,

View File

@ -22,7 +22,7 @@ const (
EthCategory = "ETHEREUM" EthCategory = "ETHEREUM"
LightCategory = "LIGHT CLIENT" LightCategory = "LIGHT CLIENT"
DevCategory = "DEVELOPER CHAIN" DevCategory = "DEVELOPER CHAIN"
EthashCategory = "ETHASH" StateCategory = "STATE HISTORY MANAGEMENT"
TxPoolCategory = "TRANSACTION POOL (EVM)" TxPoolCategory = "TRANSACTION POOL (EVM)"
BlobPoolCategory = "TRANSACTION POOL (BLOB)" BlobPoolCategory = "TRANSACTION POOL (BLOB)"
PerfCategory = "PERFORMANCE TUNING" PerfCategory = "PERFORMANCE TUNING"

View File

@ -98,7 +98,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) {
if config.OverrideVerkle != nil { if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle overrides.OverrideVerkle = config.OverrideVerkle
} }
chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, trie.NewDatabase(chainDb), config.Genesis, &overrides) triedb := trie.NewDatabase(chainDb, trie.HashDefaults)
chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, triedb, config.Genesis, &overrides)
if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat {
return nil, genesisErr return nil, genesisErr
} }

View File

@ -406,7 +406,7 @@ func testGetProofs(t *testing.T, protocol int) {
accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}}
for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ { for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ {
header := bc.GetHeaderByNumber(i) header := bc.GetHeaderByNumber(i)
trie, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) trie, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB())
for _, acc := range accounts { for _, acc := range accounts {
req := ProofReq{ req := ProofReq{
@ -457,7 +457,7 @@ func testGetStaleProof(t *testing.T, protocol int) {
var expected []rlp.RawValue var expected []rlp.RawValue
if wantOK { if wantOK {
proofsV2 := light.NewNodeSet() proofsV2 := light.NewNodeSet()
t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) t, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB())
t.Prove(account, proofsV2) t.Prove(account, proofsV2)
expected = proofsV2.NodeList() expected = proofsV2.NodeList()
} }
@ -513,7 +513,7 @@ func testGetCHTProofs(t *testing.T, protocol int) {
AuxData: [][]byte{rlp}, AuxData: [][]byte{rlp},
} }
root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash())
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix)))) trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix)), trie.HashDefaults))
trie.Prove(key, &proofsV2.Proofs) trie.Prove(key, &proofsV2.Proofs)
// Assemble the requests for the different protocols // Assemble the requests for the different protocols
requestsV2 := []HelperTrieReq{{ requestsV2 := []HelperTrieReq{{
@ -578,7 +578,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) {
var proofs HelperTrieResps var proofs HelperTrieResps
root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash())
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix)))) trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix)), trie.HashDefaults))
trie.Prove(key, &proofs.Proofs) trie.Prove(key, &proofs.Proofs)
// Send the proof request and verify the response // Send the proof request and verify the response

View File

@ -104,7 +104,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon
for _, addr := range acc { for _, addr := range acc {
if bc != nil { if bc != nil {
header := bc.GetHeaderByHash(bhash) header := bc.GetHeaderByHash(bhash)
st, err = state.New(header.Root, state.NewDatabase(db), nil) st, err = state.New(header.Root, bc.StateCache(), nil)
} else { } else {
header := lc.GetHeaderByHash(bhash) header := lc.GetHeaderByHash(bhash)
st = light.NewState(ctx, header, lc.Odr()) st = light.NewState(ctx, header, lc.Odr())

View File

@ -390,7 +390,8 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie {
if root == (common.Hash{}) { if root == (common.Hash{}) {
return nil return nil
} }
trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) triedb := trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix), trie.HashDefaults)
trie, _ := trie.New(trie.TrieID(root), triedb)
return trie return trie
} }

View File

@ -303,9 +303,8 @@ func handleGetCode(msg Decoder) (serveRequestFn, uint64, uint64, error) {
p.bumpInvalid() p.bumpInvalid()
continue continue
} }
triedb := bc.StateCache().TrieDB()
address := common.BytesToAddress(request.AccountAddress) address := common.BytesToAddress(request.AccountAddress)
account, err := getAccount(triedb, header.Root, address) account, err := getAccount(bc.TrieDB(), header.Root, address)
if err != nil { if err != nil {
p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", address, "err", err) p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", address, "err", err)
p.bumpInvalid() p.bumpInvalid()
@ -424,7 +423,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) {
default: default:
// Account key specified, open a storage trie // Account key specified, open a storage trie
address := common.BytesToAddress(request.AccountAddress) address := common.BytesToAddress(request.AccountAddress)
account, err := getAccount(statedb.TrieDB(), root, address) account, err := getAccount(bc.TrieDB(), root, address)
if err != nil { if err != nil {
p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", address, "err", err) p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", address, "err", err)
p.bumpInvalid() p.bumpInvalid()

View File

@ -49,6 +49,7 @@ import (
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
) )
var ( var (
@ -188,7 +189,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index
BaseFee: big.NewInt(params.InitialBaseFee), BaseFee: big.NewInt(params.InitialBaseFee),
} }
) )
genesis := gspec.MustCommit(db) genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
chain, _ := light.NewLightChain(odr, gspec.Config, engine) chain, _ := light.NewLightChain(odr, gspec.Config, engine)
client := &LightEthereum{ client := &LightEthereum{
@ -226,7 +227,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da
BaseFee: big.NewInt(params.InitialBaseFee), BaseFee: big.NewInt(params.InitialBaseFee),
} }
) )
genesis := gspec.MustCommit(db) genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
// create a simulation backend and pre-commit several customized block to the database. // create a simulation backend and pre-commit several customized block to the database.
simulation := backends.NewSimulatedBackendWithDatabase(db, gspec.Alloc, 100000000) simulation := backends.NewSimulatedBackendWithDatabase(db, gspec.Alloc, 100000000)

View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
) )
// So we can deterministically seed different blockchains // So we can deterministically seed different blockchains
@ -55,7 +56,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [
func newCanonical(n int) (ethdb.Database, *LightChain, error) { func newCanonical(n int) (ethdb.Database, *LightChain, error) {
db := rawdb.NewMemoryDatabase() db := rawdb.NewMemoryDatabase()
gspec := core.Genesis{Config: params.TestChainConfig} gspec := core.Genesis{Config: params.TestChainConfig}
genesis := gspec.MustCommit(db) genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker()) blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker())
// Create and inject the requested chain // Create and inject the requested chain
@ -75,7 +76,7 @@ func newTestLightChain() *LightChain {
Difficulty: big.NewInt(1), Difficulty: big.NewInt(1),
Config: params.TestChainConfig, Config: params.TestChainConfig,
} }
gspec.MustCommit(db) gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker()) lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker())
if err != nil { if err != nil {
panic(err) panic(err)

View File

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
) )
var ( var (
@ -282,7 +283,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) {
t.Fatal(err) t.Fatal(err)
} }
gspec.MustCommit(ldb) gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults))
odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig}
lightchain, err := NewLightChain(odr, gspec.Config, ethash.NewFullFaker()) lightchain, err := NewLightChain(odr, gspec.Config, ethash.NewFullFaker())
if err != nil { if err != nil {

View File

@ -145,7 +145,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis
diskdb: db, diskdb: db,
odr: odr, odr: odr,
trieTable: trieTable, trieTable: trieTable,
triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down triedb: trie.NewDatabase(trieTable, trie.HashDefaults),
sectionSize: size, sectionSize: size,
disablePruning: disablePruning, disablePruning: disablePruning,
} }
@ -348,7 +348,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin
diskdb: db, diskdb: db,
odr: odr, odr: odr,
trieTable: trieTable, trieTable: trieTable,
triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down triedb: trie.NewDatabase(trieTable, trie.HashDefaults),
parentSize: parentSize, parentSize: parentSize,
size: size, size: size,
disablePruning: disablePruning, disablePruning: disablePruning,

View File

@ -215,7 +215,8 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
} else { } else {
id = trie.StateTrieID(t.id.StateRoot) id = trie.StateTrieID(t.id.StateRoot)
} }
t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database())) triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults)
t.trie, err = trie.New(id, triedb)
} }
if err == nil { if err == nil {
err = fn() err = fn()
@ -247,7 +248,8 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator {
} else { } else {
id = trie.StateTrieID(t.id.StateRoot) id = trie.StateTrieID(t.id.StateRoot)
} }
t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database())) triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults)
t, err := trie.New(id, triedb)
if err == nil { if err == nil {
it.t.trie = t it.t.trie = t
} }

View File

@ -50,7 +50,7 @@ func TestNodeIterator(t *testing.T) {
panic(err) panic(err)
} }
gspec.MustCommit(lightdb) gspec.MustCommit(lightdb, trie.NewDatabase(lightdb, trie.HashDefaults))
ctx := context.Background() ctx := context.Background()
odr := &testOdr{sdb: fulldb, ldb: lightdb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} odr := &testOdr{sdb: fulldb, ldb: lightdb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig}
head := blockchain.CurrentHeader() head := blockchain.CurrentHeader()

View File

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
) )
type testTxRelay struct { type testTxRelay struct {
@ -96,7 +97,7 @@ func TestTxPool(t *testing.T) {
panic(err) panic(err)
} }
gspec.MustCommit(ldb) gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults))
odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig}
relay := &testTxRelay{ relay := &testTxRelay{
send: make(chan int, 1), send: make(chan int, 1),

View File

@ -288,8 +288,9 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) {
} }
// Create chainConfig // Create chainConfig
chainDB := rawdb.NewMemoryDatabase() chainDB := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(chainDB, nil)
genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345")) genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345"))
chainConfig, _, err := core.SetupGenesisBlock(chainDB, trie.NewDatabase(chainDB), genesis) chainConfig, _, err := core.SetupGenesisBlock(chainDB, triedb, genesis)
if err != nil { if err != nil {
t.Fatalf("can't create new chain config: %v", err) t.Fatalf("can't create new chain config: %v", err)
} }
@ -300,7 +301,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) {
if err != nil { if err != nil {
t.Fatalf("can't create new chain %v", err) t.Fatalf("can't create new chain %v", err)
} }
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(chainDB), nil) statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache(), nil)
blockchain := &testBlockChain{chainConfig, statedb, 10000000, new(event.Feed)} blockchain := &testBlockChain{chainConfig, statedb, 10000000, new(event.Feed)}
pool := legacypool.New(testTxPoolConfig, blockchain) pool := legacypool.New(testTxPoolConfig, blockchain)

View File

@ -18,6 +18,8 @@ package tests
import ( import (
"testing" "testing"
"github.com/ethereum/go-ethereum/core/rawdb"
) )
func TestBlockchain(t *testing.T) { func TestBlockchain(t *testing.T) {
@ -48,11 +50,17 @@ func TestBlockchain(t *testing.T) {
bt.skipLoad(`.*randomStatetest94.json.*`) bt.skipLoad(`.*randomStatetest94.json.*`)
bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
if err := bt.checkFailure(t, test.Run(false, nil)); err != nil { if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil {
t.Errorf("test without snapshotter failed: %v", err) t.Errorf("test in hash mode without snapshotter failed: %v", err)
} }
if err := bt.checkFailure(t, test.Run(true, nil)); err != nil { if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil {
t.Errorf("test with snapshotter failed: %v", err) t.Errorf("test in hash mode with snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil {
t.Errorf("test in path mode without snapshotter failed: %v", err)
}
if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil {
t.Errorf("test in path mode with snapshotter failed: %v", err)
} }
}) })
// There is also a LegacyTests folder, containing blockchain tests generated // There is also a LegacyTests folder, containing blockchain tests generated

View File

@ -38,6 +38,9 @@ import (
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
) )
// A BlockTest checks handling of entire blocks. // A BlockTest checks handling of entire blocks.
@ -100,16 +103,30 @@ type btHeaderMarshaling struct {
BaseFeePerGas *math.HexOrDecimal256 BaseFeePerGas *math.HexOrDecimal256
} }
func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error { func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error {
config, ok := Forks[t.json.Network] config, ok := Forks[t.json.Network]
if !ok { if !ok {
return UnsupportedForkError{t.json.Network} return UnsupportedForkError{t.json.Network}
} }
// import pre accounts & construct test genesis block & state root // import pre accounts & construct test genesis block & state root
db := rawdb.NewMemoryDatabase() var (
db = rawdb.NewMemoryDatabase()
tconf = &trie.Config{}
)
if scheme == rawdb.PathScheme {
tconf.PathDB = pathdb.Defaults
} else {
tconf.HashDB = hashdb.Defaults
}
// Commit genesis state
gspec := t.genesis(config) gspec := t.genesis(config)
gblock := gspec.MustCommit(db) triedb := trie.NewDatabase(db, tconf)
gblock, err := gspec.Commit(db, triedb)
if err != nil {
return err
}
triedb.Close() // close the db to prevent memory leak
if gblock.Hash() != t.json.Genesis.Hash { if gblock.Hash() != t.json.Genesis.Hash {
return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6])
} }
@ -119,7 +136,7 @@ func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error {
// Wrap the original engine within the beacon-engine // Wrap the original engine within the beacon-engine
engine := beacon.New(ethash.NewFaker()) engine := beacon.New(ethash.NewFaker())
cache := &core.CacheConfig{TrieCleanLimit: 0} cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme}
if snapshotter { if snapshotter {
cache.SnapshotLimit = 1 cache.SnapshotLimit = 1
cache.SnapshotWait = true cache.SnapshotWait = true

View File

@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addresses []common.Address, txHashes []co
} }
func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) {
chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults))
bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults))
for i := 0; i < testChainLen; i++ { for i := 0; i < testChainLen; i++ {
// The element in CHT is <big-endian block number> -> <block hash> // The element in CHT is <big-endian block number> -> <block hash>
key := make([]byte, 8) key := make([]byte, 8)

View File

@ -56,7 +56,7 @@ func (f *fuzzer) readInt() uint64 {
} }
func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv) vals := make(map[string]*kv)
size := f.readInt() size := f.readInt()
// Fill it with some fluff // Fill it with some fluff

View File

@ -136,10 +136,10 @@ func (f *fuzzer) fuzz() int {
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
var ( var (
spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA)) dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA), nil)
trieA = trie.NewEmpty(dbA) trieA = trie.NewEmpty(dbA)
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB)) dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB), nil)
trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme()) rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme())
}) })

View File

@ -143,7 +143,7 @@ func Fuzz(input []byte) int {
func runRandTest(rt randTest) error { func runRandTest(rt randTest) error {
var ( var (
triedb = trie.NewDatabase(rawdb.NewMemoryDatabase()) triedb = trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)
tr = trie.NewEmpty(triedb) tr = trie.NewEmpty(triedb)
origin = types.EmptyRootHash origin = types.EmptyRootHash
values = make(map[string]string) // tracks content of the trie values = make(map[string]string) // tracks content of the trie

View File

@ -30,6 +30,8 @@ import (
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/eth/tracers/logger"
@ -78,21 +80,52 @@ func TestState(t *testing.T) {
subtest := subtest subtest := subtest
key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
t.Run(key+"/trie", func(t *testing.T) { t.Run(key+"/hash/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
_, _, err := test.Run(subtest, vmconfig, false) var result error
return st.checkFailure(t, err) test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
result = st.checkFailure(t, err)
})
return result
}) })
}) })
t.Run(key+"/snap", func(t *testing.T) { t.Run(key+"/hash/snap", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
snaps, statedb, err := test.Run(subtest, vmconfig, true) var result error
if snaps != nil && statedb != nil { test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if _, err := snaps.Journal(statedb.IntermediateRoot(false)); err != nil { if snaps != nil && state != nil {
return err if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil {
result = err
return
} }
} }
return st.checkFailure(t, err) result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/path/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
var result error
test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
result = st.checkFailure(t, err)
})
return result
})
})
t.Run(key+"/path/snap", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
var result error
test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
if snaps != nil && state != nil {
if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil {
result = err
return
}
}
result = st.checkFailure(t, err)
})
return result
}) })
}) })
} }
@ -190,7 +223,8 @@ func runBenchmark(b *testing.B, t *StateTest) {
vmconfig.ExtraEips = eips vmconfig.ExtraEips = eips
block := t.genesis(config).ToBlock() block := t.genesis(config).ToBlock()
_, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false) triedb, _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme)
defer triedb.Close()
var baseFee *big.Int var baseFee *big.Int
if rules.IsLondon { if rules.IsLondon {

View File

@ -39,6 +39,8 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"golang.org/x/crypto/sha3" "golang.org/x/crypto/sha3"
) )
@ -187,43 +189,50 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error {
} }
// Run executes a specific subtest and verifies the post-state and logs // Run executes a specific subtest and verifies the post-state and logs
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) { func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) {
snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme)
if checkedErr := t.checkError(subtest, err); checkedErr != nil {
return snaps, statedb, checkedErr // Invoke the callback at the end of function for further analysis.
defer func() {
postCheck(result, snaps, statedb)
if triedb != nil {
triedb.Close()
}
}()
checkedErr := t.checkError(subtest, err)
if checkedErr != nil {
return checkedErr
} }
// The error has been checked; if it was unexpected, it's already returned. // The error has been checked; if it was unexpected, it's already returned.
if err != nil { if err != nil {
// Here, an error exists but it was expected. // Here, an error exists but it was expected.
// We do not check the post state or logs. // We do not check the post state or logs.
return snaps, statedb, nil return nil
} }
post := t.json.Post[subtest.Fork][subtest.Index] post := t.json.Post[subtest.Fork][subtest.Index]
// N.B: We need to do this in a two-step process, because the first Commit takes care // N.B: We need to do this in a two-step process, because the first Commit takes care
// of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed. // of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed.
if root != common.Hash(post.Root) { if root != common.Hash(post.Root) {
return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
} }
if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) {
return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
} }
// Re-init the post-state instance for further operation statedb, _ = state.New(root, statedb.Database(), snaps)
statedb, err = state.New(root, statedb.Database(), snaps) return nil
if err != nil {
return nil, nil, err
}
return snaps, statedb, nil
} }
// RunNoVerify runs a specific subtest and returns the statedb and post-state root // RunNoVerify runs a specific subtest and returns the statedb and post-state root
func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, common.Hash, error) { func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) {
config, eips, err := GetChainConfig(subtest.Fork) config, eips, err := GetChainConfig(subtest.Fork)
if err != nil { if err != nil {
return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork}
} }
vmconfig.ExtraEips = eips vmconfig.ExtraEips = eips
block := t.genesis(config).ToBlock() block := t.genesis(config).ToBlock()
snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme)
var baseFee *big.Int var baseFee *big.Int
if config.IsLondon(new(big.Int)) { if config.IsLondon(new(big.Int)) {
@ -237,7 +246,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
post := t.json.Post[subtest.Fork][subtest.Index] post := t.json.Post[subtest.Fork][subtest.Index]
msg, err := t.json.Tx.toMessage(post, baseFee) msg, err := t.json.Tx.toMessage(post, baseFee)
if err != nil { if err != nil {
return nil, nil, common.Hash{}, err triedb.Close()
return nil, nil, nil, common.Hash{}, err
} }
// Try to recover tx with current signer // Try to recover tx with current signer
@ -245,11 +255,13 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
var ttx types.Transaction var ttx types.Transaction
err := ttx.UnmarshalBinary(post.TxBytes) err := ttx.UnmarshalBinary(post.TxBytes)
if err != nil { if err != nil {
return nil, nil, common.Hash{}, err triedb.Close()
return nil, nil, nil, common.Hash{}, err
} }
if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil {
return nil, nil, common.Hash{}, err triedb.Close()
return nil, nil, nil, common.Hash{}, err
} }
} }
@ -268,6 +280,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
context.Difficulty = big.NewInt(0) context.Difficulty = big.NewInt(0)
} }
evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) evm := vm.NewEVM(context, txContext, statedb, config, vmconfig)
// Execute the message. // Execute the message.
snapshot := statedb.Snapshot() snapshot := statedb.Snapshot()
gaspool := new(core.GasPool) gaspool := new(core.GasPool)
@ -282,17 +295,25 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
// - there are only 'bad' transactions, which aren't executed. In those cases, // - there are only 'bad' transactions, which aren't executed. In those cases,
// the coinbase gets no txfee, so isn't created, and thus needs to be touched // the coinbase gets no txfee, so isn't created, and thus needs to be touched
statedb.AddBalance(block.Coinbase(), new(big.Int)) statedb.AddBalance(block.Coinbase(), new(big.Int))
// Commit block
// Commit state mutations into database.
root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number())) root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number()))
return snaps, statedb, root, err return triedb, snaps, statedb, root, err
} }
func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas] return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas]
} }
func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) (*snapshot.Tree, *state.StateDB) { func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) {
sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) tconf := &trie.Config{Preimages: true}
if scheme == rawdb.HashScheme {
tconf.HashDB = hashdb.Defaults
} else {
tconf.PathDB = pathdb.Defaults
}
triedb := trie.NewDatabase(db, tconf)
sdb := state.NewDatabaseWithNodeDB(db, triedb)
statedb, _ := state.New(types.EmptyRootHash, sdb, nil) statedb, _ := state.New(types.EmptyRootHash, sdb, nil)
for addr, a := range accounts { for addr, a := range accounts {
statedb.SetCode(addr, a.Code) statedb.SetCode(addr, a.Code)
@ -313,10 +334,10 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo
NoBuild: false, NoBuild: false,
AsyncBuild: false, AsyncBuild: false,
} }
snaps, _ = snapshot.New(snapconfig, db, sdb.TrieDB(), root) snaps, _ = snapshot.New(snapconfig, db, triedb, root)
} }
statedb, _ = state.New(root, sdb, snaps) statedb, _ = state.New(root, sdb, snaps)
return snaps, statedb return triedb, snaps, statedb
} }
func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis {

View File

@ -21,6 +21,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb"
"github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/trienode"
@ -29,14 +30,21 @@ import (
// Config defines all necessary options for database. // Config defines all necessary options for database.
type Config struct { type Config struct {
Cache int // Memory allowance (MB) to use for caching trie nodes in memory Preimages bool // Flag whether the preimage of node key is recorded
Preimages bool // Flag whether the preimage of trie key is recorded HashDB *hashdb.Config // Configs for hash-based scheme
PathDB *pathdb.Config // Configs for experimental path-based scheme, not used yet. PathDB *pathdb.Config // Configs for experimental path-based scheme
// Testing hooks // Testing hooks
OnCommit func(states *triestate.Set) // Hook invoked when commit is performed OnCommit func(states *triestate.Set) // Hook invoked when commit is performed
} }
// HashDefaults represents a config for using hash-based scheme with
// default settings.
var HashDefaults = &Config{
Preimages: false,
HashDB: hashdb.Defaults,
}
// backend defines the methods needed to access/update trie nodes in different // backend defines the methods needed to access/update trie nodes in different
// state scheme. // state scheme.
type backend interface { type backend interface {
@ -91,22 +99,30 @@ func prepare(diskdb ethdb.Database, config *Config) *Database {
} }
} }
// NewDatabase initializes the trie database with default settings, namely // NewDatabase initializes the trie database with default settings, note
// the legacy hash-based scheme is used by default. // the legacy hash-based scheme is used by default.
func NewDatabase(diskdb ethdb.Database) *Database { func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
return NewDatabaseWithConfig(diskdb, nil) // Sanitize the config and use the default one if it's not specified.
if config == nil {
config = HashDefaults
} }
var preimages *preimageStore
// NewDatabaseWithConfig initializes the trie database with provided configs. if config.Preimages {
// The path-based scheme is not activated yet, always initialized with legacy preimages = newPreimageStore(diskdb)
// hash-based scheme by default. }
func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database { db := &Database{
var cleans int config: config,
if config != nil && config.Cache != 0 { diskdb: diskdb,
cleans = config.Cache * 1024 * 1024 preimages: preimages,
}
if config.HashDB != nil && config.PathDB != nil {
log.Crit("Both 'hash' and 'path' mode are configured")
}
if config.PathDB != nil {
db.backend = pathdb.New(diskdb, config.PathDB)
} else {
db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{})
} }
db := prepare(diskdb, config)
db.backend = hashdb.New(diskdb, cleans, mptResolver{})
return db return db
} }
@ -240,3 +256,60 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
} }
return hdb.Node(hash) return hdb.Node(hash)
} }
// Recover rollbacks the database to a specified historical point. The state is
// supported as the rollback destination only if it's canonical state and the
// corresponding trie histories are existent. It's only supported by path-based
// database and will return an error for others.
func (db *Database) Recover(target common.Hash) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.Recover(target, &trieLoader{db: db})
}
// Recoverable returns the indicator if the specified state is enabled to be
// recovered. It's only supported by path-based database and will return an
// error for others.
func (db *Database) Recoverable(root common.Hash) (bool, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return false, errors.New("not supported")
}
return pdb.Recoverable(root), nil
}
// Reset wipes all available journal from the persistent database and discard
// all caches and diff layers. Using the given root to create a new disk layer.
// It's only supported by path-based database and will return an error for others.
func (db *Database) Reset(root common.Hash) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.Reset(root)
}
// Journal commits an entire diff hierarchy to disk into a single journal entry.
// This is meant to be used during shutdown to persist the snapshot without
// flattening everything down (bad for reorgs). It's only supported by path-based
// database and will return an error for others.
func (db *Database) Journal(root common.Hash) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.Journal(root)
}
// SetBufferSize sets the node buffer size to the provided value(in bytes).
// It's only supported by path-based database and will return an error for
// others.
func (db *Database) SetBufferSize(size int) error {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return errors.New("not supported")
}
return pdb.SetBufferSize(size)
}

View File

@ -27,7 +27,7 @@ import (
func newTestDatabase(diskdb ethdb.Database, scheme string) *Database { func newTestDatabase(diskdb ethdb.Database, scheme string) *Database {
db := prepare(diskdb, nil) db := prepare(diskdb, nil)
if scheme == rawdb.HashScheme { if scheme == rawdb.HashScheme {
db.backend = hashdb.New(diskdb, 0, mptResolver{}) db.backend = hashdb.New(diskdb, &hashdb.Config{}, mptResolver{})
} else { } else {
db.backend = pathdb.New(diskdb, &pathdb.Config{}) // disable clean/dirty cache db.backend = pathdb.New(diskdb, &pathdb.Config{}) // disable clean/dirty cache
} }

View File

@ -18,7 +18,6 @@ package trie
import ( import (
"bytes" "bytes"
"encoding/binary"
"fmt" "fmt"
"math/rand" "math/rand"
"testing" "testing"
@ -27,13 +26,11 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/trienode"
) )
func TestEmptyIterator(t *testing.T) { func TestEmptyIterator(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
iter := trie.MustNodeIterator(nil) iter := trie.MustNodeIterator(nil)
seen := make(map[string]struct{}) seen := make(map[string]struct{})
@ -46,7 +43,7 @@ func TestEmptyIterator(t *testing.T) {
} }
func TestIterator(t *testing.T) { func TestIterator(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase()) db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
vals := []struct{ k, v string }{ vals := []struct{ k, v string }{
{"do", "verb"}, {"do", "verb"},
@ -89,7 +86,7 @@ func (k *kv) less(other *kv) bool {
} }
func TestIteratorLargeData(t *testing.T) { func TestIteratorLargeData(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv) vals := make(map[string]*kv)
for i := byte(0); i < 255; i++ { for i := byte(0); i < 255; i++ {
@ -208,7 +205,7 @@ var testdata2 = []kvs{
} }
func TestIteratorSeek(t *testing.T) { func TestIteratorSeek(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for _, val := range testdata1 { for _, val := range testdata1 {
trie.MustUpdate([]byte(val.k), []byte(val.v)) trie.MustUpdate([]byte(val.k), []byte(val.v))
} }
@ -249,7 +246,7 @@ func checkIteratorOrder(want []kvs, it *Iterator) error {
} }
func TestDifferenceIterator(t *testing.T) { func TestDifferenceIterator(t *testing.T) {
dba := NewDatabase(rawdb.NewMemoryDatabase()) dba := NewDatabase(rawdb.NewMemoryDatabase(), nil)
triea := NewEmpty(dba) triea := NewEmpty(dba)
for _, val := range testdata1 { for _, val := range testdata1 {
triea.MustUpdate([]byte(val.k), []byte(val.v)) triea.MustUpdate([]byte(val.k), []byte(val.v))
@ -258,7 +255,7 @@ func TestDifferenceIterator(t *testing.T) {
dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil)
triea, _ = New(TrieID(rootA), dba) triea, _ = New(TrieID(rootA), dba)
dbb := NewDatabase(rawdb.NewMemoryDatabase()) dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trieb := NewEmpty(dbb) trieb := NewEmpty(dbb)
for _, val := range testdata2 { for _, val := range testdata2 {
trieb.MustUpdate([]byte(val.k), []byte(val.v)) trieb.MustUpdate([]byte(val.k), []byte(val.v))
@ -291,7 +288,7 @@ func TestDifferenceIterator(t *testing.T) {
} }
func TestUnionIterator(t *testing.T) { func TestUnionIterator(t *testing.T) {
dba := NewDatabase(rawdb.NewMemoryDatabase()) dba := NewDatabase(rawdb.NewMemoryDatabase(), nil)
triea := NewEmpty(dba) triea := NewEmpty(dba)
for _, val := range testdata1 { for _, val := range testdata1 {
triea.MustUpdate([]byte(val.k), []byte(val.v)) triea.MustUpdate([]byte(val.k), []byte(val.v))
@ -300,7 +297,7 @@ func TestUnionIterator(t *testing.T) {
dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil)
triea, _ = New(TrieID(rootA), dba) triea, _ = New(TrieID(rootA), dba)
dbb := NewDatabase(rawdb.NewMemoryDatabase()) dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trieb := NewEmpty(dbb) trieb := NewEmpty(dbb)
for _, val := range testdata2 { for _, val := range testdata2 {
trieb.MustUpdate([]byte(val.k), []byte(val.v)) trieb.MustUpdate([]byte(val.k), []byte(val.v))
@ -344,7 +341,7 @@ func TestUnionIterator(t *testing.T) {
} }
func TestIteratorNoDups(t *testing.T) { func TestIteratorNoDups(t *testing.T) {
tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for _, val := range testdata1 { for _, val := range testdata1 {
tr.MustUpdate([]byte(val.k), []byte(val.v)) tr.MustUpdate([]byte(val.k), []byte(val.v))
} }
@ -537,96 +534,6 @@ func TestIteratorNodeBlob(t *testing.T) {
testIteratorNodeBlob(t, rawdb.PathScheme) testIteratorNodeBlob(t, rawdb.PathScheme)
} }
type loggingDb struct {
getCount uint64
backend ethdb.KeyValueStore
}
func (l *loggingDb) Has(key []byte) (bool, error) {
return l.backend.Has(key)
}
func (l *loggingDb) Get(key []byte) ([]byte, error) {
l.getCount++
return l.backend.Get(key)
}
func (l *loggingDb) Put(key []byte, value []byte) error {
return l.backend.Put(key, value)
}
func (l *loggingDb) Delete(key []byte) error {
return l.backend.Delete(key)
}
func (l *loggingDb) NewBatch() ethdb.Batch {
return l.backend.NewBatch()
}
func (l *loggingDb) NewBatchWithSize(size int) ethdb.Batch {
return l.backend.NewBatchWithSize(size)
}
func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
return l.backend.NewIterator(prefix, start)
}
func (l *loggingDb) NewSnapshot() (ethdb.Snapshot, error) {
return l.backend.NewSnapshot()
}
func (l *loggingDb) Stat(property string) (string, error) {
return l.backend.Stat(property)
}
func (l *loggingDb) Compact(start []byte, limit []byte) error {
return l.backend.Compact(start, limit)
}
func (l *loggingDb) Close() error {
return l.backend.Close()
}
// makeLargeTestTrie create a sample test trie
func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) {
// Create an empty trie
logDb := &loggingDb{0, memorydb.New()}
triedb := NewDatabase(rawdb.NewDatabase(logDb))
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data
for i := 0; i < 10000; i++ {
key := make([]byte, 32)
val := make([]byte, 32)
binary.BigEndian.PutUint64(key, uint64(i))
binary.BigEndian.PutUint64(val, uint64(i))
key = crypto.Keccak256(key)
val = crypto.Keccak256(val)
trie.MustUpdate(key, val)
}
root, nodes, _ := trie.Commit(false)
triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil)
triedb.Commit(root, false)
// Return the generated trie
trie, _ = NewStateTrie(TrieID(root), triedb)
return triedb, trie, logDb
}
// Tests that the node iterator indeed walks over the entire database contents.
func TestNodeIteratorLargeTrie(t *testing.T) {
// Create some arbitrary test trie to iterate
db, trie, logDb := makeLargeTestTrie()
db.Cap(0) // flush everything
// Do a seek operation
trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885"))
// master: 24 get operations
// this pr: 6 get operations
if have, want := logDb.getCount, uint64(6); have != want {
t.Fatalf("Too many lookups during seek, have %d want %d", have, want)
}
}
func testIteratorNodeBlob(t *testing.T, scheme string) { func testIteratorNodeBlob(t *testing.T, scheme string) {
var ( var (
db = rawdb.NewMemoryDatabase() db = rawdb.NewMemoryDatabase()
@ -700,7 +607,7 @@ func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) {
} }
hash = common.BytesToHash(key) hash = common.BytesToHash(key)
} else { } else {
ok, remain := rawdb.IsAccountTrieNode(key) ok, remain := rawdb.ResolveAccountTrieNodeKey(key)
if !ok { if !ok {
return false, nil, common.Hash{} return false, nil, common.Hash{}
} }

View File

@ -94,7 +94,7 @@ func TestProof(t *testing.T) {
} }
func TestOneElementProof(t *testing.T) { func TestOneElementProof(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "k", "v") updateString(trie, "k", "v")
for i, prover := range makeProvers(trie) { for i, prover := range makeProvers(trie) {
proof := prover([]byte("k")) proof := prover([]byte("k"))
@ -145,7 +145,7 @@ func TestBadProof(t *testing.T) {
// Tests that missing keys can also be proven. The test explicitly uses a single // Tests that missing keys can also be proven. The test explicitly uses a single
// entry trie and checks for missing keys both before and after the single entry. // entry trie and checks for missing keys both before and after the single entry.
func TestMissingKeyProof(t *testing.T) { func TestMissingKeyProof(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "k", "v") updateString(trie, "k", "v")
for i, key := range []string{"a", "j", "l", "z"} { for i, key := range []string{"a", "j", "l", "z"} {
@ -395,7 +395,7 @@ func TestOneElementRangeProof(t *testing.T) {
} }
// Test the mini trie with only a single element. // Test the mini trie with only a single element.
tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
entry := &kv{randBytes(32), randBytes(20), false} entry := &kv{randBytes(32), randBytes(20), false}
tinyTrie.MustUpdate(entry.k, entry.v) tinyTrie.MustUpdate(entry.k, entry.v)
@ -467,7 +467,7 @@ func TestAllElementsProof(t *testing.T) {
// TestSingleSideRangeProof tests the range starts from zero. // TestSingleSideRangeProof tests the range starts from zero.
func TestSingleSideRangeProof(t *testing.T) { func TestSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ { for i := 0; i < 64; i++ {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv var entries []*kv
for i := 0; i < 4096; i++ { for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false} value := &kv{randBytes(32), randBytes(20), false}
@ -502,7 +502,7 @@ func TestSingleSideRangeProof(t *testing.T) {
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
func TestReverseSingleSideRangeProof(t *testing.T) { func TestReverseSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ { for i := 0; i < 64; i++ {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv var entries []*kv
for i := 0; i < 4096; i++ { for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false} value := &kv{randBytes(32), randBytes(20), false}
@ -609,7 +609,7 @@ func TestBadRangeProof(t *testing.T) {
// TestGappedRangeProof focuses on the small trie with embedded nodes. // TestGappedRangeProof focuses on the small trie with embedded nodes.
// If the gapped node is embedded in the trie, it should be detected too. // If the gapped node is embedded in the trie, it should be detected too.
func TestGappedRangeProof(t *testing.T) { func TestGappedRangeProof(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv // Sorted entries var entries []*kv // Sorted entries
for i := byte(0); i < 10; i++ { for i := byte(0); i < 10; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -683,7 +683,7 @@ func TestSameSideProofs(t *testing.T) {
} }
func TestHasRightElement(t *testing.T) { func TestHasRightElement(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
var entries []*kv var entries []*kv
for i := 0; i < 4096; i++ { for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false} value := &kv{randBytes(32), randBytes(20), false}
@ -1036,7 +1036,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) {
} }
func randomTrie(n int) (*Trie, map[string]*kv) { func randomTrie(n int) (*Trie, map[string]*kv) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv) vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ { for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
@ -1055,7 +1055,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) {
} }
func nonRandomTrie(n int) (*Trie, map[string]*kv) { func nonRandomTrie(n int) (*Trie, map[string]*kv) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := make(map[string]*kv) vals := make(map[string]*kv)
max := uint64(0xffffffffffffffff) max := uint64(0xffffffffffffffff)
for i := uint64(0); i < uint64(n); i++ { for i := uint64(0); i < uint64(n); i++ {
@ -1080,7 +1080,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) {
common.Hex2Bytes("02"), common.Hex2Bytes("02"),
common.Hex2Bytes("03"), common.Hex2Bytes("03"),
} }
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i, key := range keys { for i, key := range keys {
trie.MustUpdate(key, vals[i]) trie.MustUpdate(key, vals[i])
} }

View File

@ -31,14 +31,14 @@ import (
) )
func newEmptySecure() *StateTrie { func newEmptySecure() *StateTrie {
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase())) trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase(), nil))
return trie return trie
} }
// makeTestStateTrie creates a large enough secure trie for testing. // makeTestStateTrie creates a large enough secure trie for testing.
func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) {
// Create an empty trie // Create an empty trie
triedb := NewDatabase(rawdb.NewMemoryDatabase()) triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data // Fill it with some arbitrary data

View File

@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) {
func TestSizeBug(t *testing.T) { func TestSizeBug(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) {
func TestEmptyBug(t *testing.T) { func TestEmptyBug(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) {
func TestValLength56(t *testing.T) { func TestValLength56(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
@ -254,7 +254,7 @@ func TestValLength56(t *testing.T) {
// which causes a lot of node-within-node. This case was found via fuzzing. // which causes a lot of node-within-node. This case was found via fuzzing.
func TestUpdateSmallNodes(t *testing.T) { func TestUpdateSmallNodes(t *testing.T) {
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
kvs := []struct { kvs := []struct {
K string K string
V string V string
@ -282,7 +282,7 @@ func TestUpdateSmallNodes(t *testing.T) {
func TestUpdateVariableKeys(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) {
t.SkipNow() t.SkipNow()
st := NewStackTrie(nil) st := NewStackTrie(nil)
nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
kvs := []struct { kvs := []struct {
K string K string
V string V string
@ -351,7 +351,7 @@ func TestStacktrieNotModifyValues(t *testing.T) {
func TestStacktrieSerialization(t *testing.T) { func TestStacktrieSerialization(t *testing.T) {
var ( var (
st = NewStackTrie(nil) st = NewStackTrie(nil)
nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
keyB = big.NewInt(1) keyB = big.NewInt(1)
keyDelta = big.NewInt(1) keyDelta = big.NewInt(1)
vals [][]byte vals [][]byte

View File

@ -109,8 +109,8 @@ type trieElement struct {
// Tests that an empty trie is not scheduled for syncing. // Tests that an empty trie is not scheduled for syncing.
func TestEmptySync(t *testing.T) { func TestEmptySync(t *testing.T) {
dbA := NewDatabase(rawdb.NewMemoryDatabase()) dbA := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
dbB := NewDatabase(rawdb.NewMemoryDatabase()) dbB := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)
dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)

View File

@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) {
// Tests if the trie diffs are tracked correctly. Tracer should capture // Tests if the trie diffs are tracked correctly. Tracer should capture
// all non-leaf dirty nodes, no matter the node is embedded or not. // all non-leaf dirty nodes, no matter the node is embedded or not.
func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
db := NewDatabase(rawdb.NewMemoryDatabase()) db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
// Determine all new nodes are tracked // Determine all new nodes are tracked
@ -104,7 +104,7 @@ func TestTrieTracerNoop(t *testing.T) {
} }
func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) { func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for _, val := range vals { for _, val := range vals {
trie.MustUpdate([]byte(val.k), []byte(val.v)) trie.MustUpdate([]byte(val.k), []byte(val.v))
} }
@ -128,7 +128,7 @@ func TestAccessList(t *testing.T) {
func testAccessList(t *testing.T, vals []struct{ k, v string }) { func testAccessList(t *testing.T, vals []struct{ k, v string }) {
var ( var (
db = NewDatabase(rawdb.NewMemoryDatabase()) db = NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie = NewEmpty(db) trie = NewEmpty(db)
orig = trie.Copy() orig = trie.Copy()
) )
@ -211,7 +211,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
// Tests origin values won't be tracked in Iterator or Prover // Tests origin values won't be tracked in Iterator or Prover
func TestAccessListLeak(t *testing.T) { func TestAccessListLeak(t *testing.T) {
var ( var (
db = NewDatabase(rawdb.NewMemoryDatabase()) db = NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie = NewEmpty(db) trie = NewEmpty(db)
) )
// Create trie from scratch // Create trie from scratch
@ -262,7 +262,7 @@ func TestAccessListLeak(t *testing.T) {
// in its parent due to the smaller size of the original tree node. // in its parent due to the smaller size of the original tree node.
func TestTinyTree(t *testing.T) { func TestTinyTree(t *testing.T) {
var ( var (
db = NewDatabase(rawdb.NewMemoryDatabase()) db = NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie = NewEmpty(db) trie = NewEmpty(db)
) )
for _, val := range tiny { for _, val := range tiny {

View File

@ -20,6 +20,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/triestate"
) )
// Reader wraps the Node method of a backing trie store. // Reader wraps the Node method of a backing trie store.
@ -83,3 +84,18 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) {
} }
return blob, nil return blob, nil
} }
// trieLoader implements triestate.TrieLoader for constructing tries.
type trieLoader struct {
db *Database
}
// OpenTrie opens the main account trie.
func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) {
return New(TrieID(root), l.db)
}
// OpenStorageTrie opens the storage trie of an account.
func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) {
return New(StorageTrieID(stateRoot, addrHash, root), l.db)
}

View File

@ -45,7 +45,7 @@ func init() {
} }
func TestEmptyTrie(t *testing.T) { func TestEmptyTrie(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
res := trie.Hash() res := trie.Hash()
exp := types.EmptyRootHash exp := types.EmptyRootHash
if res != exp { if res != exp {
@ -54,7 +54,7 @@ func TestEmptyTrie(t *testing.T) {
} }
func TestNull(t *testing.T) { func TestNull(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
key := make([]byte, 32) key := make([]byte, 32)
value := []byte("test") value := []byte("test")
trie.MustUpdate(key, value) trie.MustUpdate(key, value)
@ -64,8 +64,13 @@ func TestNull(t *testing.T) {
} }
func TestMissingRoot(t *testing.T) { func TestMissingRoot(t *testing.T) {
testMissingRoot(t, rawdb.HashScheme)
testMissingRoot(t, rawdb.PathScheme)
}
func testMissingRoot(t *testing.T, scheme string) {
root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
trie, err := New(TrieID(root), NewDatabase(rawdb.NewMemoryDatabase())) trie, err := New(TrieID(root), newTestDatabase(rawdb.NewMemoryDatabase(), scheme))
if trie != nil { if trie != nil {
t.Error("New returned non-nil trie for invalid root") t.Error("New returned non-nil trie for invalid root")
} }
@ -161,7 +166,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) {
} }
func TestInsert(t *testing.T) { func TestInsert(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "doe", "reindeer") updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy") updateString(trie, "dog", "puppy")
@ -173,7 +178,7 @@ func TestInsert(t *testing.T) {
t.Errorf("case 1: exp %x got %x", exp, root) t.Errorf("case 1: exp %x got %x", exp, root)
} }
trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
@ -184,7 +189,7 @@ func TestInsert(t *testing.T) {
} }
func TestGet(t *testing.T) { func TestGet(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase()) db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
updateString(trie, "doe", "reindeer") updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy") updateString(trie, "dog", "puppy")
@ -209,7 +214,7 @@ func TestGet(t *testing.T) {
} }
func TestDelete(t *testing.T) { func TestDelete(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := []struct{ k, v string }{ vals := []struct{ k, v string }{
{"do", "verb"}, {"do", "verb"},
{"ether", "wookiedoo"}, {"ether", "wookiedoo"},
@ -236,7 +241,7 @@ func TestDelete(t *testing.T) {
} }
func TestEmptyValues(t *testing.T) { func TestEmptyValues(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
vals := []struct{ k, v string }{ vals := []struct{ k, v string }{
{"do", "verb"}, {"do", "verb"},
@ -260,7 +265,7 @@ func TestEmptyValues(t *testing.T) {
} }
func TestReplication(t *testing.T) { func TestReplication(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase()) db := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
vals := []struct{ k, v string }{ vals := []struct{ k, v string }{
{"do", "verb"}, {"do", "verb"},
@ -321,7 +326,7 @@ func TestReplication(t *testing.T) {
} }
func TestLargeValue(t *testing.T) { func TestLargeValue(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99}) trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99})
trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32))
trie.Hash() trie.Hash()
@ -604,7 +609,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000 const benchElemCount = 20000
func benchGet(b *testing.B) { func benchGet(b *testing.B) {
triedb := NewDatabase(rawdb.NewMemoryDatabase()) triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(triedb) trie := NewEmpty(triedb)
k := make([]byte, 32) k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ { for i := 0; i < benchElemCount; i++ {
@ -621,7 +626,7 @@ func benchGet(b *testing.B) {
} }
func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
k := make([]byte, 32) k := make([]byte, 32)
b.ReportAllocs() b.ReportAllocs()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
@ -651,7 +656,7 @@ func BenchmarkHash(b *testing.B) {
// entries, then adding N more. // entries, then adding N more.
addresses, accounts := makeAccounts(2 * b.N) addresses, accounts := makeAccounts(2 * b.N)
// Insert the accounts into the trie and hash it // Insert the accounts into the trie and hash it
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
i := 0 i := 0
for ; i < len(addresses)/2; i++ { for ; i < len(addresses)/2; i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
@ -682,7 +687,7 @@ func BenchmarkCommitAfterHash(b *testing.B) {
func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) {
// Make the random benchmark deterministic // Make the random benchmark deterministic
addresses, accounts := makeAccounts(b.N) addresses, accounts := makeAccounts(b.N)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ { for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
} }
@ -696,7 +701,7 @@ func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) {
func TestTinyTrie(t *testing.T) { func TestTinyTrie(t *testing.T) {
// Create a realistic account trie to hash // Create a realistic account trie to hash
_, accounts := makeAccounts(5) _, accounts := makeAccounts(5)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3])
if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root {
t.Errorf("1: got %x, exp %x", root, exp) t.Errorf("1: got %x, exp %x", root, exp)
@ -709,7 +714,7 @@ func TestTinyTrie(t *testing.T) {
if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
t.Errorf("3: got %x, exp %x", root, exp) t.Errorf("3: got %x, exp %x", root, exp)
} }
checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
it := NewIterator(trie.MustNodeIterator(nil)) it := NewIterator(trie.MustNodeIterator(nil))
for it.Next() { for it.Next() {
checktr.MustUpdate(it.Key, it.Value) checktr.MustUpdate(it.Key, it.Value)
@ -722,7 +727,7 @@ func TestTinyTrie(t *testing.T) {
func TestCommitAfterHash(t *testing.T) { func TestCommitAfterHash(t *testing.T) {
// Create a realistic account trie to hash // Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000) addresses, accounts := makeAccounts(1000)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ { for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
} }
@ -788,11 +793,17 @@ func (s *spongeDb) Stat(property string) (string, error) { panic("implement
func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") }
func (s *spongeDb) Close() error { return nil } func (s *spongeDb) Close() error { return nil }
func (s *spongeDb) Put(key []byte, value []byte) error { func (s *spongeDb) Put(key []byte, value []byte) error {
valbrief := value var (
keybrief = key
valbrief = value
)
if len(keybrief) > 8 {
keybrief = keybrief[:8]
}
if len(valbrief) > 8 { if len(valbrief) > 8 {
valbrief = valbrief[:8] valbrief = valbrief[:8]
} }
s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, key[:8], len(value), valbrief)) s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief))
s.sponge.Write(key) s.sponge.Write(key)
s.sponge.Write(value) s.sponge.Write(value)
return nil return nil
@ -830,7 +841,7 @@ func TestCommitSequence(t *testing.T) {
addresses, accounts := makeAccounts(tc.count) addresses, accounts := makeAccounts(tc.count)
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(rawdb.NewDatabase(s)) db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
// Fill the trie with elements // Fill the trie with elements
for i := 0; i < tc.count; i++ { for i := 0; i < tc.count; i++ {
@ -861,7 +872,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) {
prng := rand.New(rand.NewSource(int64(i))) prng := rand.New(rand.NewSource(int64(i)))
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} s := &spongeDb{sponge: sha3.NewLegacyKeccak256()}
db := NewDatabase(rawdb.NewDatabase(s)) db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
// Fill the trie with elements // Fill the trie with elements
for i := 0; i < tc.count; i++ { for i := 0; i < tc.count; i++ {
@ -893,7 +904,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
prng := rand.New(rand.NewSource(int64(count))) prng := rand.New(rand.NewSource(int64(count)))
// This spongeDb is used to check the sequence of disk-db-writes // This spongeDb is used to check the sequence of disk-db-writes
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(rawdb.NewDatabase(s)) db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits // Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
@ -952,7 +963,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
// not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do.
func TestCommitSequenceSmallRoot(t *testing.T) { func TestCommitSequenceSmallRoot(t *testing.T) {
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(rawdb.NewDatabase(s)) db := NewDatabase(rawdb.NewDatabase(s), nil)
trie := NewEmpty(db) trie := NewEmpty(db)
// Another sponge is used for the stacktrie commits // Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
@ -1029,7 +1040,7 @@ func BenchmarkHashFixedSize(b *testing.B) {
func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs() b.ReportAllocs()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ { for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
} }
@ -1080,7 +1091,7 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) {
func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs() b.ReportAllocs()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil))
for i := 0; i < len(addresses); i++ { for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
} }
@ -1132,7 +1143,7 @@ func BenchmarkDerefRootFixedSize(b *testing.B) {
func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs() b.ReportAllocs()
triedb := NewDatabase(rawdb.NewMemoryDatabase()) triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil)
trie := NewEmpty(triedb) trie := NewEmpty(triedb)
for i := 0; i < len(addresses); i++ { for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])

View File

@ -65,6 +65,20 @@ type ChildResolver interface {
ForEach(node []byte, onChild func(common.Hash)) ForEach(node []byte, onChild func(common.Hash))
} }
// Config contains the settings for database.
type Config struct {
CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes
}
// Defaults is the default setting for database if it's not specified.
// Notably, clean cache is disabled explicitly,
var Defaults = &Config{
// Explicitly set clean cache size to 0 to avoid creating fastcache,
// otherwise database must be closed when it's no longer needed to
// prevent memory leak.
CleanCacheSize: 0,
}
// Database is an intermediate write layer between the trie data structures and // Database is an intermediate write layer between the trie data structures and
// the disk database. The aim is to accumulate trie writes in-memory and only // the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder. // periodically flush a couple tries to disk, garbage collecting the remainder.
@ -122,12 +136,13 @@ func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash commo
} }
// New initializes the hash-based node database. // New initializes the hash-based node database.
func New(diskdb ethdb.Database, size int, resolver ChildResolver) *Database { func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Database {
// Initialize the clean cache if the specified cache allowance if config == nil {
// is non-zero. Note, the size is in bytes. config = Defaults
}
var cleans *fastcache.Cache var cleans *fastcache.Cache
if size > 0 { if config.CleanCacheSize > 0 {
cleans = fastcache.New(size) cleans = fastcache.New(config.CleanCacheSize)
} }
return &Database{ return &Database{
diskdb: diskdb, diskdb: diskdb,
@ -621,7 +636,13 @@ func (db *Database) Size() common.StorageSize {
} }
// Close closes the trie database and releases all held resources. // Close closes the trie database and releases all held resources.
func (db *Database) Close() error { return nil } func (db *Database) Close() error {
if db.cleans != nil {
db.cleans.Reset()
db.cleans = nil
}
return nil
}
// Scheme returns the node scheme used in the database. // Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string { func (db *Database) Scheme() string {

View File

@ -33,8 +33,26 @@ import (
"github.com/ethereum/go-ethereum/trie/triestate" "github.com/ethereum/go-ethereum/trie/triestate"
) )
const (
// maxDiffLayers is the maximum diff layers allowed in the layer tree. // maxDiffLayers is the maximum diff layers allowed in the layer tree.
const maxDiffLayers = 128 maxDiffLayers = 128
// defaultCleanSize is the default memory allowance of clean cache.
defaultCleanSize = 16 * 1024 * 1024
// maxBufferSize is the maximum memory allowance of node buffer.
// Too large nodebuffer will cause the system to pause for a long
// time when write happens. Also, the largest batch that pebble can
// support is 4GB, node will panic if batch size exceeds this limit.
maxBufferSize = 256 * 1024 * 1024
// DefaultBufferSize is the default memory allowance of node buffer
// that aggregates the writes from above until it's flushed into the
// disk. It's meant to be used once the initial sync is finished.
// Do not increase the buffer size arbitrarily, otherwise the system
// pause time will increase when the database writes happen.
DefaultBufferSize = 64 * 1024 * 1024
)
// layer is the interface implemented by all state layers which includes some // layer is the interface implemented by all state layers which includes some
// public methods and some additional methods for internal usage. // public methods and some additional methods for internal usage.
@ -68,30 +86,33 @@ type layer interface {
// Config contains the settings for database. // Config contains the settings for database.
type Config struct { type Config struct {
StateLimit uint64 // Number of recent blocks to maintain state history for StateHistory uint64 // Number of recent blocks to maintain state history for
CleanSize int // Maximum memory allowance (in bytes) for caching clean nodes CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes
DirtySize int // Maximum memory allowance (in bytes) for caching dirty nodes DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes
ReadOnly bool // Flag whether the database is opened in read only mode. ReadOnly bool // Flag whether the database is opened in read only mode.
} }
var ( // sanitize checks the provided user configurations and changes anything that's
// defaultCleanSize is the default memory allowance of clean cache. // unreasonable or unworkable.
defaultCleanSize = 16 * 1024 * 1024 func (c *Config) sanitize() *Config {
conf := *c
// defaultBufferSize is the default memory allowance of node buffer if conf.DirtyCacheSize > maxBufferSize {
// that aggregates the writes from above until it's flushed into the log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize))
// disk. Do not increase the buffer size arbitrarily, otherwise the conf.DirtyCacheSize = maxBufferSize
// system pause time will increase when the database writes happen. }
defaultBufferSize = 128 * 1024 * 1024 return &conf
) }
// Defaults contains default settings for Ethereum mainnet. // Defaults contains default settings for Ethereum mainnet.
var Defaults = &Config{ var Defaults = &Config{
StateLimit: params.FullImmutabilityThreshold, StateHistory: params.FullImmutabilityThreshold,
CleanSize: defaultCleanSize, CleanCacheSize: defaultCleanSize,
DirtySize: defaultBufferSize, DirtyCacheSize: DefaultBufferSize,
} }
// ReadOnly is the config in order to open database in read only mode.
var ReadOnly = &Config{ReadOnly: true}
// Database is a multiple-layered structure for maintaining in-memory trie nodes. // Database is a multiple-layered structure for maintaining in-memory trie nodes.
// It consists of one persistent base layer backed by a key-value store, on top // It consists of one persistent base layer backed by a key-value store, on top
// of which arbitrarily many in-memory diff layers are stacked. The memory diffs // of which arbitrarily many in-memory diff layers are stacked. The memory diffs
@ -123,9 +144,11 @@ func New(diskdb ethdb.Database, config *Config) *Database {
if config == nil { if config == nil {
config = Defaults config = Defaults
} }
config = config.sanitize()
db := &Database{ db := &Database{
readOnly: config.ReadOnly, readOnly: config.ReadOnly,
bufferSize: config.DirtySize, bufferSize: config.DirtyCacheSize,
config: config, config: config,
diskdb: diskdb, diskdb: diskdb,
} }
@ -140,7 +163,7 @@ func New(diskdb ethdb.Database, config *Config) *Database {
// mechanism also ensures that at most one **non-readOnly** database // mechanism also ensures that at most one **non-readOnly** database
// is opened at the same time to prevent accidental mutation. // is opened at the same time to prevent accidental mutation.
if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly { if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly {
freezer, err := rawdb.NewStateHistoryFreezer(ancient, false) freezer, err := rawdb.NewStateFreezer(ancient, false)
if err != nil { if err != nil {
log.Crit("Failed to open state history freezer", "err", err) log.Crit("Failed to open state history freezer", "err", err)
} }
@ -344,7 +367,14 @@ func (db *Database) Close() error {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
// Set the database to read-only mode to prevent all
// following mutations.
db.readOnly = true db.readOnly = true
// Release the memory held by clean cache.
db.tree.bottom().resetCache()
// Close the attached state history freezer.
if db.freezer == nil { if db.freezer == nil {
return nil return nil
} }
@ -382,6 +412,10 @@ func (db *Database) SetBufferSize(size int) error {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
if size > maxBufferSize {
log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize))
size = maxBufferSize
}
db.bufferSize = size db.bufferSize = size
return db.tree.bottom().setBufferSize(db.bufferSize) return db.tree.bottom().setBufferSize(db.bufferSize)
} }

View File

@ -46,7 +46,8 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm
h.Update(key.Bytes(), val) h.Update(key.Bytes(), val)
} }
} }
return h.Commit(false) root, nodes, _ := h.Commit(false)
return root, nodes
} }
func generateAccount(storageRoot common.Hash) types.StateAccount { func generateAccount(storageRoot common.Hash) types.StateAccount {
@ -98,7 +99,7 @@ type tester struct {
func newTester(t *testing.T) *tester { func newTester(t *testing.T) *tester {
var ( var (
disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
db = New(disk, &Config{CleanSize: 256 * 1024, DirtySize: 256 * 1024}) db = New(disk, &Config{CleanCacheSize: 256 * 1024, DirtyCacheSize: 256 * 1024})
obj = &tester{ obj = &tester{
db: db, db: db,
preimages: make(map[common.Hash]common.Address), preimages: make(map[common.Hash]common.Address),

View File

@ -29,7 +29,7 @@ import (
func emptyLayer() *diskLayer { func emptyLayer() *diskLayer {
return &diskLayer{ return &diskLayer{
db: New(rawdb.NewMemoryDatabase(), nil), db: New(rawdb.NewMemoryDatabase(), nil),
buffer: newNodeBuffer(defaultBufferSize, nil, 0), buffer: newNodeBuffer(DefaultBufferSize, nil, 0),
} }
} }

View File

@ -47,8 +47,8 @@ func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.C
// Initialize a clean cache if the memory allowance is not zero // Initialize a clean cache if the memory allowance is not zero
// or reuse the provided cache if it is not nil (inherited from // or reuse the provided cache if it is not nil (inherited from
// the original disk layer). // the original disk layer).
if cleans == nil && db.config.CleanSize != 0 { if cleans == nil && db.config.CleanCacheSize != 0 {
cleans = fastcache.New(db.config.CleanSize) cleans = fastcache.New(db.config.CleanCacheSize)
} }
return &diskLayer{ return &diskLayer{
root: root, root: root,
@ -177,7 +177,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
// corresponding states(journal), the stored state history will // corresponding states(journal), the stored state history will
// be truncated in the next restart. // be truncated in the next restart.
if dl.db.freezer != nil { if dl.db.freezer != nil {
err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateLimit) err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateHistory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -276,6 +276,20 @@ func (dl *diskLayer) size() common.StorageSize {
return common.StorageSize(dl.buffer.size) return common.StorageSize(dl.buffer.size)
} }
// resetCache releases the memory held by clean cache to prevent memory leak.
func (dl *diskLayer) resetCache() {
dl.lock.RLock()
defer dl.lock.RUnlock()
// Stale disk layer loses the ownership of clean cache.
if dl.stale {
return
}
if dl.cleans != nil {
dl.cleans.Reset()
}
}
// hasher is used to compute the sha256 hash of the provided data. // hasher is used to compute the sha256 hash of the provided data.
type hasher struct{ sha crypto.KeccakState } type hasher struct{ sha crypto.KeccakState }

View File

@ -226,7 +226,7 @@ func TestTruncateTailHistories(t *testing.T) {
// openFreezer initializes the freezer instance for storing state histories. // openFreezer initializes the freezer instance for storing state histories.
func openFreezer(datadir string, readOnly bool) (*rawdb.ResettableFreezer, error) { func openFreezer(datadir string, readOnly bool) (*rawdb.ResettableFreezer, error) {
return rawdb.NewStateHistoryFreezer(datadir, readOnly) return rawdb.NewStateFreezer(datadir, readOnly)
} }
func compareSet[k comparable](a, b map[k][]byte) bool { func compareSet[k comparable](a, b map[k][]byte) bool {

View File

@ -80,7 +80,7 @@ func (h *testHasher) Delete(key []byte) error {
// Commit computes the new hash of the states and returns the set with all // Commit computes the new hash of the states and returns the set with all
// state changes. // state changes.
func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) {
var ( var (
nodes = make(map[common.Hash][]byte) nodes = make(map[common.Hash][]byte)
set = trienode.NewNodeSet(h.owner) set = trienode.NewNodeSet(h.owner)
@ -108,7 +108,7 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
if root == types.EmptyRootHash && h.root != types.EmptyRootHash { if root == types.EmptyRootHash && h.root != types.EmptyRootHash {
set.AddNode(nil, trienode.NewDeleted()) set.AddNode(nil, trienode.NewDeleted())
} }
return root, set return root, set, nil
} }
// hash performs the hash computation upon the provided states. // hash performs the hash computation upon the provided states.

View File

@ -43,7 +43,7 @@ type Trie interface {
// Commit the trie and returns a set of dirty nodes generated along with // Commit the trie and returns a set of dirty nodes generated along with
// the new root hash. // the new root hash.
Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error)
} }
// TrieLoader wraps functions to load tries. // TrieLoader wraps functions to load tries.
@ -129,7 +129,10 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre
return nil, fmt.Errorf("failed to revert state, err: %w", err) return nil, fmt.Errorf("failed to revert state, err: %w", err)
} }
} }
root, result := tr.Commit(false) root, result, err := tr.Commit(false)
if err != nil {
return nil, err
}
if root != prevRoot { if root != prevRoot {
return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root) return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root)
} }
@ -181,7 +184,10 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error {
return err return err
} }
} }
root, result := st.Commit(false) root, result, err := st.Commit(false)
if err != nil {
return err
}
if root != prev.Root { if root != prev.Root {
return errors.New("failed to reset storage trie") return errors.New("failed to reset storage trie")
} }
@ -232,7 +238,10 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error {
return err return err
} }
} }
root, result := st.Commit(false) root, result, err := st.Commit(false)
if err != nil {
return err
}
if root != types.EmptyRootHash { if root != types.EmptyRootHash {
return errors.New("failed to clear storage trie") return errors.New("failed to clear storage trie")
} }