forked from cerc-io/plugeth
all: activate pbss as experimental feature (#26274)
* all: activate pbss * core/rawdb: fix compilation error * cma, core, eth, les, trie: address comments * cmd, core, eth, trie: polish code * core, cmd, eth: address comments * cmd, core, eth, les, light, tests: address comment * cmd/utils: shorten log message * trie/triedb/pathdb: limit node buffer size to 1gb * cmd/utils: fix opening non-existing db * cmd/utils: rename flag name * cmd, core: group chain history flags and fix tests * core, eth, trie: fix memory leak in snapshot generation * cmd, eth, internal: deprecate flags * all: enable state tests for pathdb, fixes * cmd, core: polish code * trie/triedb/pathdb: limit the node buffer size to 256mb --------- Co-authored-by: Martin Holst Swende <martin@swende.se> Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
co-authored by
Martin Holst Swende
Péter Szilágyi
parent
5e89ff4d6b
commit
503f1f7ada
@@ -58,7 +58,7 @@ type Database interface {
|
||||
// DiskDB returns the underlying key-value disk database.
|
||||
DiskDB() ethdb.KeyValueStore
|
||||
|
||||
// TrieDB retrieves the low level trie database used for data storage.
|
||||
// TrieDB returns the underlying trie database for managing trie nodes.
|
||||
TrieDB() *trie.Database
|
||||
}
|
||||
|
||||
@@ -147,7 +147,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
|
||||
disk: db,
|
||||
codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
|
||||
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
|
||||
triedb: trie.NewDatabaseWithConfig(db, config),
|
||||
triedb: trie.NewDatabase(db, config),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,9 +26,14 @@ import (
|
||||
|
||||
// Tests that the node iterator indeed walks over the entire database contents.
|
||||
func TestNodeIteratorCoverage(t *testing.T) {
|
||||
testNodeIteratorCoverage(t, rawdb.HashScheme)
|
||||
testNodeIteratorCoverage(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testNodeIteratorCoverage(t *testing.T, scheme string) {
|
||||
// Create some arbitrary test state to iterate
|
||||
db, sdb, root, _ := makeTestState()
|
||||
sdb.TrieDB().Commit(root, false)
|
||||
db, sdb, ndb, root, _ := makeTestState(scheme)
|
||||
ndb.Commit(root, false)
|
||||
|
||||
state, err := New(root, sdb, nil)
|
||||
if err != nil {
|
||||
@@ -48,7 +53,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
|
||||
)
|
||||
it := db.NewIterator(nil, nil)
|
||||
for it.Next() {
|
||||
ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value())
|
||||
ok, hash := isTrieNode(scheme, it.Key(), it.Value())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
@@ -90,11 +95,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) {
|
||||
return true, common.BytesToHash(key)
|
||||
}
|
||||
} else {
|
||||
ok, _ := rawdb.IsAccountTrieNode(key)
|
||||
ok := rawdb.IsAccountTrieNode(key)
|
||||
if ok {
|
||||
return true, crypto.Keccak256Hash(val)
|
||||
}
|
||||
ok, _, _ = rawdb.IsStorageTrieNode(key)
|
||||
ok = rawdb.IsStorageTrieNode(key)
|
||||
if ok {
|
||||
return true, crypto.Keccak256Hash(val)
|
||||
}
|
||||
|
||||
@@ -85,13 +85,16 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
|
||||
if headBlock == nil {
|
||||
return nil, errors.New("failed to load head block")
|
||||
}
|
||||
// Offline pruning is only supported in legacy hash based scheme.
|
||||
triedb := trie.NewDatabase(db, trie.HashDefaults)
|
||||
|
||||
snapconfig := snapshot.Config{
|
||||
CacheSize: 256,
|
||||
Recovery: false,
|
||||
NoBuild: true,
|
||||
AsyncBuild: false,
|
||||
}
|
||||
snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
|
||||
snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
|
||||
if err != nil {
|
||||
return nil, err // The relevant snapshot(s) might not exist
|
||||
}
|
||||
@@ -361,7 +364,9 @@ func RecoverPruning(datadir string, db ethdb.Database) error {
|
||||
NoBuild: true,
|
||||
AsyncBuild: false,
|
||||
}
|
||||
snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
|
||||
// Offline pruning is only supported in legacy hash based scheme.
|
||||
triedb := trie.NewDatabase(db, trie.HashDefaults)
|
||||
snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
|
||||
if err != nil {
|
||||
return err // The relevant snapshot(s) might not exist
|
||||
}
|
||||
@@ -403,7 +408,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
|
||||
if genesis == nil {
|
||||
return errors.New("missing genesis block")
|
||||
}
|
||||
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
|
||||
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -427,7 +432,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
|
||||
}
|
||||
if acc.Root != types.EmptyRootHash {
|
||||
id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
|
||||
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
|
||||
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -356,7 +356,8 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
|
||||
var resolver trie.NodeResolver
|
||||
if len(result.keys) > 0 {
|
||||
mdb := rawdb.NewMemoryDatabase()
|
||||
tdb := trie.NewDatabase(mdb)
|
||||
tdb := trie.NewDatabase(mdb, trie.HashDefaults)
|
||||
defer tdb.Close()
|
||||
snapTrie := trie.NewEmpty(tdb)
|
||||
for i, key := range result.keys {
|
||||
snapTrie.Update(key, result.vals[i])
|
||||
|
||||
@@ -30,6 +30,8 @@ import (
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
|
||||
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
|
||||
"github.com/ethereum/go-ethereum/trie/trienode"
|
||||
"golang.org/x/crypto/sha3"
|
||||
)
|
||||
@@ -45,10 +47,15 @@ func hashData(input []byte) common.Hash {
|
||||
|
||||
// Tests that snapshot generation from an empty database.
|
||||
func TestGeneration(t *testing.T) {
|
||||
testGeneration(t, rawdb.HashScheme)
|
||||
testGeneration(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGeneration(t *testing.T, scheme string) {
|
||||
// We can't use statedb to make a test trie (circular dependency), so make
|
||||
// a fake one manually. We're going with a small account trie of 3 accounts,
|
||||
// two of which also has the same 3-slot storage trie attached.
|
||||
var helper = newHelper()
|
||||
var helper = newHelper(scheme)
|
||||
stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
|
||||
|
||||
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
|
||||
@@ -79,10 +86,15 @@ func TestGeneration(t *testing.T) {
|
||||
|
||||
// Tests that snapshot generation with existent flat state.
|
||||
func TestGenerateExistentState(t *testing.T) {
|
||||
testGenerateExistentState(t, rawdb.HashScheme)
|
||||
testGenerateExistentState(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateExistentState(t *testing.T, scheme string) {
|
||||
// We can't use statedb to make a test trie (circular dependency), so make
|
||||
// a fake one manually. We're going with a small account trie of 3 accounts,
|
||||
// two of which also has the same 3-slot storage trie attached.
|
||||
var helper = newHelper()
|
||||
var helper = newHelper(scheme)
|
||||
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
|
||||
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
|
||||
@@ -148,9 +160,15 @@ type testHelper struct {
|
||||
nodes *trienode.MergedNodeSet
|
||||
}
|
||||
|
||||
func newHelper() *testHelper {
|
||||
func newHelper(scheme string) *testHelper {
|
||||
diskdb := rawdb.NewMemoryDatabase()
|
||||
triedb := trie.NewDatabase(diskdb)
|
||||
config := &trie.Config{}
|
||||
if scheme == rawdb.PathScheme {
|
||||
config.PathDB = &pathdb.Config{} // disable caching
|
||||
} else {
|
||||
config.HashDB = &hashdb.Config{} // disable caching
|
||||
}
|
||||
triedb := trie.NewDatabase(diskdb, config)
|
||||
accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb)
|
||||
return &testHelper{
|
||||
diskdb: diskdb,
|
||||
@@ -233,7 +251,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
|
||||
// - extra slots in the middle
|
||||
// - extra slots in the end
|
||||
func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
|
||||
helper := newHelper()
|
||||
testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme)
|
||||
testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) {
|
||||
helper := newHelper(scheme)
|
||||
|
||||
// Account one, empty root but non-empty database
|
||||
helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
|
||||
@@ -325,7 +348,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
|
||||
// - wrong accounts
|
||||
// - extra accounts
|
||||
func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
|
||||
helper := newHelper()
|
||||
testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme)
|
||||
testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) {
|
||||
helper := newHelper(scheme)
|
||||
|
||||
helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
|
||||
helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
|
||||
@@ -380,10 +408,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
|
||||
// Tests that snapshot generation errors out correctly in case of a missing trie
|
||||
// node in the account trie.
|
||||
func TestGenerateCorruptAccountTrie(t *testing.T) {
|
||||
testGenerateCorruptAccountTrie(t, rawdb.HashScheme)
|
||||
testGenerateCorruptAccountTrie(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateCorruptAccountTrie(t *testing.T, scheme string) {
|
||||
// We can't use statedb to make a test trie (circular dependency), so make
|
||||
// a fake one manually. We're going with a small account trie of 3 accounts,
|
||||
// without any storage slots to keep the test smaller.
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
|
||||
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
|
||||
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
|
||||
@@ -391,9 +424,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
|
||||
|
||||
root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
|
||||
|
||||
// Delete an account trie leaf and ensure the generator chokes
|
||||
helper.triedb.Commit(root, false)
|
||||
helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
|
||||
// Delete an account trie node and ensure the generator chokes
|
||||
targetPath := []byte{0xc}
|
||||
targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7")
|
||||
|
||||
rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme)
|
||||
|
||||
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
|
||||
select {
|
||||
@@ -414,11 +449,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
|
||||
// trie node for a storage trie. It's similar to internal corruption but it is
|
||||
// handled differently inside the generator.
|
||||
func TestGenerateMissingStorageTrie(t *testing.T) {
|
||||
testGenerateMissingStorageTrie(t, rawdb.HashScheme)
|
||||
testGenerateMissingStorageTrie(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateMissingStorageTrie(t *testing.T, scheme string) {
|
||||
// We can't use statedb to make a test trie (circular dependency), so make
|
||||
// a fake one manually. We're going with a small account trie of 3 accounts,
|
||||
// two of which also has the same 3-slot storage trie attached.
|
||||
helper := newHelper()
|
||||
|
||||
var (
|
||||
acc1 = hashData([]byte("acc-1"))
|
||||
acc3 = hashData([]byte("acc-3"))
|
||||
helper = newHelper(scheme)
|
||||
)
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
|
||||
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
|
||||
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
|
||||
@@ -427,8 +470,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
|
||||
|
||||
root := helper.Commit()
|
||||
|
||||
// Delete a storage trie root and ensure the generator chokes
|
||||
helper.diskdb.Delete(stRoot.Bytes())
|
||||
// Delete storage trie root of account one and three.
|
||||
rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme)
|
||||
rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme)
|
||||
|
||||
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
|
||||
select {
|
||||
@@ -448,10 +492,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
|
||||
// Tests that snapshot generation errors out correctly in case of a missing trie
|
||||
// node in a storage trie.
|
||||
func TestGenerateCorruptStorageTrie(t *testing.T) {
|
||||
testGenerateCorruptStorageTrie(t, rawdb.HashScheme)
|
||||
testGenerateCorruptStorageTrie(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateCorruptStorageTrie(t *testing.T, scheme string) {
|
||||
// We can't use statedb to make a test trie (circular dependency), so make
|
||||
// a fake one manually. We're going with a small account trie of 3 accounts,
|
||||
// two of which also has the same 3-slot storage trie attached.
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
|
||||
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
|
||||
@@ -461,8 +510,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
|
||||
|
||||
root := helper.Commit()
|
||||
|
||||
// Delete a storage trie leaf and ensure the generator chokes
|
||||
helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
|
||||
// Delete a node in the storage trie.
|
||||
targetPath := []byte{0x4}
|
||||
targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371")
|
||||
rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme)
|
||||
rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme)
|
||||
|
||||
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
|
||||
select {
|
||||
@@ -481,7 +533,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
|
||||
|
||||
// Tests that snapshot generation when an extra account with storage exists in the snap state.
|
||||
func TestGenerateWithExtraAccounts(t *testing.T) {
|
||||
helper := newHelper()
|
||||
testGenerateWithExtraAccounts(t, rawdb.HashScheme)
|
||||
testGenerateWithExtraAccounts(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateWithExtraAccounts(t *testing.T, scheme string) {
|
||||
helper := newHelper(scheme)
|
||||
{
|
||||
// Account one in the trie
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
|
||||
@@ -549,10 +606,15 @@ func enableLogging() {
|
||||
|
||||
// Tests that snapshot generation when an extra account with storage exists in the snap state.
|
||||
func TestGenerateWithManyExtraAccounts(t *testing.T) {
|
||||
testGenerateWithManyExtraAccounts(t, rawdb.HashScheme)
|
||||
testGenerateWithManyExtraAccounts(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
|
||||
if false {
|
||||
enableLogging()
|
||||
}
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
{
|
||||
// Account one in the trie
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
|
||||
@@ -605,11 +667,16 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
|
||||
// So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted.
|
||||
// But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished.
|
||||
func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
|
||||
testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme)
|
||||
testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) {
|
||||
accountCheckRange = 3
|
||||
if false {
|
||||
enableLogging()
|
||||
}
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
{
|
||||
acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
|
||||
val, _ := rlp.EncodeToBytes(acc)
|
||||
@@ -642,11 +709,16 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
|
||||
// TestGenerateWithMalformedSnapdata tests what happes if we have some junk
|
||||
// in the snapshot database, which cannot be parsed back to an account
|
||||
func TestGenerateWithMalformedSnapdata(t *testing.T) {
|
||||
testGenerateWithMalformedSnapdata(t, rawdb.HashScheme)
|
||||
testGenerateWithMalformedSnapdata(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) {
|
||||
accountCheckRange = 3
|
||||
if false {
|
||||
enableLogging()
|
||||
}
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
{
|
||||
acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
|
||||
val, _ := rlp.EncodeToBytes(acc)
|
||||
@@ -679,10 +751,15 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGenerateFromEmptySnap(t *testing.T) {
|
||||
testGenerateFromEmptySnap(t, rawdb.HashScheme)
|
||||
testGenerateFromEmptySnap(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateFromEmptySnap(t *testing.T, scheme string) {
|
||||
//enableLogging()
|
||||
accountCheckRange = 10
|
||||
storageCheckRange = 20
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
// Add 1K accounts to the trie
|
||||
for i := 0; i < 400; i++ {
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
|
||||
@@ -714,8 +791,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
|
||||
// This hits a case where the snap verification passes, but there are more elements in the trie
|
||||
// which we must also add.
|
||||
func TestGenerateWithIncompleteStorage(t *testing.T) {
|
||||
testGenerateWithIncompleteStorage(t, rawdb.HashScheme)
|
||||
testGenerateWithIncompleteStorage(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateWithIncompleteStorage(t *testing.T, scheme string) {
|
||||
storageCheckRange = 4
|
||||
helper := newHelper()
|
||||
helper := newHelper(scheme)
|
||||
stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
|
||||
stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
|
||||
// We add 8 accounts, each one is missing exactly one of the storage slots. This means
|
||||
@@ -813,7 +895,12 @@ func populateDangling(disk ethdb.KeyValueStore) {
|
||||
//
|
||||
// This test will populate some dangling storages to see if they can be cleaned up.
|
||||
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
|
||||
var helper = newHelper()
|
||||
testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme)
|
||||
testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) {
|
||||
var helper = newHelper(scheme)
|
||||
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
|
||||
helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
|
||||
@@ -848,7 +935,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
|
||||
//
|
||||
// This test will populate some dangling storages to see if they can be cleaned up.
|
||||
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
|
||||
var helper = newHelper()
|
||||
testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme)
|
||||
testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) {
|
||||
var helper = newHelper(scheme)
|
||||
|
||||
stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
|
||||
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
|
||||
|
||||
@@ -179,7 +179,7 @@ func (test *stateTest) run() bool {
|
||||
storageList = append(storageList, copy2DSet(states.Storages))
|
||||
}
|
||||
disk = rawdb.NewMemoryDatabase()
|
||||
tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit})
|
||||
tdb = trie.NewDatabase(disk, &trie.Config{OnCommit: onCommit})
|
||||
sdb = NewDatabaseWithNodeDB(disk, tdb)
|
||||
byzantium = rand.Intn(2) == 0
|
||||
)
|
||||
|
||||
+42
-15
@@ -36,14 +36,19 @@ import (
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
|
||||
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
|
||||
)
|
||||
|
||||
// Tests that updating a state trie does not leak any database writes prior to
|
||||
// actually committing the state.
|
||||
func TestUpdateLeaks(t *testing.T) {
|
||||
// Create an empty state database
|
||||
db := rawdb.NewMemoryDatabase()
|
||||
state, _ := New(types.EmptyRootHash, NewDatabase(db), nil)
|
||||
var (
|
||||
db = rawdb.NewMemoryDatabase()
|
||||
tdb = trie.NewDatabase(db, nil)
|
||||
)
|
||||
state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil)
|
||||
|
||||
// Update it with some accounts
|
||||
for i := byte(0); i < 255; i++ {
|
||||
@@ -59,7 +64,7 @@ func TestUpdateLeaks(t *testing.T) {
|
||||
}
|
||||
|
||||
root := state.IntermediateRoot(false)
|
||||
if err := state.Database().TrieDB().Commit(root, false); err != nil {
|
||||
if err := tdb.Commit(root, false); err != nil {
|
||||
t.Errorf("can not commit trie %v to persistent database", root.Hex())
|
||||
}
|
||||
|
||||
@@ -77,8 +82,10 @@ func TestIntermediateLeaks(t *testing.T) {
|
||||
// Create two state databases, one transitioning to the final state, the other final from the beginning
|
||||
transDb := rawdb.NewMemoryDatabase()
|
||||
finalDb := rawdb.NewMemoryDatabase()
|
||||
transState, _ := New(types.EmptyRootHash, NewDatabase(transDb), nil)
|
||||
finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb), nil)
|
||||
transNdb := trie.NewDatabase(transDb, nil)
|
||||
finalNdb := trie.NewDatabase(finalDb, nil)
|
||||
transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil)
|
||||
finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil)
|
||||
|
||||
modify := func(state *StateDB, addr common.Address, i, tweak byte) {
|
||||
state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
|
||||
@@ -110,7 +117,7 @@ func TestIntermediateLeaks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to commit transition state: %v", err)
|
||||
}
|
||||
if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil {
|
||||
if err = transNdb.Commit(transRoot, false); err != nil {
|
||||
t.Errorf("can not commit trie %v to persistent database", transRoot.Hex())
|
||||
}
|
||||
|
||||
@@ -118,7 +125,7 @@ func TestIntermediateLeaks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to commit final state: %v", err)
|
||||
}
|
||||
if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil {
|
||||
if err = finalNdb.Commit(finalRoot, false); err != nil {
|
||||
t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex())
|
||||
}
|
||||
|
||||
@@ -747,9 +754,28 @@ func TestDeleteCreateRevert(t *testing.T) {
|
||||
// the Commit operation fails with an error
|
||||
// If we are missing trie nodes, we should not continue writing to the trie
|
||||
func TestMissingTrieNodes(t *testing.T) {
|
||||
testMissingTrieNodes(t, rawdb.HashScheme)
|
||||
testMissingTrieNodes(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testMissingTrieNodes(t *testing.T, scheme string) {
|
||||
// Create an initial state with a few accounts
|
||||
memDb := rawdb.NewMemoryDatabase()
|
||||
db := NewDatabase(memDb)
|
||||
var (
|
||||
triedb *trie.Database
|
||||
memDb = rawdb.NewMemoryDatabase()
|
||||
)
|
||||
if scheme == rawdb.PathScheme {
|
||||
triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{
|
||||
CleanCacheSize: 0,
|
||||
DirtyCacheSize: 0,
|
||||
}}) // disable caching
|
||||
} else {
|
||||
triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{
|
||||
CleanCacheSize: 0,
|
||||
}}) // disable caching
|
||||
}
|
||||
db := NewDatabaseWithNodeDB(memDb, triedb)
|
||||
|
||||
var root common.Hash
|
||||
state, _ := New(types.EmptyRootHash, db, nil)
|
||||
addr := common.BytesToAddress([]byte("so"))
|
||||
@@ -762,7 +788,7 @@ func TestMissingTrieNodes(t *testing.T) {
|
||||
root, _ = state.Commit(0, false)
|
||||
t.Logf("root: %x", root)
|
||||
// force-flush
|
||||
state.Database().TrieDB().Cap(0)
|
||||
triedb.Commit(root, false)
|
||||
}
|
||||
// Create a new state on the old root
|
||||
state, _ = New(root, db, nil)
|
||||
@@ -969,7 +995,8 @@ func TestFlushOrderDataLoss(t *testing.T) {
|
||||
// Create a state trie with many accounts and slots
|
||||
var (
|
||||
memdb = rawdb.NewMemoryDatabase()
|
||||
statedb = NewDatabase(memdb)
|
||||
triedb = trie.NewDatabase(memdb, nil)
|
||||
statedb = NewDatabaseWithNodeDB(memdb, triedb)
|
||||
state, _ = New(types.EmptyRootHash, statedb, nil)
|
||||
)
|
||||
for a := byte(0); a < 10; a++ {
|
||||
@@ -982,11 +1009,11 @@ func TestFlushOrderDataLoss(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to commit state trie: %v", err)
|
||||
}
|
||||
statedb.TrieDB().Reference(root, common.Hash{})
|
||||
if err := statedb.TrieDB().Cap(1024); err != nil {
|
||||
triedb.Reference(root, common.Hash{})
|
||||
if err := triedb.Cap(1024); err != nil {
|
||||
t.Fatalf("failed to cap trie dirty cache: %v", err)
|
||||
}
|
||||
if err := statedb.TrieDB().Commit(root, false); err != nil {
|
||||
if err := triedb.Commit(root, false); err != nil {
|
||||
t.Fatalf("failed to commit state trie: %v", err)
|
||||
}
|
||||
// Reopen the state trie from flushed disk and verify it
|
||||
@@ -1040,7 +1067,7 @@ func TestStateDBTransientStorage(t *testing.T) {
|
||||
func TestResetObject(t *testing.T) {
|
||||
var (
|
||||
disk = rawdb.NewMemoryDatabase()
|
||||
tdb = trie.NewDatabase(disk)
|
||||
tdb = trie.NewDatabase(disk, nil)
|
||||
db = NewDatabaseWithNodeDB(disk, tdb)
|
||||
snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash)
|
||||
state, _ = New(types.EmptyRootHash, db, snaps)
|
||||
|
||||
+116
-53
@@ -28,6 +28,8 @@ import (
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
|
||||
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
|
||||
)
|
||||
|
||||
// testAccount is the data associated with an account used by the state tests.
|
||||
@@ -39,10 +41,17 @@ type testAccount struct {
|
||||
}
|
||||
|
||||
// makeTestState create a sample test state to test node-wise reconstruction.
|
||||
func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
|
||||
func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) {
|
||||
// Create an empty state
|
||||
config := &trie.Config{Preimages: true}
|
||||
if scheme == rawdb.PathScheme {
|
||||
config.PathDB = pathdb.Defaults
|
||||
} else {
|
||||
config.HashDB = hashdb.Defaults
|
||||
}
|
||||
db := rawdb.NewMemoryDatabase()
|
||||
sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true})
|
||||
nodeDb := trie.NewDatabase(db, config)
|
||||
sdb := NewDatabaseWithNodeDB(db, nodeDb)
|
||||
state, _ := New(types.EmptyRootHash, sdb, nil)
|
||||
|
||||
// Fill it with some arbitrary data
|
||||
@@ -67,24 +76,27 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
|
||||
obj.SetState(hash, hash)
|
||||
}
|
||||
}
|
||||
state.updateStateObject(obj)
|
||||
accounts = append(accounts, acc)
|
||||
}
|
||||
root, _ := state.Commit(0, false)
|
||||
|
||||
// Return the generated state
|
||||
return db, sdb, root, accounts
|
||||
return db, sdb, nodeDb, root, accounts
|
||||
}
|
||||
|
||||
// checkStateAccounts cross references a reconstructed state with an expected
|
||||
// account array.
|
||||
func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
|
||||
func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) {
|
||||
var config trie.Config
|
||||
if scheme == rawdb.PathScheme {
|
||||
config.PathDB = pathdb.Defaults
|
||||
}
|
||||
// Check root availability and state contents
|
||||
state, err := New(root, NewDatabase(db), nil)
|
||||
state, err := New(root, NewDatabaseWithConfig(db, &config), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create state trie at %x: %v", root, err)
|
||||
}
|
||||
if err := checkStateConsistency(db, root); err != nil {
|
||||
if err := checkStateConsistency(db, scheme, root); err != nil {
|
||||
t.Fatalf("inconsistent state trie at %x: %v", root, err)
|
||||
}
|
||||
for i, acc := range accounts {
|
||||
@@ -101,8 +113,12 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou
|
||||
}
|
||||
|
||||
// checkStateConsistency checks that all data of a state root is present.
|
||||
func checkStateConsistency(db ethdb.Database, root common.Hash) error {
|
||||
state, err := New(root, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil)
|
||||
func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error {
|
||||
config := &trie.Config{Preimages: true}
|
||||
if scheme == rawdb.PathScheme {
|
||||
config.PathDB = pathdb.Defaults
|
||||
}
|
||||
state, err := New(root, NewDatabaseWithConfig(db, config), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -114,8 +130,14 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
|
||||
|
||||
// Tests that an empty state is not scheduled for syncing.
|
||||
func TestEmptyStateSync(t *testing.T) {
|
||||
db := trie.NewDatabase(rawdb.NewMemoryDatabase())
|
||||
sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, db.Scheme())
|
||||
dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)
|
||||
dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults})
|
||||
|
||||
sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbA.Scheme())
|
||||
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
|
||||
t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
|
||||
}
|
||||
sync = NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbB.Scheme())
|
||||
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
|
||||
t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes)
|
||||
}
|
||||
@@ -124,22 +146,28 @@ func TestEmptyStateSync(t *testing.T) {
|
||||
// Tests that given a root hash, a state can sync iteratively on a single thread,
|
||||
// requesting retrieval tasks and returning all of them in one go.
|
||||
func TestIterativeStateSyncIndividual(t *testing.T) {
|
||||
testIterativeStateSync(t, 1, false, false)
|
||||
testIterativeStateSync(t, 1, false, false, rawdb.HashScheme)
|
||||
testIterativeStateSync(t, 1, false, false, rawdb.PathScheme)
|
||||
}
|
||||
func TestIterativeStateSyncBatched(t *testing.T) {
|
||||
testIterativeStateSync(t, 100, false, false)
|
||||
testIterativeStateSync(t, 100, false, false, rawdb.HashScheme)
|
||||
testIterativeStateSync(t, 100, false, false, rawdb.PathScheme)
|
||||
}
|
||||
func TestIterativeStateSyncIndividualFromDisk(t *testing.T) {
|
||||
testIterativeStateSync(t, 1, true, false)
|
||||
testIterativeStateSync(t, 1, true, false, rawdb.HashScheme)
|
||||
testIterativeStateSync(t, 1, true, false, rawdb.PathScheme)
|
||||
}
|
||||
func TestIterativeStateSyncBatchedFromDisk(t *testing.T) {
|
||||
testIterativeStateSync(t, 100, true, false)
|
||||
testIterativeStateSync(t, 100, true, false, rawdb.HashScheme)
|
||||
testIterativeStateSync(t, 100, true, false, rawdb.PathScheme)
|
||||
}
|
||||
func TestIterativeStateSyncIndividualByPath(t *testing.T) {
|
||||
testIterativeStateSync(t, 1, false, true)
|
||||
testIterativeStateSync(t, 1, false, true, rawdb.HashScheme)
|
||||
testIterativeStateSync(t, 1, false, true, rawdb.PathScheme)
|
||||
}
|
||||
func TestIterativeStateSyncBatchedByPath(t *testing.T) {
|
||||
testIterativeStateSync(t, 100, false, true)
|
||||
testIterativeStateSync(t, 100, false, true, rawdb.HashScheme)
|
||||
testIterativeStateSync(t, 100, false, true, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
// stateElement represents the element in the state trie(bytecode or trie node).
|
||||
@@ -150,17 +178,17 @@ type stateElement struct {
|
||||
syncPath trie.SyncPath
|
||||
}
|
||||
|
||||
func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
|
||||
func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, scheme string) {
|
||||
// Create a random state to copy
|
||||
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
|
||||
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
|
||||
if commit {
|
||||
srcDb.TrieDB().Commit(srcRoot, false)
|
||||
ndb.Commit(srcRoot, false)
|
||||
}
|
||||
srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB())
|
||||
srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), ndb)
|
||||
|
||||
// Create a destination state and sync with the scheduler
|
||||
dstDb := rawdb.NewMemoryDatabase()
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
|
||||
|
||||
var (
|
||||
nodeElements []stateElement
|
||||
@@ -175,9 +203,11 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
|
||||
})
|
||||
}
|
||||
for i := 0; i < len(codes); i++ {
|
||||
codeElements = append(codeElements, stateElement{
|
||||
code: codes[i],
|
||||
})
|
||||
codeElements = append(codeElements, stateElement{code: codes[i]})
|
||||
}
|
||||
reader, err := ndb.Reader(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("state is not existent, %#x", srcRoot)
|
||||
}
|
||||
for len(nodeElements)+len(codeElements) > 0 {
|
||||
var (
|
||||
@@ -205,7 +235,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
|
||||
t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err)
|
||||
}
|
||||
id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root)
|
||||
stTrie, err := trie.New(id, srcDb.TrieDB())
|
||||
stTrie, err := trie.New(id, ndb)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err)
|
||||
}
|
||||
@@ -216,7 +246,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
|
||||
nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data}
|
||||
}
|
||||
} else {
|
||||
data, err := srcDb.TrieDB().Node(node.hash)
|
||||
owner, inner := trie.ResolvePath([]byte(node.path))
|
||||
data, err := reader.Node(owner, inner, node.hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to retrieve node data for key %v", []byte(node.path))
|
||||
}
|
||||
@@ -260,18 +291,23 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
|
||||
copyPreimages(srcDisk, dstDb)
|
||||
|
||||
// Cross check that the two states are in sync
|
||||
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
|
||||
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
|
||||
}
|
||||
|
||||
// Tests that the trie scheduler can correctly reconstruct the state even if only
|
||||
// partial results are returned, and the others sent only later.
|
||||
func TestIterativeDelayedStateSync(t *testing.T) {
|
||||
testIterativeDelayedStateSync(t, rawdb.HashScheme)
|
||||
testIterativeDelayedStateSync(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testIterativeDelayedStateSync(t *testing.T, scheme string) {
|
||||
// Create a random state to copy
|
||||
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
|
||||
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
|
||||
|
||||
// Create a destination state and sync with the scheduler
|
||||
dstDb := rawdb.NewMemoryDatabase()
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
|
||||
|
||||
var (
|
||||
nodeElements []stateElement
|
||||
@@ -286,9 +322,11 @@ func TestIterativeDelayedStateSync(t *testing.T) {
|
||||
})
|
||||
}
|
||||
for i := 0; i < len(codes); i++ {
|
||||
codeElements = append(codeElements, stateElement{
|
||||
code: codes[i],
|
||||
})
|
||||
codeElements = append(codeElements, stateElement{code: codes[i]})
|
||||
}
|
||||
reader, err := ndb.Reader(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("state is not existent, %#x", srcRoot)
|
||||
}
|
||||
for len(nodeElements)+len(codeElements) > 0 {
|
||||
// Sync only half of the scheduled nodes
|
||||
@@ -313,7 +351,8 @@ func TestIterativeDelayedStateSync(t *testing.T) {
|
||||
if len(nodeElements) > 0 {
|
||||
nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1)
|
||||
for i, element := range nodeElements[:len(nodeResults)] {
|
||||
data, err := srcDb.TrieDB().Node(element.hash)
|
||||
owner, inner := trie.ResolvePath([]byte(element.path))
|
||||
data, err := reader.Node(owner, inner, element.hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to retrieve contract bytecode for %x", element.code)
|
||||
}
|
||||
@@ -353,22 +392,28 @@ func TestIterativeDelayedStateSync(t *testing.T) {
|
||||
copyPreimages(srcDisk, dstDb)
|
||||
|
||||
// Cross check that the two states are in sync
|
||||
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
|
||||
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
|
||||
}
|
||||
|
||||
// Tests that given a root hash, a trie can sync iteratively on a single thread,
|
||||
// requesting retrieval tasks and returning all of them in one go, however in a
|
||||
// random order.
|
||||
func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) }
|
||||
func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) }
|
||||
func TestIterativeRandomStateSyncIndividual(t *testing.T) {
|
||||
testIterativeRandomStateSync(t, 1, rawdb.HashScheme)
|
||||
testIterativeRandomStateSync(t, 1, rawdb.PathScheme)
|
||||
}
|
||||
func TestIterativeRandomStateSyncBatched(t *testing.T) {
|
||||
testIterativeRandomStateSync(t, 100, rawdb.HashScheme)
|
||||
testIterativeRandomStateSync(t, 100, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testIterativeRandomStateSync(t *testing.T, count int) {
|
||||
func testIterativeRandomStateSync(t *testing.T, count int, scheme string) {
|
||||
// Create a random state to copy
|
||||
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
|
||||
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
|
||||
|
||||
// Create a destination state and sync with the scheduler
|
||||
dstDb := rawdb.NewMemoryDatabase()
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
|
||||
|
||||
nodeQueue := make(map[string]stateElement)
|
||||
codeQueue := make(map[common.Hash]struct{})
|
||||
@@ -383,6 +428,10 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
|
||||
for _, hash := range codes {
|
||||
codeQueue[hash] = struct{}{}
|
||||
}
|
||||
reader, err := ndb.Reader(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("state is not existent, %#x", srcRoot)
|
||||
}
|
||||
for len(nodeQueue)+len(codeQueue) > 0 {
|
||||
// Fetch all the queued nodes in a random order
|
||||
if len(codeQueue) > 0 {
|
||||
@@ -403,7 +452,8 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
|
||||
if len(nodeQueue) > 0 {
|
||||
results := make([]trie.NodeSyncResult, 0, len(nodeQueue))
|
||||
for path, element := range nodeQueue {
|
||||
data, err := srcDb.TrieDB().Node(element.hash)
|
||||
owner, inner := trie.ResolvePath([]byte(element.path))
|
||||
data, err := reader.Node(owner, inner, element.hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path)
|
||||
}
|
||||
@@ -415,7 +465,6 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Feed the retrieved results back and queue new tasks
|
||||
batch := dstDb.NewBatch()
|
||||
if err := sched.Commit(batch); err != nil {
|
||||
t.Fatalf("failed to commit data: %v", err)
|
||||
@@ -441,18 +490,23 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
|
||||
copyPreimages(srcDisk, dstDb)
|
||||
|
||||
// Cross check that the two states are in sync
|
||||
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
|
||||
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
|
||||
}
|
||||
|
||||
// Tests that the trie scheduler can correctly reconstruct the state even if only
|
||||
// partial results are returned (Even those randomly), others sent only later.
|
||||
func TestIterativeRandomDelayedStateSync(t *testing.T) {
|
||||
testIterativeRandomDelayedStateSync(t, rawdb.HashScheme)
|
||||
testIterativeRandomDelayedStateSync(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) {
|
||||
// Create a random state to copy
|
||||
srcDisk, srcDb, srcRoot, srcAccounts := makeTestState()
|
||||
srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
|
||||
|
||||
// Create a destination state and sync with the scheduler
|
||||
dstDb := rawdb.NewMemoryDatabase()
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
|
||||
|
||||
nodeQueue := make(map[string]stateElement)
|
||||
codeQueue := make(map[common.Hash]struct{})
|
||||
@@ -467,6 +521,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
|
||||
for _, hash := range codes {
|
||||
codeQueue[hash] = struct{}{}
|
||||
}
|
||||
reader, err := ndb.Reader(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("state is not existent, %#x", srcRoot)
|
||||
}
|
||||
for len(nodeQueue)+len(codeQueue) > 0 {
|
||||
// Sync only half of the scheduled nodes, even those in random order
|
||||
if len(codeQueue) > 0 {
|
||||
@@ -495,7 +553,8 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
|
||||
for path, element := range nodeQueue {
|
||||
delete(nodeQueue, path)
|
||||
|
||||
data, err := srcDb.TrieDB().Node(element.hash)
|
||||
owner, inner := trie.ResolvePath([]byte(element.path))
|
||||
data, err := reader.Node(owner, inner, element.hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to retrieve node data for %x", element.hash)
|
||||
}
|
||||
@@ -535,14 +594,19 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
|
||||
copyPreimages(srcDisk, dstDb)
|
||||
|
||||
// Cross check that the two states are in sync
|
||||
checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
|
||||
checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts)
|
||||
}
|
||||
|
||||
// Tests that at any point in time during a sync, only complete sub-tries are in
|
||||
// the database.
|
||||
func TestIncompleteStateSync(t *testing.T) {
|
||||
testIncompleteStateSync(t, rawdb.HashScheme)
|
||||
testIncompleteStateSync(t, rawdb.PathScheme)
|
||||
}
|
||||
|
||||
func testIncompleteStateSync(t *testing.T, scheme string) {
|
||||
// Create a random state to copy
|
||||
db, srcDb, srcRoot, srcAccounts := makeTestState()
|
||||
db, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme)
|
||||
|
||||
// isCodeLookup to save some hashing
|
||||
var isCode = make(map[common.Hash]struct{})
|
||||
@@ -555,14 +619,14 @@ func TestIncompleteStateSync(t *testing.T) {
|
||||
|
||||
// Create a destination state and sync with the scheduler
|
||||
dstDb := rawdb.NewMemoryDatabase()
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme())
|
||||
sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme())
|
||||
|
||||
var (
|
||||
addedCodes []common.Hash
|
||||
addedPaths []string
|
||||
addedHashes []common.Hash
|
||||
)
|
||||
reader, err := srcDb.TrieDB().Reader(srcRoot)
|
||||
reader, err := ndb.Reader(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("state is not available %x", srcRoot)
|
||||
}
|
||||
@@ -649,12 +713,11 @@ func TestIncompleteStateSync(t *testing.T) {
|
||||
for _, node := range addedCodes {
|
||||
val := rawdb.ReadCode(dstDb, node)
|
||||
rawdb.DeleteCode(dstDb, node)
|
||||
if err := checkStateConsistency(dstDb, srcRoot); err == nil {
|
||||
if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil {
|
||||
t.Errorf("trie inconsistency not caught, missing: %x", node)
|
||||
}
|
||||
rawdb.WriteCode(dstDb, node, val)
|
||||
}
|
||||
scheme := srcDb.TrieDB().Scheme()
|
||||
for i, path := range addedPaths {
|
||||
owner, inner := trie.ResolvePath([]byte(path))
|
||||
hash := addedHashes[i]
|
||||
@@ -663,7 +726,7 @@ func TestIncompleteStateSync(t *testing.T) {
|
||||
t.Error("missing trie node")
|
||||
}
|
||||
rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme)
|
||||
if err := checkStateConsistency(dstDb, srcRoot); err == nil {
|
||||
if err := checkStateConsistency(dstDb, scheme, srcRoot); err == nil {
|
||||
t.Errorf("trie inconsistency not caught, missing: %v", path)
|
||||
}
|
||||
rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme)
|
||||
|
||||
Reference in New Issue
Block a user