roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

automated merge

Browse Source
This commit is contained in:
philip-morlier
2023-07-11 16:01:17 -07:00
parent 366f480bc1 e501b3b05d
commit e516d5842c
327 changed files with 15001 additions and 11151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trie/committer.go
+35 -42
View File
@@ -20,26 +20,23 @@ import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// leaf represents a trie leaf node
type leaf struct {
blob []byte // raw blob of leaf
parent common.Hash // the hash of parent node
}
// committer is the tool used for the trie Commit operation. The committer will
// capture all dirty nodes during the commit process and keep them cached in
// insertion order.
type committer struct {
nodes *NodeSet
nodes *trienode.NodeSet
tracer *tracer
collectLeaf bool
}
// newCommitter creates a new committer or picks one from the pool.
func newCommitter(nodeset *NodeSet, collectLeaf bool) *committer {
func newCommitter(nodeset *trienode.NodeSet, tracer *tracer, collectLeaf bool) *committer {
return &committer{
nodes: nodeset,
tracer: tracer,
collectLeaf: collectLeaf,
}
}
@@ -134,24 +131,22 @@ func (c *committer) store(path []byte, n node) node {
// The node is embedded in its parent, in other words, this node
// will not be stored in the database independently, mark it as
// deleted only if the node was existent in database before.
if _, ok := c.nodes.accessList[string(path)]; ok {
c.nodes.markDeleted(path)
prev, ok := c.tracer.accessList[string(path)]
if ok {
c.nodes.AddNode(path, trienode.NewWithPrev(common.Hash{}, nil, prev))
}
return n
}
// We have the hash already, estimate the RLP encoding-size of the node.
// The size is used for mem tracking, does not need to be exact
var (
size = estimateSize(n)
nhash = common.BytesToHash(hash)
mnode = &memoryNode{
hash: nhash,
node: simplifyNode(n),
size: uint16(size),
}
)
// Collect the dirty node to nodeset for return.
c.nodes.markUpdated(path, mnode)
var (
nhash = common.BytesToHash(hash)
node = trienode.NewWithPrev(
nhash,
nodeToBytes(n),
c.tracer.accessList[string(path)],
)
)
c.nodes.AddNode(path, node)
// Collect the corresponding leaf node if it's required. We don't check
// full node since it's impossible to store value in fullNode. The key
@@ -159,38 +154,36 @@ func (c *committer) store(path []byte, n node) node {
if c.collectLeaf {
if sn, ok := n.(*shortNode); ok {
if val, ok := sn.Val.(valueNode); ok {
c.nodes.addLeaf(&leaf{blob: val, parent: nhash})
c.nodes.AddLeaf(nhash, val)
}
}
}
return hash
}
// estimateSize estimates the size of an rlp-encoded node, without actually
// rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie
// with 1000 leaves, the only errors above 1% are on small shortnodes, where this
// method overestimates by 2 or 3 bytes (e.g. 37 instead of 35)
func estimateSize(n node) int {
// mptResolver the children resolver in merkle-patricia-tree.
type mptResolver struct{}
// ForEach implements childResolver, decodes the provided node and
// traverses the children inside.
func (resolver mptResolver) ForEach(node []byte, onChild func(common.Hash)) {
forGatherChildren(mustDecodeNodeUnsafe(nil, node), onChild)
}
// forGatherChildren traverses the node hierarchy and invokes the callback
// for all the hashnode children.
func forGatherChildren(n node, onChild func(hash common.Hash)) {
switch n := n.(type) {
case *shortNode:
// A short node contains a compacted key, and a value.
return 3 + len(n.Key) + estimateSize(n.Val)
forGatherChildren(n.Val, onChild)
case *fullNode:
// A full node contains up to 16 hashes (some nils), and a key
s := 3
for i := 0; i < 16; i++ {
if child := n.Children[i]; child != nil {
s += estimateSize(child)
} else {
s++
}
forGatherChildren(n.Children[i], onChild)
}
return s
case valueNode:
return 1 + len(n)
case hashNode:
return 1 + len(n)
onChild(common.BytesToHash(n))
case valueNode, nil:
default:
panic(fmt.Sprintf("node type %T", n))
panic(fmt.Sprintf("unknown node type: %T", n))
}
}
trie/database_test.go
+11 -9
View File
@@ -17,17 +17,19 @@
package trie
import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
)
// Tests that the trie database returns a missing trie node error if attempting
// to retrieve the meta root.
func TestDatabaseMetarootFetch(t *testing.T) {
db := NewDatabase(rawdb.NewMemoryDatabase())
if _, err := db.Node(common.Hash{}); err == nil {
t.Fatalf("metaroot retrieval succeeded")
// newTestDatabase initializes the trie database with specified scheme.
func newTestDatabase(diskdb ethdb.Database, scheme string) *Database {
db := prepare(diskdb, nil)
if scheme == rawdb.HashScheme {
db.backend = hashdb.New(diskdb, db.cleans, mptResolver{})
}
//} else {
// db.backend = snap.New(diskdb, db.cleans, nil)
//}
return db
}
trie/database_wrap.go
+267
View File
@@ -0,0 +1,267 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"errors"
"runtime"
"time"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// Config defines all necessary options for database.
type Config struct {
Cache int // Memory allowance (MB) to use for caching trie nodes in memory
Journal string // Journal of clean cache to survive node restarts
Preimages bool // Flag whether the preimage of trie key is recorded
}
// backend defines the methods needed to access/update trie nodes in different
// state scheme.
type backend interface {
// Scheme returns the identifier of used storage scheme.
Scheme() string
// Initialized returns an indicator if the state data is already initialized
// according to the state scheme.
Initialized(genesisRoot common.Hash) bool
// Size returns the current storage size of the memory cache in front of the
// persistent database layer.
Size() common.StorageSize
// Update performs a state transition by committing dirty nodes contained
// in the given set in order to update state from the specified parent to
// the specified root.
Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error
// Commit writes all relevant trie nodes belonging to the specified state
// to disk. Report specifies whether logs will be displayed in info level.
Commit(root common.Hash, report bool) error
// Close closes the trie database backend and releases all held resources.
Close() error
}
// Database is the wrapper of the underlying backend which is shared by different
// types of node backend as an entrypoint. It's responsible for all interactions
// relevant with trie nodes and node preimages.
type Database struct {
config *Config // Configuration for trie database
diskdb ethdb.Database // Persistent database to store the snapshot
cleans *fastcache.Cache // Megabytes permitted using for read caches
preimages *preimageStore // The store for caching preimages
backend backend // The backend for managing trie nodes
}
// prepare initializes the database with provided configs, but the
// database backend is still left as nil.
func prepare(diskdb ethdb.Database, config *Config) *Database {
var cleans *fastcache.Cache
if config != nil && config.Cache > 0 {
if config.Journal == "" {
cleans = fastcache.New(config.Cache * 1024 * 1024)
} else {
cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024)
}
}
var preimages *preimageStore
if config != nil && config.Preimages {
preimages = newPreimageStore(diskdb)
}
return &Database{
config: config,
diskdb: diskdb,
cleans: cleans,
preimages: preimages,
}
}
// NewDatabase initializes the trie database with default settings, namely
// the legacy hash-based scheme is used by default.
func NewDatabase(diskdb ethdb.Database) *Database {
return NewDatabaseWithConfig(diskdb, nil)
}
// NewDatabaseWithConfig initializes the trie database with provided configs.
// The path-based scheme is not activated yet, always initialized with legacy
// hash-based scheme by default.
func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database {
db := prepare(diskdb, config)
db.backend = hashdb.New(diskdb, db.cleans, mptResolver{})
return db
}
// Reader returns a reader for accessing all trie nodes with provided state root.
// Nil is returned in case the state is not available.
func (db *Database) Reader(blockRoot common.Hash) Reader {
return db.backend.(*hashdb.Database).Reader(blockRoot)
}
// Update performs a state transition by committing dirty nodes contained in the
// given set in order to update state from the specified parent to the specified
// root. The held pre-images accumulated up to this point will be flushed in case
// the size exceeds the threshold.
func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error {
if db.preimages != nil {
db.preimages.commit(false)
}
return db.backend.Update(root, parent, nodes)
}
// Commit iterates over all the children of a particular node, writes them out
// to disk. As a side effect, all pre-images accumulated up to this point are
// also written.
func (db *Database) Commit(root common.Hash, report bool) error {
if db.preimages != nil {
db.preimages.commit(true)
}
return db.backend.Commit(root, report)
}
// Size returns the storage size of dirty trie nodes in front of the persistent
// database and the size of cached preimages.
func (db *Database) Size() (common.StorageSize, common.StorageSize) {
var (
storages common.StorageSize
preimages common.StorageSize
)
storages = db.backend.Size()
if db.preimages != nil {
preimages = db.preimages.size()
}
return storages, preimages
}
// Initialized returns an indicator if the state data is already initialized
// according to the state scheme.
func (db *Database) Initialized(genesisRoot common.Hash) bool {
return db.backend.Initialized(genesisRoot)
}
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string {
return db.backend.Scheme()
}
// Close flushes the dangling preimages to disk and closes the trie database.
// It is meant to be called when closing the blockchain object, so that all
// resources held can be released correctly.
func (db *Database) Close() error {
if db.preimages != nil {
db.preimages.commit(true)
}
return db.backend.Close()
}
// saveCache saves clean state cache to given directory path
// using specified CPU cores.
func (db *Database) saveCache(dir string, threads int) error {
if db.cleans == nil {
return nil
}
log.Info("Writing clean trie cache to disk", "path", dir, "threads", threads)
start := time.Now()
err := db.cleans.SaveToFileConcurrent(dir, threads)
if err != nil {
log.Error("Failed to persist clean trie cache", "error", err)
return err
}
log.Info("Persisted the clean trie cache", "path", dir, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}
// SaveCache atomically saves fast cache data to the given dir using all
// available CPU cores.
func (db *Database) SaveCache(dir string) error {
return db.saveCache(dir, runtime.GOMAXPROCS(0))
}
// SaveCachePeriodically atomically saves fast cache data to the given dir with
// the specified interval. All dump operation will only use a single CPU core.
func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, stopCh <-chan struct{}) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
db.saveCache(dir, 1)
case <-stopCh:
return
}
}
}
// Cap iteratively flushes old but still referenced trie nodes until the total
// memory usage goes below the given threshold. The held pre-images accumulated
// up to this point will be flushed in case the size exceeds the threshold.
//
// It's only supported by hash-based database and will return an error for others.
func (db *Database) Cap(limit common.StorageSize) error {
hdb, ok := db.backend.(*hashdb.Database)
if !ok {
return errors.New("not supported")
}
if db.preimages != nil {
db.preimages.commit(false)
}
return hdb.Cap(limit)
}
// Reference adds a new reference from a parent node to a child node. This function
// is used to add reference between internal trie node and external node(e.g. storage
// trie root), all internal trie nodes are referenced together by database itself.
//
// It's only supported by hash-based database and will return an error for others.
func (db *Database) Reference(root common.Hash, parent common.Hash) error {
hdb, ok := db.backend.(*hashdb.Database)
if !ok {
return errors.New("not supported")
}
hdb.Reference(root, parent)
return nil
}
// Dereference removes an existing reference from a root node. It's only
// supported by hash-based database and will return an error for others.
func (db *Database) Dereference(root common.Hash) error {
hdb, ok := db.backend.(*hashdb.Database)
if !ok {
return errors.New("not supported")
}
hdb.Dereference(root)
return nil
}
// Node retrieves the rlp-encoded node blob with provided node hash. It's
// only supported by hash-based database and will return an error for others.
// Note, this function should be deprecated once ETH66 is deprecated.
func (db *Database) Node(hash common.Hash) ([]byte, error) {
hdb, ok := db.backend.(*hashdb.Database)
if !ok {
return nil, errors.New("not supported")
}
return hdb.Node(hash)
}
trie/iterator.go
+9 -2
View File
@@ -387,7 +387,14 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) {
// loaded blob will be tracked, while it's not required here since
// all loaded nodes won't be linked to trie at all and track nodes
// may lead to out-of-memory issue.
return it.trie.reader.node(path, common.BytesToHash(hash))
blob, err := it.trie.reader.node(path, common.BytesToHash(hash))
if err != nil {
return nil, err
}
// The raw-blob format nodes are loaded either from the
// clean cache or the database, they are all in their own
// copy and safe to use unsafe decoder.
return mustDecodeNodeUnsafe(hash, blob), nil
}
func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) {
@@ -401,7 +408,7 @@ func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error)
// loaded blob will be tracked, while it's not required here since
// all loaded nodes won't be linked to trie at all and track nodes
// may lead to out-of-memory issue.
return it.trie.reader.nodeBlob(path, common.BytesToHash(hash))
return it.trie.reader.node(path, common.BytesToHash(hash))
}
func (st *nodeIteratorState) resolve(it *nodeIterator, path []byte) error {
trie/iterator_test.go
+164 -79
View File
@@ -25,9 +25,11 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/trie/trienode"
)
func TestEmptyIterator(t *testing.T) {
@@ -61,7 +63,7 @@ func TestIterator(t *testing.T) {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
found := make(map[string]string)
@@ -115,39 +117,61 @@ func TestIteratorLargeData(t *testing.T) {
}
}
type iterationElement struct {
hash common.Hash
path []byte
blob []byte
}
// Tests that the node iterator indeed walks over the entire database contents.
func TestNodeIteratorCoverage(t *testing.T) {
testNodeIteratorCoverage(t, rawdb.HashScheme)
//testNodeIteratorCoverage(t, rawdb.PathScheme)
}
func testNodeIteratorCoverage(t *testing.T, scheme string) {
// Create some arbitrary test trie to iterate
db, trie, _ := makeTestTrie()
db, nodeDb, trie, _ := makeTestTrie(scheme)
// Gather all the node hashes found by the iterator
hashes := make(map[common.Hash]struct{})
var elements = make(map[common.Hash]iterationElement)
for it := trie.NodeIterator(nil); it.Next(true); {
if it.Hash() != (common.Hash{}) {
hashes[it.Hash()] = struct{}{}
}
}
// Cross check the hashes and the database itself
for hash := range hashes {
if _, err := db.Node(hash); err != nil {
t.Errorf("failed to retrieve reported node %x: %v", hash, err)
}
}
for hash, obj := range db.dirties {
if obj != nil && hash != (common.Hash{}) {
if _, ok := hashes[hash]; !ok {
t.Errorf("state entry not reported %x", hash)
elements[it.Hash()] = iterationElement{
hash: it.Hash(),
path: common.CopyBytes(it.Path()),
blob: common.CopyBytes(it.NodeBlob()),
}
}
}
it := db.diskdb.NewIterator(nil, nil)
// Cross check the hashes and the database itself
for _, element := range elements {
if blob, err := nodeDb.Reader(trie.Hash()).Node(common.Hash{}, element.path, element.hash); err != nil {
t.Errorf("failed to retrieve reported node %x: %v", element.hash, err)
} else if !bytes.Equal(blob, element.blob) {
t.Errorf("node blob is different, want %v got %v", element.blob, blob)
}
}
var (
count int
it = db.NewIterator(nil, nil)
)
for it.Next() {
key := it.Key()
if _, ok := hashes[common.BytesToHash(key)]; !ok {
t.Errorf("state entry not reported %x", key)
res, _, _ := isTrieNode(nodeDb.Scheme(), it.Key(), it.Value())
if !res {
continue
}
count += 1
if elem, ok := elements[crypto.Keccak256Hash(it.Value())]; !ok {
t.Error("state entry not reported")
} else if !bytes.Equal(it.Value(), elem.blob) {
t.Errorf("node blob is different, want %v got %v", elem.blob, it.Value())
}
}
it.Release()
if count != len(elements) {
t.Errorf("state entry is mismatched %d %d", count, len(elements))
}
}
type kvs struct{ k, v string }
@@ -223,7 +247,7 @@ func TestDifferenceIterator(t *testing.T) {
triea.MustUpdate([]byte(val.k), []byte(val.v))
}
rootA, nodesA := triea.Commit(false)
dba.Update(NewWithNodeSet(nodesA))
dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA))
triea, _ = New(TrieID(rootA), dba)
dbb := NewDatabase(rawdb.NewMemoryDatabase())
@@ -232,7 +256,7 @@ func TestDifferenceIterator(t *testing.T) {
trieb.MustUpdate([]byte(val.k), []byte(val.v))
}
rootB, nodesB := trieb.Commit(false)
dbb.Update(NewWithNodeSet(nodesB))
dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB))
trieb, _ = New(TrieID(rootB), dbb)
found := make(map[string]string)
@@ -265,7 +289,7 @@ func TestUnionIterator(t *testing.T) {
triea.MustUpdate([]byte(val.k), []byte(val.v))
}
rootA, nodesA := triea.Commit(false)
dba.Update(NewWithNodeSet(nodesA))
dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA))
triea, _ = New(TrieID(rootA), dba)
dbb := NewDatabase(rawdb.NewMemoryDatabase())
@@ -274,7 +298,7 @@ func TestUnionIterator(t *testing.T) {
trieb.MustUpdate([]byte(val.k), []byte(val.v))
}
rootB, nodesB := trieb.Commit(false)
dbb.Update(NewWithNodeSet(nodesB))
dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB))
trieb, _ = New(TrieID(rootB), dbb)
di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)})
@@ -320,79 +344,98 @@ func TestIteratorNoDups(t *testing.T) {
}
// This test checks that nodeIterator.Next can be retried after inserting missing trie nodes.
func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueAfterError(t, false) }
func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) }
func TestIteratorContinueAfterError(t *testing.T) {
testIteratorContinueAfterError(t, false, rawdb.HashScheme)
testIteratorContinueAfterError(t, true, rawdb.HashScheme)
// testIteratorContinueAfterError(t, false, rawdb.PathScheme)
// testIteratorContinueAfterError(t, true, rawdb.PathScheme)
}
func testIteratorContinueAfterError(t *testing.T, memonly bool) {
func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) {
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
tdb := newTestDatabase(diskdb, scheme)
tr := NewEmpty(triedb)
tr := NewEmpty(tdb)
for _, val := range testdata1 {
tr.MustUpdate([]byte(val.k), []byte(val.v))
}
_, nodes := tr.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
root, nodes := tr.Commit(false)
tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(tr.Hash(), false)
tdb.Commit(root, false)
}
tr, _ = New(TrieID(root), tdb)
wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
var (
diskKeys [][]byte
memKeys []common.Hash
paths [][]byte
hashes []common.Hash
)
if memonly {
memKeys = triedb.Nodes()
for path, n := range nodes.Nodes {
paths = append(paths, []byte(path))
hashes = append(hashes, n.Hash)
}
} else {
it := diskdb.NewIterator(nil, nil)
for it.Next() {
diskKeys = append(diskKeys, it.Key())
ok, path, hash := isTrieNode(tdb.Scheme(), it.Key(), it.Value())
if !ok {
continue
}
paths = append(paths, path)
hashes = append(hashes, hash)
}
it.Release()
}
for i := 0; i < 20; i++ {
// Create trie that will load all nodes from DB.
tr, _ := New(TrieID(tr.Hash()), triedb)
tr, _ := New(TrieID(tr.Hash()), tdb)
// Remove a random node from the database. It can't be the root node
// because that one is already loaded.
var (
rkey common.Hash
rval []byte
robj *cachedNode
rval []byte
rpath []byte
rhash common.Hash
)
for {
if memonly {
rkey = memKeys[rand.Intn(len(memKeys))]
rpath = paths[rand.Intn(len(paths))]
n := nodes.Nodes[string(rpath)]
if n == nil {
continue
}
rhash = n.Hash
} else {
copy(rkey[:], diskKeys[rand.Intn(len(diskKeys))])
index := rand.Intn(len(paths))
rpath = paths[index]
rhash = hashes[index]
}
if rkey != tr.Hash() {
if rhash != tr.Hash() {
break
}
}
if memonly {
robj = triedb.dirties[rkey]
delete(triedb.dirties, rkey)
tr.reader.banned = map[string]struct{}{string(rpath): {}}
} else {
rval, _ = diskdb.Get(rkey[:])
diskdb.Delete(rkey[:])
rval = rawdb.ReadTrieNode(diskdb, common.Hash{}, rpath, rhash, tdb.Scheme())
rawdb.DeleteTrieNode(diskdb, common.Hash{}, rpath, rhash, tdb.Scheme())
}
// Iterate until the error is hit.
seen := make(map[string]bool)
it := tr.NodeIterator(nil)
checkIteratorNoDups(t, it, seen)
missing, ok := it.Error().(*MissingNodeError)
if !ok || missing.NodeHash != rkey {
if !ok || missing.NodeHash != rhash {
t.Fatal("didn't hit missing node, got", it.Error())
}
// Add the node back and continue iteration.
if memonly {
triedb.dirties[rkey] = robj
delete(tr.reader.banned, string(rpath))
} else {
diskdb.Put(rkey[:], rval)
rawdb.WriteTrieNode(diskdb, common.Hash{}, rpath, rhash, rval, tdb.Scheme())
}
checkIteratorNoDups(t, it, seen)
if it.Error() != nil {
@@ -407,42 +450,48 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
// Similar to the test above, this one checks that failure to create nodeIterator at a
// certain key prefix behaves correctly when Next is called. The expectation is that Next
// should retry seeking before returning true for the first time.
func TestIteratorContinueAfterSeekErrorDisk(t *testing.T) {
testIteratorContinueAfterSeekError(t, false)
}
func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) {
testIteratorContinueAfterSeekError(t, true)
func TestIteratorContinueAfterSeekError(t *testing.T) {
testIteratorContinueAfterSeekError(t, false, rawdb.HashScheme)
testIteratorContinueAfterSeekError(t, true, rawdb.HashScheme)
// testIteratorContinueAfterSeekError(t, false, rawdb.PathScheme)
// testIteratorContinueAfterSeekError(t, true, rawdb.PathScheme)
}
func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme string) {
// Commit test trie to db, then remove the node containing "bars".
var (
barNodePath []byte
barNodeHash = common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
)
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
triedb := newTestDatabase(diskdb, scheme)
ctr := NewEmpty(triedb)
for _, val := range testdata1 {
ctr.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := ctr.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
for path, n := range nodes.Nodes {
if n.Hash == barNodeHash {
barNodePath = []byte(path)
break
}
}
triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(root, false)
}
barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
var (
barNodeBlob []byte
barNodeObj *cachedNode
)
tr, _ := New(TrieID(root), triedb)
if memonly {
barNodeObj = triedb.dirties[barNodeHash]
delete(triedb.dirties, barNodeHash)
tr.reader.banned = map[string]struct{}{string(barNodePath): {}}
} else {
barNodeBlob, _ = diskdb.Get(barNodeHash[:])
diskdb.Delete(barNodeHash[:])
barNodeBlob = rawdb.ReadTrieNode(diskdb, common.Hash{}, barNodePath, barNodeHash, triedb.Scheme())
rawdb.DeleteTrieNode(diskdb, common.Hash{}, barNodePath, barNodeHash, triedb.Scheme())
}
// Create a new iterator that seeks to "bars". Seeking can't proceed because
// the node is missing.
tr, _ := New(TrieID(root), triedb)
it := tr.NodeIterator([]byte("bars"))
missing, ok := it.Error().(*MissingNodeError)
if !ok {
@@ -452,9 +501,9 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
}
// Reinsert the missing node.
if memonly {
triedb.dirties[barNodeHash] = barNodeObj
delete(tr.reader.banned, string(barNodePath))
} else {
diskdb.Put(barNodeHash[:], barNodeBlob)
rawdb.WriteTrieNode(diskdb, common.Hash{}, barNodePath, barNodeHash, barNodeBlob, triedb.Scheme())
}
// Check that iteration produces the right set of values.
if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil {
@@ -475,6 +524,11 @@ func checkIteratorNoDups(t *testing.T, it NodeIterator, seen map[string]bool) in
return len(seen)
}
func TestIteratorNodeBlob(t *testing.T) {
testIteratorNodeBlob(t, rawdb.HashScheme)
//testIteratorNodeBlob(t, rawdb.PathScheme)
}
type loggingDb struct {
getCount uint64
backend ethdb.KeyValueStore
@@ -530,7 +584,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) {
// Create an empty trie
logDb := &loggingDb{0, memorydb.New()}
triedb := NewDatabase(rawdb.NewDatabase(logDb))
trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb)
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data
for i := 0; i < 10000; i++ {
@@ -542,8 +596,8 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) {
val = crypto.Keccak256(val)
trie.MustUpdate(key, val)
}
_, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
root, nodes := trie.Commit(false)
triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
// Return the generated trie
return triedb, trie, logDb
}
@@ -562,10 +616,10 @@ func TestNodeIteratorLargeTrie(t *testing.T) {
}
}
func TestIteratorNodeBlob(t *testing.T) {
func testIteratorNodeBlob(t *testing.T, scheme string) {
var (
db = rawdb.NewMemoryDatabase()
triedb = NewDatabase(db)
triedb = newTestDatabase(db, scheme)
trie = NewEmpty(triedb)
)
vals := []struct{ k, v string }{
@@ -582,11 +636,12 @@ func TestIteratorNodeBlob(t *testing.T) {
all[val.k] = val.v
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
_, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
triedb.Cap(0)
root, nodes := trie.Commit(false)
triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
triedb.Commit(root, false)
found := make(map[common.Hash][]byte)
var found = make(map[common.Hash][]byte)
trie, _ = New(TrieID(root), triedb)
it := trie.NodeIterator(nil)
for it.Next(true) {
if it.Hash() == (common.Hash{}) {
@@ -600,9 +655,13 @@ func TestIteratorNodeBlob(t *testing.T) {
var count int
for dbIter.Next() {
got, present := found[common.BytesToHash(dbIter.Key())]
ok, _, _ := isTrieNode(triedb.Scheme(), dbIter.Key(), dbIter.Value())
if !ok {
continue
}
got, present := found[crypto.Keccak256Hash(dbIter.Value())]
if !present {
t.Fatalf("Miss trie node %v", dbIter.Key())
t.Fatal("Miss trie node")
}
if !bytes.Equal(got, dbIter.Value()) {
t.Fatalf("Unexpected trie node want %v got %v", dbIter.Value(), got)
@@ -613,3 +672,29 @@ func TestIteratorNodeBlob(t *testing.T) {
t.Fatal("Find extra trie node via iterator")
}
}
// isTrieNode is a helper function which reports if the provided
// database entry belongs to a trie node or not. Note in tests
// only single layer trie is used, namely storage trie is not
// considered at all.
func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) {
var (
path []byte
hash common.Hash
)
if scheme == rawdb.HashScheme {
ok := rawdb.IsLegacyTrieNode(key, val)
if !ok {
return false, nil, common.Hash{}
}
hash = common.BytesToHash(key)
} else {
ok, remain := rawdb.IsAccountTrieNode(key)
if !ok {
return false, nil, common.Hash{}
}
path = common.CopyBytes(remain)
hash = crypto.Keccak256Hash(val)
}
return true, path, hash
}
trie/node.go
+13
View File
@@ -99,6 +99,19 @@ func (n valueNode) fstring(ind string) string {
return fmt.Sprintf("%x ", []byte(n))
}
// rawNode is a simple binary blob used to differentiate between collapsed trie
// nodes and already encoded RLP binary blobs (while at the same time store them
// in the same cache fields).
type rawNode []byte
func (n rawNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
func (n rawNode) fstring(ind string) string { panic("this should never end up in a live trie") }
func (n rawNode) EncodeRLP(w io.Writer) error {
_, err := w.Write(n)
return err
}
// mustDecodeNode is a wrapper of decodeNode and panic if any error is encountered.
func mustDecodeNode(hash, buf []byte) node {
n, err := decodeNode(hash, buf)
trie/node_enc.go
-23
View File
@@ -59,29 +59,6 @@ func (n valueNode) encode(w rlp.EncoderBuffer) {
w.WriteBytes(n)
}
func (n rawFullNode) encode(w rlp.EncoderBuffer) {
offset := w.List()
for _, c := range n {
if c != nil {
c.encode(w)
} else {
w.Write(rlp.EmptyString)
}
}
w.ListEnd(offset)
}
func (n *rawShortNode) encode(w rlp.EncoderBuffer) {
offset := w.List()
w.WriteBytes(n.Key)
if n.Val != nil {
n.Val.encode(w)
} else {
w.Write(rlp.EmptyString)
}
w.ListEnd(offset)
}
func (n rawNode) encode(w rlp.EncoderBuffer) {
w.Write(n)
}
trie/nodeset.go
-218
View File
@@ -1,218 +0,0 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"fmt"
"reflect"
"sort"
"strings"
"github.com/ethereum/go-ethereum/common"
)
// memoryNode is all the information we know about a single cached trie node
// in the memory.
type memoryNode struct {
hash common.Hash // Node hash, computed by hashing rlp value, empty for deleted nodes
size uint16 // Byte size of the useful cached data, 0 for deleted nodes
node node // Cached collapsed trie node, or raw rlp data, nil for deleted nodes
}
// memoryNodeSize is the raw size of a memoryNode data structure without any
// node data included. It's an approximate size, but should be a lot better
// than not counting them.
// nolint:unused
var memoryNodeSize = int(reflect.TypeOf(memoryNode{}).Size())
// memorySize returns the total memory size used by this node.
// nolint:unused
func (n *memoryNode) memorySize(pathlen int) int {
return int(n.size) + memoryNodeSize + pathlen
}
// rlp returns the raw rlp encoded blob of the cached trie node, either directly
// from the cache, or by regenerating it from the collapsed node.
// nolint:unused
func (n *memoryNode) rlp() []byte {
if node, ok := n.node.(rawNode); ok {
return node
}
return nodeToBytes(n.node)
}
// obj returns the decoded and expanded trie node, either directly from the cache,
// or by regenerating it from the rlp encoded blob.
// nolint:unused
func (n *memoryNode) obj() node {
if node, ok := n.node.(rawNode); ok {
return mustDecodeNode(n.hash[:], node)
}
return expandNode(n.hash[:], n.node)
}
// isDeleted returns the indicator if the node is marked as deleted.
func (n *memoryNode) isDeleted() bool {
return n.hash == (common.Hash{})
}
// nodeWithPrev wraps the memoryNode with the previous node value.
// nolint: unused
type nodeWithPrev struct {
*memoryNode
prev []byte // RLP-encoded previous value, nil means it's non-existent
}
// unwrap returns the internal memoryNode object.
// nolint:unused
func (n *nodeWithPrev) unwrap() *memoryNode {
return n.memoryNode
}
// memorySize returns the total memory size used by this node. It overloads
// the function in memoryNode by counting the size of previous value as well.
// nolint: unused
func (n *nodeWithPrev) memorySize(pathlen int) int {
return n.memoryNode.memorySize(pathlen) + len(n.prev)
}
// NodeSet contains all dirty nodes collected during the commit operation.
// Each node is keyed by path. It's not thread-safe to use.
type NodeSet struct {
owner common.Hash // the identifier of the trie
nodes map[string]*memoryNode // the set of dirty nodes(inserted, updated, deleted)
leaves []*leaf // the list of dirty leaves
updates int // the count of updated and inserted nodes
deletes int // the count of deleted nodes
// The list of accessed nodes, which records the original node value.
// The origin value is expected to be nil for newly inserted node
// and is expected to be non-nil for other types(updated, deleted).
accessList map[string][]byte
}
// NewNodeSet initializes an empty node set to be used for tracking dirty nodes
// from a specific account or storage trie. The owner is zero for the account
// trie and the owning account address hash for storage tries.
func NewNodeSet(owner common.Hash, accessList map[string][]byte) *NodeSet {
return &NodeSet{
owner: owner,
nodes: make(map[string]*memoryNode),
accessList: accessList,
}
}
// forEachWithOrder iterates the dirty nodes with the order from bottom to top,
// right to left, nodes with the longest path will be iterated first.
func (set *NodeSet) forEachWithOrder(callback func(path string, n *memoryNode)) {
var paths sort.StringSlice
for path := range set.nodes {
paths = append(paths, path)
}
// Bottom-up, longest path first
sort.Sort(sort.Reverse(paths))
for _, path := range paths {
callback(path, set.nodes[path])
}
}
// markUpdated marks the node as dirty(newly-inserted or updated).
func (set *NodeSet) markUpdated(path []byte, node *memoryNode) {
set.nodes[string(path)] = node
set.updates += 1
}
// markDeleted marks the node as deleted.
func (set *NodeSet) markDeleted(path []byte) {
set.nodes[string(path)] = &memoryNode{}
set.deletes += 1
}
// addLeaf collects the provided leaf node into set.
func (set *NodeSet) addLeaf(node *leaf) {
set.leaves = append(set.leaves, node)
}
// Size returns the number of dirty nodes in set.
func (set *NodeSet) Size() (int, int) {
return set.updates, set.deletes
}
// Hashes returns the hashes of all updated nodes. TODO(rjl493456442) how can
// we get rid of it?
func (set *NodeSet) Hashes() []common.Hash {
var ret []common.Hash
for _, node := range set.nodes {
ret = append(ret, node.hash)
}
return ret
}
// Summary returns a string-representation of the NodeSet.
func (set *NodeSet) Summary() string {
var out = new(strings.Builder)
fmt.Fprintf(out, "nodeset owner: %v\n", set.owner)
if set.nodes != nil {
for path, n := range set.nodes {
// Deletion
if n.isDeleted() {
fmt.Fprintf(out, " [-]: %x prev: %x\n", path, set.accessList[path])
continue
}
// Insertion
origin, ok := set.accessList[path]
if !ok {
fmt.Fprintf(out, " [+]: %x -> %v\n", path, n.hash)
continue
}
// Update
fmt.Fprintf(out, " [*]: %x -> %v prev: %x\n", path, n.hash, origin)
}
}
for _, n := range set.leaves {
fmt.Fprintf(out, "[leaf]: %v\n", n)
}
return out.String()
}
// MergedNodeSet represents a merged dirty node set for a group of tries.
type MergedNodeSet struct {
sets map[common.Hash]*NodeSet
}
// NewMergedNodeSet initializes an empty merged set.
func NewMergedNodeSet() *MergedNodeSet {
return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)}
}
// NewWithNodeSet constructs a merged nodeset with the provided single set.
func NewWithNodeSet(set *NodeSet) *MergedNodeSet {
merged := NewMergedNodeSet()
merged.Merge(set)
return merged
}
// Merge merges the provided dirty nodes of a trie into the set. The assumption
// is held that no duplicated set belonging to the same trie will be merged twice.
func (set *MergedNodeSet) Merge(other *NodeSet) error {
_, present := set.sets[other.owner]
if present {
return fmt.Errorf("duplicate trie for owner %#x", other.owner)
}
set.sets[other.owner] = other
return nil
}
trie/proof.go
+5 -2
View File
@@ -64,12 +64,15 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e
// loaded blob will be tracked, while it's not required here since
// all loaded nodes won't be linked to trie at all and track nodes
// may lead to out-of-memory issue.
var err error
tn, err = t.reader.node(prefix, common.BytesToHash(n))
blob, err := t.reader.node(prefix, common.BytesToHash(n))
if err != nil {
log.Error("Unhandled trie error in Trie.Prove", "err", err)
return err
}
// The raw-blob format nodes are loaded either from the
// clean cache or the database, they are all in their own
// copy and safe to use unsafe decoder.
tn = mustDecodeNodeUnsafe(n, blob)
default:
panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
}
trie/secure_trie.go
+2 -1
View File
@@ -20,6 +20,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// SecureTrie is the old name of StateTrie.
@@ -212,7 +213,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte {
// All cached preimages will be also flushed if preimages recording is enabled.
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet) {
func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
// Write all the pre-images to the actual disk database
if len(t.getSecKeyCache()) > 0 {
if t.preimages != nil {
trie/secure_trie_test.go
+5 -3
View File
@@ -25,11 +25,13 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/trie/trienode"
)
func newEmptySecure() *StateTrie {
trie, _ := NewStateTrie(TrieID(common.Hash{}), NewDatabase(rawdb.NewMemoryDatabase()))
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase()))
return trie
}
@@ -37,7 +39,7 @@ func newEmptySecure() *StateTrie {
func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb)
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data
content := make(map[string][]byte)
@@ -59,7 +61,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) {
}
}
root, nodes := trie.Commit(false)
if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil {
panic(fmt.Errorf("failed to commit db %v", err))
}
// Re-create the trie based on the new state
trie/stacktrie.go
+6 -6
View File
@@ -420,17 +420,17 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) {
return
case branchNode:
var nodes rawFullNode
var nodes fullNode
for i, child := range st.children {
if child == nil {
nodes[i] = nilValueNode
nodes.Children[i] = nilValueNode
continue
}
child.hashRec(hasher, append(path, byte(i)))
if len(child.val) < 32 {
nodes[i] = rawNode(child.val)
nodes.Children[i] = rawNode(child.val)
} else {
nodes[i] = hashNode(child.val)
nodes.Children[i] = hashNode(child.val)
}
// Release child back to pool.
@@ -444,7 +444,7 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) {
case extNode:
st.children[0].hashRec(hasher, append(path, st.key...))
n := rawShortNode{Key: hexToCompact(st.key)}
n := shortNode{Key: hexToCompact(st.key)}
if len(st.children[0].val) < 32 {
n.Val = rawNode(st.children[0].val)
} else {
@@ -460,7 +460,7 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) {
case leafNode:
st.key = append(st.key, byte(16))
n := rawShortNode{Key: hexToCompact(st.key), Val: valueNode(st.val)}
n := shortNode{Key: hexToCompact(st.key), Val: valueNode(st.val)}
n.encode(hasher.encbuf)
encodedNode = hasher.encodedBytes()
trie/sync_test.go
+226 -65
View File
@@ -25,14 +25,17 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// makeTestTrie create a sample test trie to test node-wise reconstruction.
func makeTestTrie() (*Database, *StateTrie, map[string][]byte) {
func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb)
db := rawdb.NewMemoryDatabase()
triedb := newTestDatabase(db, scheme)
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
// Fill it with some arbitrary data
content := make(map[string][]byte)
@@ -54,23 +57,27 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) {
}
}
root, nodes := trie.Commit(false)
if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil {
panic(fmt.Errorf("failed to commit db %v", err))
}
if err := triedb.Commit(root, false); err != nil {
panic(err)
}
// Re-create the trie based on the new state
trie, _ = NewStateTrie(TrieID(root), triedb)
return triedb, trie, content
return db, triedb, trie, content
}
// checkTrieContents cross references a reconstructed trie with an expected data
// content map.
func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) {
func checkTrieContents(t *testing.T, db ethdb.Database, scheme string, root []byte, content map[string][]byte) {
// Check root availability and trie contents
trie, err := NewStateTrie(TrieID(common.BytesToHash(root)), db)
ndb := newTestDatabase(db, scheme)
trie, err := NewStateTrie(TrieID(common.BytesToHash(root)), ndb)
if err != nil {
t.Fatalf("failed to create trie at %x: %v", root, err)
}
if err := checkTrieConsistency(db, common.BytesToHash(root)); err != nil {
if err := checkTrieConsistency(db, scheme, common.BytesToHash(root)); err != nil {
t.Fatalf("inconsistent trie at %x: %v", root, err)
}
for key, val := range content {
@@ -81,9 +88,9 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri
}
// checkTrieConsistency checks that all nodes in a trie are indeed present.
func checkTrieConsistency(db *Database, root common.Hash) error {
// Create and iterate a trie rooted in a subnode
trie, err := NewStateTrie(TrieID(root), db)
func checkTrieConsistency(db ethdb.Database, scheme string, root common.Hash) error {
ndb := newTestDatabase(db, scheme)
trie, err := NewStateTrie(TrieID(root), ndb)
if err != nil {
return nil // Consider a non existent state consistent
}
@@ -104,11 +111,16 @@ type trieElement struct {
func TestEmptySync(t *testing.T) {
dbA := NewDatabase(rawdb.NewMemoryDatabase())
dbB := NewDatabase(rawdb.NewMemoryDatabase())
emptyA, _ := New(TrieID(common.Hash{}), dbA)
emptyB, _ := New(TrieID(types.EmptyRootHash), dbB)
//dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)
//dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)
for i, trie := range []*Trie{emptyA, emptyB} {
sync := NewSync(trie.Hash(), memorydb.New(), nil, []*Database{dbA, dbB}[i].Scheme())
emptyA := NewEmpty(dbA)
emptyB, _ := New(TrieID(types.EmptyRootHash), dbB)
//emptyC := NewEmpty(dbC)
//emptyD, _ := New(TrieID(types.EmptyRootHash), dbD)
for i, trie := range []*Trie{emptyA, emptyB /*emptyC, emptyD*/} {
sync := NewSync(trie.Hash(), memorydb.New(), nil, []*Database{dbA, dbB /*dbC, dbD*/}[i].Scheme())
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes)
}
@@ -117,18 +129,23 @@ func TestEmptySync(t *testing.T) {
// Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go.
func TestIterativeSyncIndividual(t *testing.T) { testIterativeSync(t, 1, false) }
func TestIterativeSyncBatched(t *testing.T) { testIterativeSync(t, 100, false) }
func TestIterativeSyncIndividualByPath(t *testing.T) { testIterativeSync(t, 1, true) }
func TestIterativeSyncBatchedByPath(t *testing.T) { testIterativeSync(t, 100, true) }
func TestIterativeSync(t *testing.T) {
testIterativeSync(t, 1, false, rawdb.HashScheme)
testIterativeSync(t, 100, false, rawdb.HashScheme)
testIterativeSync(t, 1, true, rawdb.HashScheme)
testIterativeSync(t, 100, true, rawdb.HashScheme)
// testIterativeSync(t, 1, false, rawdb.PathScheme)
// testIterativeSync(t, 100, false, rawdb.PathScheme)
// testIterativeSync(t, 1, true, rawdb.PathScheme)
// testIterativeSync(t, 100, true, rawdb.PathScheme)
}
func testIterativeSync(t *testing.T, count int, bypath bool) {
func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) {
// Create a random trie to copy
srcDb, srcTrie, srcData := makeTestTrie()
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
@@ -146,7 +163,8 @@ func testIterativeSync(t *testing.T, count int, bypath bool) {
results := make([]NodeSyncResult, len(elements))
if !bypath {
for i, element := range elements {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err)
}
@@ -183,18 +201,22 @@ func testIterativeSync(t *testing.T, count int, bypath bool) {
}
}
// Cross check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned, and the others sent only later.
func TestIterativeDelayedSync(t *testing.T) {
testIterativeDelayedSync(t, rawdb.HashScheme)
//testIterativeDelayedSync(t, rawdb.PathScheme)
}
func testIterativeDelayedSync(t *testing.T, scheme string) {
// Create a random trie to copy
srcDb, srcTrie, srcData := makeTestTrie()
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
@@ -212,7 +234,8 @@ func TestIterativeDelayedSync(t *testing.T) {
// Sync only half of the scheduled nodes
results := make([]NodeSyncResult, len(elements)/2+1)
for i, element := range elements[:len(results)] {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
@@ -240,22 +263,25 @@ func TestIterativeDelayedSync(t *testing.T) {
}
}
// Cross check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
}
// Tests that given a root hash, a trie can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go, however in a
// random order.
func TestIterativeRandomSyncIndividual(t *testing.T) { testIterativeRandomSync(t, 1) }
func TestIterativeRandomSyncBatched(t *testing.T) { testIterativeRandomSync(t, 100) }
func TestIterativeRandomSyncIndividual(t *testing.T) {
testIterativeRandomSync(t, 1, rawdb.HashScheme)
testIterativeRandomSync(t, 100, rawdb.HashScheme)
// testIterativeRandomSync(t, 1, rawdb.PathScheme)
// testIterativeRandomSync(t, 100, rawdb.PathScheme)
}
func testIterativeRandomSync(t *testing.T, count int) {
func testIterativeRandomSync(t *testing.T, count int, scheme string) {
// Create a random trie to copy
srcDb, srcTrie, srcData := makeTestTrie()
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
@@ -273,7 +299,8 @@ func testIterativeRandomSync(t *testing.T, count int) {
// Fetch all the queued nodes in a random order
results := make([]NodeSyncResult, 0, len(queue))
for path, element := range queue {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
@@ -302,18 +329,22 @@ func testIterativeRandomSync(t *testing.T, count int) {
}
}
// Cross check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
}
// Tests that the trie scheduler can correctly reconstruct the state even if only
// partial results are returned (Even those randomly), others sent only later.
func TestIterativeRandomDelayedSync(t *testing.T) {
testIterativeRandomDelayedSync(t, rawdb.HashScheme)
// testIterativeRandomDelayedSync(t, rawdb.PathScheme)
}
func testIterativeRandomDelayedSync(t *testing.T, scheme string) {
// Create a random trie to copy
srcDb, srcTrie, srcData := makeTestTrie()
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
@@ -331,7 +362,8 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
// Sync only half of the scheduled nodes, even those in random order
results := make([]NodeSyncResult, 0, len(queue)/2+1)
for path, element := range queue {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
@@ -365,18 +397,22 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
}
}
// Cross check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
}
// Tests that a trie sync will not request nodes multiple times, even if they
// have such references.
func TestDuplicateAvoidanceSync(t *testing.T) {
testDuplicateAvoidanceSync(t, rawdb.HashScheme)
// testDuplicateAvoidanceSync(t, rawdb.PathScheme)
}
func testDuplicateAvoidanceSync(t *testing.T, scheme string) {
// Create a random trie to copy
srcDb, srcTrie, srcData := makeTestTrie()
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
@@ -395,7 +431,8 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
for len(elements) > 0 {
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
@@ -428,27 +465,33 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
}
}
// Cross check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
}
// Tests that at any point in time during a sync, only complete sub-tries are in
// the database.
func TestIncompleteSync(t *testing.T) {
func TestIncompleteSyncHash(t *testing.T) {
testIncompleteSync(t, rawdb.HashScheme)
// testIncompleteSync(t, rawdb.PathScheme)
}
func testIncompleteSync(t *testing.T, scheme string) {
t.Parallel()
// Create a random trie to copy
srcDb, srcTrie, _ := makeTestTrie()
_, srcDb, srcTrie, _ := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
// at the testing trie.
var (
added []common.Hash
elements []trieElement
root = srcTrie.Hash()
addedKeys []string
addedHashes []common.Hash
elements []trieElement
root = srcTrie.Hash()
)
paths, nodes, _ := sched.Missing(1)
for i := 0; i < len(paths); i++ {
@@ -462,7 +505,8 @@ func TestIncompleteSync(t *testing.T) {
// Fetch a batch of trie nodes
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
@@ -483,11 +527,8 @@ func TestIncompleteSync(t *testing.T) {
for _, result := range results {
hash := crypto.Keccak256Hash(result.Data)
if hash != root {
added = append(added, hash)
}
// Check that all known sub-tries in the synced trie are complete
if err := checkTrieConsistency(triedb, hash); err != nil {
t.Fatalf("trie inconsistent: %v", err)
addedKeys = append(addedKeys, result.Path)
addedHashes = append(addedHashes, crypto.Keccak256Hash(result.Data))
}
}
// Fetch the next batch to retrieve
@@ -502,25 +543,31 @@ func TestIncompleteSync(t *testing.T) {
}
}
// Sanity check that removing any node from the database is detected
for _, hash := range added {
value, _ := diskdb.Get(hash.Bytes())
diskdb.Delete(hash.Bytes())
if err := checkTrieConsistency(triedb, root); err == nil {
t.Fatalf("trie inconsistency not caught, missing: %x", hash)
for i, path := range addedKeys {
owner, inner := ResolvePath([]byte(path))
nodeHash := addedHashes[i]
value := rawdb.ReadTrieNode(diskdb, owner, inner, nodeHash, scheme)
rawdb.DeleteTrieNode(diskdb, owner, inner, nodeHash, scheme)
if err := checkTrieConsistency(diskdb, srcDb.Scheme(), root); err == nil {
t.Fatalf("trie inconsistency not caught, missing: %x", path)
}
diskdb.Put(hash.Bytes(), value)
rawdb.WriteTrieNode(diskdb, owner, inner, nodeHash, value, scheme)
}
}
// Tests that trie nodes get scheduled lexicographically when having the same
// depth.
func TestSyncOrdering(t *testing.T) {
testSyncOrdering(t, rawdb.HashScheme)
// testSyncOrdering(t, rawdb.PathScheme)
}
func testSyncOrdering(t *testing.T, scheme string) {
// Create a random trie to copy
srcDb, srcTrie, srcData := makeTestTrie()
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler, tracking the requests
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
@@ -542,7 +589,8 @@ func TestSyncOrdering(t *testing.T) {
for len(elements) > 0 {
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
data, err := srcDb.Node(element.hash)
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
@@ -571,7 +619,7 @@ func TestSyncOrdering(t *testing.T) {
}
}
// Cross check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
// Check that the trie nodes have been requested path-ordered
for i := 0; i < len(reqs)-1; i++ {
@@ -585,3 +633,116 @@ func TestSyncOrdering(t *testing.T) {
}
}
}
func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database) {
// Create a destination trie and sync with the scheduler
sched := NewSync(root, db, nil, srcDb.Scheme())
// The code requests are ignored here since there is no code
// at the testing trie.
paths, nodes, _ := sched.Missing(1)
var elements []trieElement
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
hash: nodes[i],
syncPath: NewSyncPath([]byte(paths[i])),
})
}
for len(elements) > 0 {
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
owner, inner := ResolvePath([]byte(element.path))
data, err := srcDb.Reader(root).Node(owner, inner, element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err)
}
results[i] = NodeSyncResult{element.path, data}
}
for index, result := range results {
if err := sched.ProcessNode(result); err != nil {
t.Fatalf("failed to process result[%d][%v] data %v %v", index, []byte(result.Path), result.Data, err)
}
}
batch := db.NewBatch()
if err := sched.Commit(batch); err != nil {
t.Fatalf("failed to commit data: %v", err)
}
batch.Write()
paths, nodes, _ = sched.Missing(1)
elements = elements[:0]
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
hash: nodes[i],
syncPath: NewSyncPath([]byte(paths[i])),
})
}
}
}
// Tests that the syncing target is keeping moving which may overwrite the stale
// states synced in the last cycle.
func TestSyncMovingTarget(t *testing.T) {
testSyncMovingTarget(t, rawdb.HashScheme)
// testSyncMovingTarget(t, rawdb.PathScheme)
}
func testSyncMovingTarget(t *testing.T, scheme string) {
// Create a random trie to copy
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
// Create a destination trie and sync with the scheduler
diskdb := rawdb.NewMemoryDatabase()
syncWith(t, srcTrie.Hash(), diskdb, srcDb)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
// Push more modifications into the src trie, to see if dest trie can still
// sync with it(overwrite stale states)
var (
preRoot = srcTrie.Hash()
diff = make(map[string][]byte)
)
for i := byte(0); i < 10; i++ {
key, val := randBytes(32), randBytes(32)
srcTrie.MustUpdate(key, val)
diff[string(key)] = val
}
root, nodes := srcTrie.Commit(false)
if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil {
panic(err)
}
if err := srcDb.Commit(root, false); err != nil {
panic(err)
}
preRoot = root
srcTrie, _ = NewStateTrie(TrieID(root), srcDb)
syncWith(t, srcTrie.Hash(), diskdb, srcDb)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), diff)
// Revert added modifications from the src trie, to see if dest trie can still
// sync with it(overwrite reverted states)
var reverted = make(map[string][]byte)
for k := range diff {
srcTrie.MustDelete([]byte(k))
reverted[k] = nil
}
for k := range srcData {
val := randBytes(32)
srcTrie.MustUpdate([]byte(k), val)
reverted[k] = val
}
root, nodes = srcTrie.Commit(false)
if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil {
panic(err)
}
if err := srcDb.Commit(root, false); err != nil {
panic(err)
}
srcTrie, _ = NewStateTrie(TrieID(root), srcDb)
syncWith(t, srcTrie.Hash(), diskdb, srcDb)
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), reverted)
}
trie/tracer.go
+8 -4
View File
@@ -16,7 +16,10 @@
package trie
import "github.com/ethereum/go-ethereum/common"
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// tracer tracks the changes of trie nodes. During the trie operations,
// some nodes can be deleted from the trie, while these deleted nodes
@@ -112,14 +115,15 @@ func (t *tracer) copy() *tracer {
}
// markDeletions puts all tracked deletions into the provided nodeset.
func (t *tracer) markDeletions(set *NodeSet) {
func (t *tracer) markDeletions(set *trienode.NodeSet) {
for path := range t.deletes {
// It's possible a few deleted nodes were embedded
// in their parent before, the deletions can be no
// effect by deleting nothing, filter them out.
if _, ok := set.accessList[path]; !ok {
prev, ok := t.accessList[path]
if !ok {
continue
}
set.markDeleted([]byte(path))
set.AddNode([]byte(path), trienode.NewWithPrev(common.Hash{}, nil, prev))
}
}
trie/tracer_test.go
+16 -9
View File
@@ -22,6 +22,8 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/trie/trienode"
)
var (
@@ -69,7 +71,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
insertSet := copySet(trie.tracer.inserts) // copy before commit
deleteSet := copySet(trie.tracer.deletes) // copy before commit
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
seen := setKeys(iterNodes(db, root))
if !compareSet(insertSet, seen) {
@@ -135,7 +137,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
if err := verifyAccessList(orig, trie, nodes); err != nil {
@@ -143,13 +145,14 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
}
// Update trie
parent := root
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
for _, val := range vals {
trie.MustUpdate([]byte(val.k), randBytes(32))
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, parent, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
if err := verifyAccessList(orig, trie, nodes); err != nil {
@@ -157,6 +160,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
}
// Add more new nodes
parent = root
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
var keys []string
@@ -166,7 +170,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
trie.MustUpdate(key, randBytes(32))
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, parent, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
if err := verifyAccessList(orig, trie, nodes); err != nil {
@@ -174,13 +178,14 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
}
// Partial deletions
parent = root
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
for _, key := range keys {
trie.MustUpdate([]byte(key), nil)
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, parent, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
if err := verifyAccessList(orig, trie, nodes); err != nil {
@@ -188,13 +193,14 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
}
// Delete all
parent = root
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
for _, val := range vals {
trie.MustUpdate([]byte(val.k), nil)
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, parent, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
if err := verifyAccessList(orig, trie, nodes); err != nil {
@@ -213,7 +219,7 @@ func TestAccessListLeak(t *testing.T) {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
var cases = []struct {
op func(tr *Trie)
@@ -263,15 +269,16 @@ func TestTinyTree(t *testing.T) {
trie.MustUpdate([]byte(val.k), randBytes(32))
}
root, set := trie.Commit(false)
db.Update(NewWithNodeSet(set))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set))
parent := root
trie, _ = New(TrieID(root), db)
orig := trie.Copy()
for _, val := range tiny {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, set = trie.Commit(false)
db.Update(NewWithNodeSet(set))
db.Update(root, parent, trienode.NewWithNodeSet(set))
trie, _ = New(TrieID(root), db)
if err := verifyAccessList(orig, trie, set); err != nil {
trie/trie.go
+7 -6
View File
@@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on
@@ -95,7 +96,7 @@ func New(id *ID, db NodeReader) (*Trie, error) {
// NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
func NewEmpty(db *Database) *Trie {
tr, _ := New(TrieID(common.Hash{}), db)
tr, _ := New(TrieID(types.EmptyRootHash), db)
return tr
}
@@ -212,7 +213,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
if hash == nil {
return nil, origNode, 0, errors.New("non-consensus node")
}
blob, err := t.reader.nodeBlob(path, common.BytesToHash(hash))
blob, err := t.reader.node(path, common.BytesToHash(hash))
return blob, origNode, 1, err
}
// Path still needs to be traversed, descend into children
@@ -549,7 +550,7 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
// node's original value. The rlp-encoded blob is preferred to be loaded from
// database because it's easy to decode node while complex to encode node to blob.
func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) {
blob, err := t.reader.nodeBlob(prefix, common.BytesToHash(n))
blob, err := t.reader.node(prefix, common.BytesToHash(n))
if err != nil {
return nil, err
}
@@ -571,10 +572,10 @@ func (t *Trie) Hash() common.Hash {
// The returned nodeset can be nil if the trie is clean (nothing to commit).
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet) {
func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
defer t.tracer.reset()
nodes := NewNodeSet(t.owner, t.tracer.accessList)
nodes := trienode.NewNodeSet(t.owner)
t.tracer.markDeletions(nodes)
// Trie is empty and can be classified into two types of situations:
@@ -595,7 +596,7 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet) {
t.root = hashedNode
return rootHash, nil
}
t.root = newCommitter(nodes, collectLeaf).Commit(t.root)
t.root = newCommitter(nodes, t.tracer, collectLeaf).Commit(t.root)
return rootHash, nodes
}
trie/trie_reader.go
+10 -35
View File
@@ -22,24 +22,19 @@ import (
"github.com/ethereum/go-ethereum/common"
)
// Reader wraps the Node and NodeBlob method of a backing trie store.
// Reader wraps the Node method of a backing trie store.
type Reader interface {
// Node retrieves the trie node with the provided trie identifier, hexary
// node path and the corresponding node hash.
// No error will be returned if the node is not found.
Node(owner common.Hash, path []byte, hash common.Hash) (node, error)
// NodeBlob retrieves the RLP-encoded trie node blob with the provided trie
// identifier, hexary node path and the corresponding node hash.
// No error will be returned if the node is not found.
NodeBlob(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)
// Node retrieves the RLP-encoded trie node blob with the provided trie
// identifier, node path and the corresponding node hash. No error will
// be returned if the node is not found.
Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)
}
// NodeReader wraps all the necessary functions for accessing trie node.
type NodeReader interface {
// GetReader returns a reader for accessing all trie nodes with provided
// Reader returns a reader for accessing all trie nodes with provided
// state root. Nil is returned in case the state is not available.
GetReader(root common.Hash) Reader
Reader(root common.Hash) Reader
}
// trieReader is a wrapper of the underlying node reader. It's not safe
@@ -52,7 +47,7 @@ type trieReader struct {
// newTrieReader initializes the trie reader with the given node reader.
func newTrieReader(stateRoot, owner common.Hash, db NodeReader) (*trieReader, error) {
reader := db.GetReader(stateRoot)
reader := db.Reader(stateRoot)
if reader == nil {
return nil, fmt.Errorf("state not found #%x", stateRoot)
}
@@ -65,30 +60,10 @@ func newEmptyReader() *trieReader {
return &trieReader{}
}
// node retrieves the trie node with the provided trie node information.
// An MissingNodeError will be returned in case the node is not found or
// any error is encountered.
func (r *trieReader) node(path []byte, hash common.Hash) (node, error) {
// Perform the logics in tests for preventing trie node access.
if r.banned != nil {
if _, ok := r.banned[string(path)]; ok {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
}
if r.reader == nil {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
node, err := r.reader.Node(r.owner, path, hash)
if err != nil || node == nil {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err}
}
return node, nil
}
// node retrieves the rlp-encoded trie node with the provided trie node
// information. An MissingNodeError will be returned in case the node is
// not found or any error is encountered.
func (r *trieReader) nodeBlob(path []byte, hash common.Hash) ([]byte, error) {
func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) {
// Perform the logics in tests for preventing trie node access.
if r.banned != nil {
if _, ok := r.banned[string(path)]; ok {
@@ -98,7 +73,7 @@ func (r *trieReader) nodeBlob(path []byte, hash common.Hash) ([]byte, error) {
if r.reader == nil {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
blob, err := r.reader.NodeBlob(r.owner, path, hash)
blob, err := r.reader.Node(r.owner, path, hash)
if err != nil || len(blob) == 0 {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err}
}
trie/trie_test.go
+64 -50
View File
@@ -35,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/trienode"
"golang.org/x/crypto/sha3"
)
@@ -73,18 +74,23 @@ func TestMissingRoot(t *testing.T) {
}
}
func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) }
func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) }
func TestMissingNode(t *testing.T) {
testMissingNode(t, false, rawdb.HashScheme)
//testMissingNode(t, false, rawdb.PathScheme)
testMissingNode(t, true, rawdb.HashScheme)
//testMissingNode(t, true, rawdb.PathScheme)
}
func testMissingNode(t *testing.T, memonly bool) {
func testMissingNode(t *testing.T, memonly bool, scheme string) {
diskdb := rawdb.NewMemoryDatabase()
triedb := NewDatabase(diskdb)
triedb := newTestDatabase(diskdb, scheme)
trie := NewEmpty(triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(root, false)
}
@@ -115,34 +121,39 @@ func testMissingNode(t *testing.T, memonly bool) {
t.Errorf("Unexpected error: %v", err)
}
hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
var (
path []byte
hash = common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
)
for p, n := range nodes.Nodes {
if n.Hash == hash {
path = common.CopyBytes([]byte(p))
break
}
}
trie, _ = New(TrieID(root), triedb)
if memonly {
delete(triedb.dirties, hash)
trie.reader.banned = map[string]struct{}{string(path): {}}
} else {
diskdb.Delete(hash[:])
rawdb.DeleteTrieNode(diskdb, common.Hash{}, path, hash, scheme)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("120000"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
err = trie.Update([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
err = trie.Delete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
@@ -192,7 +203,7 @@ func TestGet(t *testing.T) {
return
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
}
}
@@ -249,8 +260,8 @@ func TestEmptyValues(t *testing.T) {
}
func TestReplication(t *testing.T) {
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(triedb)
db := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(db)
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@@ -263,13 +274,13 @@ func TestReplication(t *testing.T) {
for _, val := range vals {
updateString(trie, val.k, val.v)
}
exp, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
root, nodes := trie.Commit(false)
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
// create a new trie on top of the database and check that lookups work.
trie2, err := New(TrieID(exp), triedb)
trie2, err := New(TrieID(root), db)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
t.Fatalf("can't recreate trie at %x: %v", root, err)
}
for _, kv := range vals {
if string(getString(trie2, kv.k)) != kv.v {
@@ -277,17 +288,17 @@ func TestReplication(t *testing.T) {
}
}
hash, nodes := trie2.Commit(false)
if hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
if hash != root {
t.Errorf("root failure. expected %x got %x", root, hash)
}
// recreate the trie after commit
if nodes != nil {
triedb.Update(NewWithNodeSet(nodes))
db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
}
trie2, err = New(TrieID(hash), triedb)
trie2, err = New(TrieID(hash), db)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
t.Fatalf("can't recreate trie at %x: %v", hash, err)
}
// perform some insertions on the new trie.
vals2 := []struct{ k, v string }{
@@ -304,8 +315,8 @@ func TestReplication(t *testing.T) {
for _, val := range vals2 {
updateString(trie2, val.k, val.v)
}
if hash := trie2.Hash(); hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
if trie2.Hash() != hash {
t.Errorf("root failure. expected %x got %x", hash, hash)
}
}
@@ -402,45 +413,42 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
return reflect.ValueOf(steps)
}
func verifyAccessList(old *Trie, new *Trie, set *NodeSet) error {
func verifyAccessList(old *Trie, new *Trie, set *trienode.NodeSet) error {
deletes, inserts, updates := diffTries(old, new)
// Check insertion set
for path := range inserts {
n, ok := set.nodes[path]
if !ok || n.isDeleted() {
n, ok := set.Nodes[path]
if !ok || n.IsDeleted() {
return errors.New("expect new node")
}
_, ok = set.accessList[path]
if ok {
if len(n.Prev) > 0 {
return errors.New("unexpected origin value")
}
}
// Check deletion set
for path, blob := range deletes {
n, ok := set.nodes[path]
if !ok || !n.isDeleted() {
n, ok := set.Nodes[path]
if !ok || !n.IsDeleted() {
return errors.New("expect deleted node")
}
v, ok := set.accessList[path]
if !ok {
if len(n.Prev) == 0 {
return errors.New("expect origin value")
}
if !bytes.Equal(v, blob) {
if !bytes.Equal(n.Prev, blob) {
return errors.New("invalid origin value")
}
}
// Check update set
for path, blob := range updates {
n, ok := set.nodes[path]
if !ok || n.isDeleted() {
n, ok := set.Nodes[path]
if !ok || n.IsDeleted() {
return errors.New("expect updated node")
}
v, ok := set.accessList[path]
if !ok {
if len(n.Prev) == 0 {
return errors.New("expect origin value")
}
if !bytes.Equal(v, blob) {
if !bytes.Equal(n.Prev, blob) {
return errors.New("invalid origin value")
}
}
@@ -448,8 +456,13 @@ func verifyAccessList(old *Trie, new *Trie, set *NodeSet) error {
}
func runRandTest(rt randTest) bool {
var scheme = rawdb.HashScheme
//if rand.Intn(2) == 0 {
// scheme = rawdb.PathScheme
//}
var (
triedb = NewDatabase(rawdb.NewMemoryDatabase())
origin = types.EmptyRootHash
triedb = newTestDatabase(rawdb.NewMemoryDatabase(), scheme)
tr = NewEmpty(triedb)
values = make(map[string]string) // tracks content of the trie
origTrie = NewEmpty(triedb)
@@ -490,7 +503,7 @@ func runRandTest(rt randTest) bool {
case opCommit:
root, nodes := tr.Commit(true)
if nodes != nil {
triedb.Update(NewWithNodeSet(nodes))
triedb.Update(root, origin, trienode.NewWithNodeSet(nodes))
}
newtr, err := New(TrieID(root), triedb)
if err != nil {
@@ -505,6 +518,7 @@ func runRandTest(rt randTest) bool {
}
tr = newtr
origTrie = tr.Copy()
origin = root
case opItercheckhash:
checktr := NewEmpty(triedb)
it := NewIterator(tr.NodeIterator(nil))
@@ -824,7 +838,7 @@ func TestCommitSequence(t *testing.T) {
}
// Flush trie -> database
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
// Flush memdb -> disk (sponge)
db.Commit(root, false)
if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) {
@@ -865,7 +879,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) {
}
// Flush trie -> database
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
// Flush memdb -> disk (sponge)
db.Commit(root, false)
if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) {
@@ -905,7 +919,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
// Flush trie -> database
root, nodes := trie.Commit(false)
// Flush memdb -> disk (sponge)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
db.Commit(root, false)
// And flush stacktrie -> disk
stRoot, err := stTrie.Commit()
@@ -953,7 +967,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) {
// Flush trie -> database
root, nodes := trie.Commit(false)
// Flush memdb -> disk (sponge)
db.Update(NewWithNodeSet(nodes))
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
db.Commit(root, false)
// And flush stacktrie -> disk
stRoot, err := stTrie.Commit()
@@ -1124,8 +1138,8 @@ func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts []
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
h := trie.Hash()
_, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
root, nodes := trie.Commit(false)
triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
b.StartTimer()
triedb.Dereference(h)
b.StopTimer()
trie/database.go → trie/triedb/hashdb/database.go
+133 -415
View File
@@ -14,14 +14,11 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
package hashdb
import (
"errors"
"fmt"
"io"
"reflect"
"runtime"
"sync"
"time"
@@ -33,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/trienode"
)
var (
@@ -59,6 +57,12 @@ var (
memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil)
)
// ChildResolver defines the required method to decode the provided
// trie node and iterate the children on top.
type ChildResolver interface {
ForEach(node []byte, onChild func(common.Hash))
}
// Database is an intermediate write layer between the trie data structures and
// the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder.
@@ -68,7 +72,8 @@ var (
// behind this split design is to provide read access to RPC handlers and sync
// servers even while the trie is executing expensive garbage collection.
type Database struct {
diskdb ethdb.Database // Persistent storage for matured trie nodes
diskdb ethdb.Database // Persistent storage for matured trie nodes
resolver ChildResolver // The handler to resolve children of nodes
cleans *fastcache.Cache // GC friendly memory cache of clean node RLPs
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty trie nodes
@@ -85,60 +90,18 @@ type Database struct {
dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata)
childrenSize common.StorageSize // Storage size of the external children tracking
preimages *preimageStore // The store for caching preimages
lock sync.RWMutex
}
// rawNode is a simple binary blob used to differentiate between collapsed trie
// nodes and already encoded RLP binary blobs (while at the same time store them
// in the same cache fields).
type rawNode []byte
func (n rawNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
func (n rawNode) fstring(ind string) string { panic("this should never end up in a live trie") }
func (n rawNode) EncodeRLP(w io.Writer) error {
_, err := w.Write(n)
return err
}
// rawFullNode represents only the useful data content of a full node, with the
// caches and flags stripped out to minimize its data storage. This type honors
// the same RLP encoding as the original parent.
type rawFullNode [17]node
func (n rawFullNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
func (n rawFullNode) fstring(ind string) string { panic("this should never end up in a live trie") }
func (n rawFullNode) EncodeRLP(w io.Writer) error {
eb := rlp.NewEncoderBuffer(w)
n.encode(eb)
return eb.Flush()
}
// rawShortNode represents only the useful data content of a short node, with the
// caches and flags stripped out to minimize its data storage. This type honors
// the same RLP encoding as the original parent.
type rawShortNode struct {
Key []byte
Val node
}
func (n rawShortNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
func (n rawShortNode) fstring(ind string) string { panic("this should never end up in a live trie") }
// cachedNode is all the information we know about a single cached trie node
// in the memory database write layer.
type cachedNode struct {
node node // Cached collapsed trie node, or raw rlp data
size uint16 // Byte size of the useful cached data
parents uint32 // Number of live nodes referencing this one
children map[common.Hash]uint16 // External children referenced by this node
flushPrev common.Hash // Previous node in the flush-list
flushNext common.Hash // Next node in the flush-list
node []byte // Encoded node blob
parents uint32 // Number of live nodes referencing this one
external map[common.Hash]struct{} // The set of external children
flushPrev common.Hash // Previous node in the flush-list
flushNext common.Hash // Next node in the flush-list
}
// cachedNodeSize is the raw size of a cachedNode data structure without any
@@ -146,181 +109,42 @@ type cachedNode struct {
// than not counting them.
var cachedNodeSize = int(reflect.TypeOf(cachedNode{}).Size())
// cachedNodeChildrenSize is the raw size of an initialized but empty external
// reference map.
const cachedNodeChildrenSize = 48
// rlp returns the raw rlp encoded blob of the cached trie node, either directly
// from the cache, or by regenerating it from the collapsed node.
func (n *cachedNode) rlp() []byte {
if node, ok := n.node.(rawNode); ok {
return node
}
return nodeToBytes(n.node)
}
// obj returns the decoded and expanded trie node, either directly from the cache,
// or by regenerating it from the rlp encoded blob.
func (n *cachedNode) obj(hash common.Hash) node {
if node, ok := n.node.(rawNode); ok {
// The raw-blob format nodes are loaded either from the
// clean cache or the database, they are all in their own
// copy and safe to use unsafe decoder.
return mustDecodeNodeUnsafe(hash[:], node)
}
return expandNode(hash[:], n.node)
}
// forChilds invokes the callback for all the tracked children of this node,
// forChildren invokes the callback for all the tracked children of this node,
// both the implicit ones from inside the node as well as the explicit ones
// from outside the node.
func (n *cachedNode) forChilds(onChild func(hash common.Hash)) {
for child := range n.children {
func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash common.Hash)) {
for child := range n.external {
onChild(child)
}
if _, ok := n.node.(rawNode); !ok {
forGatherChildren(n.node, onChild)
}
resolver.ForEach(n.node, onChild)
}
// forGatherChildren traverses the node hierarchy of a collapsed storage node and
// invokes the callback for all the hashnode children.
func forGatherChildren(n node, onChild func(hash common.Hash)) {
switch n := n.(type) {
case *rawShortNode:
forGatherChildren(n.Val, onChild)
case rawFullNode:
for i := 0; i < 16; i++ {
forGatherChildren(n[i], onChild)
}
case hashNode:
onChild(common.BytesToHash(n))
case valueNode, nil, rawNode:
default:
panic(fmt.Sprintf("unknown node type: %T", n))
// New initializes the hash-based node database.
func New(diskdb ethdb.Database, cleans *fastcache.Cache, resolver ChildResolver) *Database {
return &Database{
diskdb: diskdb,
resolver: resolver,
cleans: cleans,
dirties: make(map[common.Hash]*cachedNode),
}
}
// simplifyNode traverses the hierarchy of an expanded memory node and discards
// all the internal caches, returning a node that only contains the raw data.
func simplifyNode(n node) node {
switch n := n.(type) {
case *shortNode:
// Short nodes discard the flags and cascade
return &rawShortNode{Key: n.Key, Val: simplifyNode(n.Val)}
case *fullNode:
// Full nodes discard the flags and cascade
node := rawFullNode(n.Children)
for i := 0; i < len(node); i++ {
if node[i] != nil {
node[i] = simplifyNode(node[i])
}
}
return node
case valueNode, hashNode, rawNode:
return n
default:
panic(fmt.Sprintf("unknown node type: %T", n))
}
}
// expandNode traverses the node hierarchy of a collapsed storage node and converts
// all fields and keys into expanded memory form.
func expandNode(hash hashNode, n node) node {
switch n := n.(type) {
case *rawShortNode:
// Short nodes need key and child expansion
return &shortNode{
Key: compactToHex(n.Key),
Val: expandNode(nil, n.Val),
flags: nodeFlag{
hash: hash,
},
}
case rawFullNode:
// Full nodes need child expansion
node := &fullNode{
flags: nodeFlag{
hash: hash,
},
}
for i := 0; i < len(node.Children); i++ {
if n[i] != nil {
node.Children[i] = expandNode(nil, n[i])
}
}
return node
case valueNode, hashNode:
return n
default:
panic(fmt.Sprintf("unknown node type: %T", n))
}
}
// Config defines all necessary options for database.
type Config struct {
Cache int // Memory allowance (MB) to use for caching trie nodes in memory
Journal string // Journal of clean cache to survive node restarts
Preimages bool // Flag whether the preimage of trie key is recorded
}
// NewDatabase creates a new trie database to store ephemeral trie content before
// its written out to disk or garbage collected. No read cache is created, so all
// data retrievals will hit the underlying disk database.
func NewDatabase(diskdb ethdb.Database) *Database {
return NewDatabaseWithConfig(diskdb, nil)
}
// NewDatabaseWithConfig creates a new trie database to store ephemeral trie content
// before its written out to disk or garbage collected. It also acts as a read cache
// for nodes loaded from disk.
func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database {
var cleans *fastcache.Cache
if config != nil && config.Cache > 0 {
if config.Journal == "" {
cleans = fastcache.New(config.Cache * 1024 * 1024)
} else {
cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024)
}
}
var preimage *preimageStore
if config != nil && config.Preimages {
preimage = newPreimageStore(diskdb)
}
db := &Database{
diskdb: diskdb,
cleans: cleans,
dirties: map[common.Hash]*cachedNode{{}: {
children: make(map[common.Hash]uint16),
}},
preimages: preimage,
}
return db
}
// insert inserts a simplified trie node into the memory database.
// All nodes inserted by this function will be reference tracked
// and in theory should only used for **trie nodes** insertion.
func (db *Database) insert(hash common.Hash, size int, node node) {
func (db *Database) insert(hash common.Hash, node []byte) {
// If the node's already cached, skip
if _, ok := db.dirties[hash]; ok {
return
}
memcacheDirtyWriteMeter.Mark(int64(size))
memcacheDirtyWriteMeter.Mark(int64(len(node)))
// Create the cached entry for this node
entry := &cachedNode{
node: node,
size: uint16(size),
flushPrev: db.newest,
}
entry.forChilds(func(child common.Hash) {
entry.forChildren(db.resolver, func(child common.Hash) {
if c := db.dirties[child]; c != nil {
c.parents++
}
@@ -333,48 +157,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) {
} else {
db.dirties[db.newest].flushNext, db.newest = hash, hash
}
db.dirtiesSize += common.StorageSize(common.HashLength + entry.size)
}
// node retrieves a cached trie node from memory, or returns nil if none can be
// found in the memory cache.
func (db *Database) node(hash common.Hash) node {
// Retrieve the node from the clean cache if available
if db.cleans != nil {
if enc := db.cleans.Get(nil, hash[:]); enc != nil {
memcacheCleanHitMeter.Mark(1)
memcacheCleanReadMeter.Mark(int64(len(enc)))
// The returned value from cache is in its own copy,
// safe to use mustDecodeNodeUnsafe for decoding.
return mustDecodeNodeUnsafe(hash[:], enc)
}
}
// Retrieve the node from the dirty cache if available
db.lock.RLock()
dirty := db.dirties[hash]
db.lock.RUnlock()
if dirty != nil {
memcacheDirtyHitMeter.Mark(1)
memcacheDirtyReadMeter.Mark(int64(dirty.size))
return dirty.obj(hash)
}
memcacheDirtyMissMeter.Mark(1)
// Content unavailable in memory, attempt to retrieve from disk
enc, err := db.diskdb.Get(hash[:])
if err != nil || enc == nil {
return nil
}
if db.cleans != nil {
db.cleans.Set(hash[:], enc)
memcacheCleanMissMeter.Mark(1)
memcacheCleanWriteMeter.Mark(int64(len(enc)))
}
// The returned value from database is in its own copy,
// safe to use mustDecodeNodeUnsafe for decoding.
return mustDecodeNodeUnsafe(hash[:], enc)
db.dirtiesSize += common.StorageSize(common.HashLength + len(node))
}
// Node retrieves an encoded cached trie node from memory. If it cannot be found
@@ -399,8 +182,8 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
if dirty != nil {
memcacheDirtyHitMeter.Mark(1)
memcacheDirtyReadMeter.Mark(int64(dirty.size))
return dirty.rlp(), nil
memcacheDirtyReadMeter.Mark(int64(len(dirty.node)))
return dirty.node, nil
}
memcacheDirtyMissMeter.Mark(1)
@@ -426,9 +209,7 @@ func (db *Database) Nodes() []common.Hash {
var hashes = make([]common.Hash, 0, len(db.dirties))
for hash := range db.dirties {
if hash != (common.Hash{}) { // Special case for "root" references/nodes
hashes = append(hashes, hash)
}
hashes = append(hashes, hash)
}
return hashes
}
@@ -451,18 +232,22 @@ func (db *Database) reference(child common.Hash, parent common.Hash) {
if !ok {
return
}
// If the reference already exists, only duplicate for roots
if db.dirties[parent].children == nil {
db.dirties[parent].children = make(map[common.Hash]uint16)
db.childrenSize += cachedNodeChildrenSize
} else if _, ok = db.dirties[parent].children[child]; ok && parent != (common.Hash{}) {
// The reference is for state root, increase the reference counter.
if parent == (common.Hash{}) {
node.parents += 1
return
}
// The reference is for external storage trie, don't duplicate if
// the reference is already existent.
if db.dirties[parent].external == nil {
db.dirties[parent].external = make(map[common.Hash]struct{})
}
if _, ok := db.dirties[parent].external[child]; ok {
return
}
node.parents++
db.dirties[parent].children[child]++
if db.dirties[parent].children[child] == 1 {
db.childrenSize += common.HashLength + 2 // uint16 counter
}
db.dirties[parent].external[child] = struct{}{}
db.childrenSize += common.HashLength
}
// Dereference removes an existing reference from a root node.
@@ -476,7 +261,7 @@ func (db *Database) Dereference(root common.Hash) {
defer db.lock.Unlock()
nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now()
db.dereference(root, common.Hash{})
db.dereference(root)
db.gcnodes += uint64(nodes - len(db.dirties))
db.gcsize += storage - db.dirtiesSize
@@ -491,23 +276,13 @@ func (db *Database) Dereference(root common.Hash) {
}
// dereference is the private locked version of Dereference.
func (db *Database) dereference(child common.Hash, parent common.Hash) {
// Dereference the parent-child
node := db.dirties[parent]
if node.children != nil && node.children[child] > 0 {
node.children[child]--
if node.children[child] == 0 {
delete(node.children, child)
db.childrenSize -= (common.HashLength + 2) // uint16 counter
}
}
// If the child does not exist, it's a previously committed node.
node, ok := db.dirties[child]
func (db *Database) dereference(hash common.Hash) {
// If the node does not exist, it's a previously committed node.
node, ok := db.dirties[hash]
if !ok {
return
}
// If there are no more references to the child, delete it and cascade
// If there are no more references to the node, delete it and cascade
if node.parents > 0 {
// This is a special cornercase where a node loaded from disk (i.e. not in the
// memcache any more) gets reinjected as a new node (short node split into full,
@@ -517,25 +292,29 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) {
}
if node.parents == 0 {
// Remove the node from the flush-list
switch child {
switch hash {
case db.oldest:
db.oldest = node.flushNext
db.dirties[node.flushNext].flushPrev = common.Hash{}
if node.flushNext != (common.Hash{}) {
db.dirties[node.flushNext].flushPrev = common.Hash{}
}
case db.newest:
db.newest = node.flushPrev
db.dirties[node.flushPrev].flushNext = common.Hash{}
if node.flushPrev != (common.Hash{}) {
db.dirties[node.flushPrev].flushNext = common.Hash{}
}
default:
db.dirties[node.flushPrev].flushNext = node.flushNext
db.dirties[node.flushNext].flushPrev = node.flushPrev
}
// Dereference all children and delete the node
node.forChilds(func(hash common.Hash) {
db.dereference(hash, child)
node.forChildren(db.resolver, func(child common.Hash) {
db.dereference(child)
})
delete(db.dirties, child)
db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size))
if node.children != nil {
db.childrenSize -= cachedNodeChildrenSize
delete(db.dirties, hash)
db.dirtiesSize -= common.StorageSize(common.HashLength + len(node.node))
if node.external != nil {
db.childrenSize -= common.StorageSize(len(node.external) * common.HashLength)
}
}
}
@@ -556,22 +335,15 @@ func (db *Database) Cap(limit common.StorageSize) error {
// db.dirtiesSize only contains the useful data in the cache, but when reporting
// the total memory consumption, the maintenance metadata is also needed to be
// counted.
size := db.dirtiesSize + common.StorageSize((len(db.dirties)-1)*cachedNodeSize)
size += db.childrenSize - common.StorageSize(len(db.dirties[common.Hash{}].children)*(common.HashLength+2))
size := db.dirtiesSize + common.StorageSize(len(db.dirties)*cachedNodeSize)
size += db.childrenSize
// If the preimage cache got large enough, push to disk. If it's still small
// leave for later to deduplicate writes.
if db.preimages != nil {
if err := db.preimages.commit(false); err != nil {
return err
}
}
// Keep committing nodes from the flush-list until we're below allowance
oldest := db.oldest
for size > limit && oldest != (common.Hash{}) {
// Fetch the oldest referenced node and push into the batch
node := db.dirties[oldest]
rawdb.WriteLegacyTrieNode(batch, oldest, node.rlp())
rawdb.WriteLegacyTrieNode(batch, oldest, node.node)
// If we exceeded the ideal batch size, commit and reset
if batch.ValueSize() >= ethdb.IdealBatchSize {
@@ -584,9 +356,9 @@ func (db *Database) Cap(limit common.StorageSize) error {
// Iterate to the next flush item, or abort if the size cap was achieved. Size
// is the total size, including the useful cached data (hash -> blob), the
// cache item metadata, as well as external children mappings.
size -= common.StorageSize(common.HashLength + int(node.size) + cachedNodeSize)
if node.children != nil {
size -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2))
size -= common.StorageSize(common.HashLength + len(node.node) + cachedNodeSize)
if node.external != nil {
size -= common.StorageSize(len(node.external) * common.HashLength)
}
oldest = node.flushNext
}
@@ -604,9 +376,9 @@ func (db *Database) Cap(limit common.StorageSize) error {
delete(db.dirties, db.oldest)
db.oldest = node.flushNext
db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size))
if node.children != nil {
db.childrenSize -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2))
db.dirtiesSize -= common.StorageSize(common.HashLength + len(node.node))
if node.external != nil {
db.childrenSize -= common.StorageSize(len(node.external) * common.HashLength)
}
}
if db.oldest != (common.Hash{}) {
@@ -645,12 +417,6 @@ func (db *Database) Commit(node common.Hash, report bool) error {
start := time.Now()
batch := db.diskdb.NewBatch()
// Move all of the accumulated preimages into a write batch
if db.preimages != nil {
if err := db.preimages.commit(true); err != nil {
return err
}
}
// Move the trie itself into the batch, flushing if enough data is accumulated
nodes, storage := len(db.dirties), db.dirtiesSize
@@ -703,7 +469,9 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane
return nil
}
var err error
node.forChilds(func(child common.Hash) {
// Dereference all children and delete the node
node.forChildren(db.resolver, func(child common.Hash) {
if err == nil {
err = db.commit(child, batch, uncacher)
}
@@ -712,7 +480,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane
return err
}
// If we've reached an optimal batch size, commit and start over
rawdb.WriteLegacyTrieNode(batch, hash, node.rlp())
rawdb.WriteLegacyTrieNode(batch, hash, node.node)
if batch.ValueSize() >= ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return err
@@ -751,19 +519,23 @@ func (c *cleaner) Put(key []byte, rlp []byte) error {
switch hash {
case c.db.oldest:
c.db.oldest = node.flushNext
c.db.dirties[node.flushNext].flushPrev = common.Hash{}
if node.flushNext != (common.Hash{}) {
c.db.dirties[node.flushNext].flushPrev = common.Hash{}
}
case c.db.newest:
c.db.newest = node.flushPrev
c.db.dirties[node.flushPrev].flushNext = common.Hash{}
if node.flushPrev != (common.Hash{}) {
c.db.dirties[node.flushPrev].flushNext = common.Hash{}
}
default:
c.db.dirties[node.flushPrev].flushNext = node.flushNext
c.db.dirties[node.flushNext].flushPrev = node.flushPrev
}
// Remove the node from the dirty cache
delete(c.db.dirties, hash)
c.db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size))
if node.children != nil {
c.db.childrenSize -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2))
c.db.dirtiesSize -= common.StorageSize(common.HashLength + len(node.node))
if node.external != nil {
c.db.childrenSize -= common.StorageSize(len(node.external) * common.HashLength)
}
// Move the flushed node into the clean cache to prevent insta-reloads
if c.db.cleans != nil {
@@ -777,9 +549,21 @@ func (c *cleaner) Delete(key []byte) error {
panic("not implemented")
}
// Update inserts the dirty nodes in provided nodeset into database and
// link the account trie with multiple storage tries if necessary.
func (db *Database) Update(nodes *MergedNodeSet) error {
// Initialized returns an indicator if state data is already initialized
// in hash-based scheme by checking the presence of genesis state.
func (db *Database) Initialized(genesisRoot common.Hash) bool {
return rawdb.HasLegacyTrieNode(db.diskdb, genesisRoot)
}
// Update inserts the dirty nodes in provided nodeset into database and link the
// account trie with multiple storage tries if necessary.
func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error {
// Ensure the parent state is present and signal a warning if not.
if parent != types.EmptyRootHash {
if blob, _ := db.Node(parent); len(blob) == 0 {
log.Error("parent state is not present")
}
}
db.lock.Lock()
defer db.lock.Unlock()
@@ -790,34 +574,34 @@ func (db *Database) Update(nodes *MergedNodeSet) error {
// Note, the storage tries must be flushed before the account trie to
// retain the invariant that children go into the dirty cache first.
var order []common.Hash
for owner := range nodes.sets {
for owner := range nodes.Sets {
if owner == (common.Hash{}) {
continue
}
order = append(order, owner)
}
if _, ok := nodes.sets[common.Hash{}]; ok {
if _, ok := nodes.Sets[common.Hash{}]; ok {
order = append(order, common.Hash{})
}
for _, owner := range order {
subset := nodes.sets[owner]
subset.forEachWithOrder(func(path string, n *memoryNode) {
if n.isDeleted() {
subset := nodes.Sets[owner]
subset.ForEachWithOrder(func(path string, n *trienode.Node) {
if n.IsDeleted() {
return // ignore deletion
}
db.insert(n.hash, int(n.size), n.node)
db.insert(n.Hash, n.Blob)
})
}
// Link up the account trie and storage trie if the node points
// to an account trie leaf.
if set, present := nodes.sets[common.Hash{}]; present {
for _, n := range set.leaves {
if set, present := nodes.Sets[common.Hash{}]; present {
for _, n := range set.Leaves {
var account types.StateAccount
if err := rlp.DecodeBytes(n.blob, &account); err != nil {
if err := rlp.DecodeBytes(n.Blob, &account); err != nil {
return err
}
if account.Root != types.EmptyRootHash {
db.reference(account.Root, n.parent)
db.reference(account.Root, n.Parent)
}
}
}
@@ -826,104 +610,38 @@ func (db *Database) Update(nodes *MergedNodeSet) error {
// Size returns the current storage size of the memory cache in front of the
// persistent database layer.
func (db *Database) Size() (common.StorageSize, common.StorageSize) {
func (db *Database) Size() common.StorageSize {
db.lock.RLock()
defer db.lock.RUnlock()
// db.dirtiesSize only contains the useful data in the cache, but when reporting
// the total memory consumption, the maintenance metadata is also needed to be
// counted.
var metadataSize = common.StorageSize((len(db.dirties) - 1) * cachedNodeSize)
var metarootRefs = common.StorageSize(len(db.dirties[common.Hash{}].children) * (common.HashLength + 2))
var preimageSize common.StorageSize
if db.preimages != nil {
preimageSize = db.preimages.size()
}
return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, preimageSize
var metadataSize = common.StorageSize(len(db.dirties) * cachedNodeSize)
return db.dirtiesSize + db.childrenSize + metadataSize
}
// GetReader retrieves a node reader belonging to the given state root.
func (db *Database) GetReader(root common.Hash) Reader {
return newHashReader(db)
}
// hashReader is reader of hashDatabase which implements the Reader interface.
type hashReader struct {
db *Database
}
// newHashReader initializes the hash reader.
func newHashReader(db *Database) *hashReader {
return &hashReader{db: db}
}
// Node retrieves the trie node with the given node hash.
// No error will be returned if the node is not found.
func (reader *hashReader) Node(_ common.Hash, _ []byte, hash common.Hash) (node, error) {
return reader.db.node(hash), nil
}
// NodeBlob retrieves the RLP-encoded trie node blob with the given node hash.
// No error will be returned if the node is not found.
func (reader *hashReader) NodeBlob(_ common.Hash, _ []byte, hash common.Hash) ([]byte, error) {
blob, _ := reader.db.Node(hash)
return blob, nil
}
// saveCache saves clean state cache to given directory path
// using specified CPU cores.
func (db *Database) saveCache(dir string, threads int) error {
if db.cleans == nil {
return nil
}
log.Info("Writing clean trie cache to disk", "path", dir, "threads", threads)
start := time.Now()
err := db.cleans.SaveToFileConcurrent(dir, threads)
if err != nil {
log.Error("Failed to persist clean trie cache", "error", err)
return err
}
log.Info("Persisted the clean trie cache", "path", dir, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}
// SaveCache atomically saves fast cache data to the given dir using all
// available CPU cores.
func (db *Database) SaveCache(dir string) error {
return db.saveCache(dir, runtime.GOMAXPROCS(0))
}
// SaveCachePeriodically atomically saves fast cache data to the given dir with
// the specified interval. All dump operation will only use a single CPU core.
func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, stopCh <-chan struct{}) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
db.saveCache(dir, 1)
case <-stopCh:
return
}
}
}
// CommitPreimages flushes the dangling preimages to disk. It is meant to be
// called when closing the blockchain object, so that preimages are persisted
// to the database.
func (db *Database) CommitPreimages() error {
db.lock.Lock()
defer db.lock.Unlock()
if db.preimages == nil {
return nil
}
return db.preimages.commit(true)
}
// Close closes the trie database and releases all held resources.
func (db *Database) Close() error { return nil }
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string {
return rawdb.HashScheme
}
// Reader retrieves a node reader belonging to the given state root.
func (db *Database) Reader(root common.Hash) *reader {
return &reader{db: db}
}
// reader is a state reader of Database which implements the Reader interface.
type reader struct {
db *Database
}
// Node retrieves the trie node with the given node hash.
// No error will be returned if the node is not found.
func (reader *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
blob, _ := reader.db.Node(hash)
return blob, nil
}
trie/trienode/node.go
+197
View File
@@ -0,0 +1,197 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
package trienode
import (
"fmt"
"sort"
"strings"
"github.com/ethereum/go-ethereum/common"
)
// Node is a wrapper which contains the encoded blob of the trie node and its
// unique hash identifier. It is general enough that can be used to represent
// trie nodes corresponding to different trie implementations.
type Node struct {
Hash common.Hash // Node hash, empty for deleted node
Blob []byte // Encoded node blob, nil for the deleted node
}
// Size returns the total memory size used by this node.
func (n *Node) Size() int {
return len(n.Blob) + common.HashLength
}
// IsDeleted returns the indicator if the node is marked as deleted.
func (n *Node) IsDeleted() bool {
return n.Hash == (common.Hash{})
}
// WithPrev wraps the Node with the previous node value attached.
type WithPrev struct {
*Node
Prev []byte // Encoded original value, nil means it's non-existent
}
// Unwrap returns the internal Node object.
func (n *WithPrev) Unwrap() *Node {
return n.Node
}
// Size returns the total memory size used by this node. It overloads
// the function in Node by counting the size of previous value as well.
func (n *WithPrev) Size() int {
return n.Node.Size() + len(n.Prev)
}
// New constructs a node with provided node information.
func New(hash common.Hash, blob []byte) *Node {
return &Node{Hash: hash, Blob: blob}
}
// NewWithPrev constructs a node with provided node information.
func NewWithPrev(hash common.Hash, blob []byte, prev []byte) *WithPrev {
return &WithPrev{
Node: New(hash, blob),
Prev: prev,
}
}
// leaf represents a trie leaf node
type leaf struct {
Blob []byte // raw blob of leaf
Parent common.Hash // the hash of parent node
}
// NodeSet contains a set of nodes collected during the commit operation.
// Each node is keyed by path. It's not thread-safe to use.
type NodeSet struct {
Owner common.Hash
Leaves []*leaf
Nodes map[string]*WithPrev
updates int // the count of updated and inserted nodes
deletes int // the count of deleted nodes
}
// NewNodeSet initializes a node set. The owner is zero for the account trie and
// the owning account address hash for storage tries.
func NewNodeSet(owner common.Hash) *NodeSet {
return &NodeSet{
Owner: owner,
Nodes: make(map[string]*WithPrev),
}
}
// ForEachWithOrder iterates the nodes with the order from bottom to top,
// right to left, nodes with the longest path will be iterated first.
func (set *NodeSet) ForEachWithOrder(callback func(path string, n *Node)) {
var paths sort.StringSlice
for path := range set.Nodes {
paths = append(paths, path)
}
// Bottom-up, longest path first
sort.Sort(sort.Reverse(paths))
for _, path := range paths {
callback(path, set.Nodes[path].Unwrap())
}
}
// AddNode adds the provided node into set.
func (set *NodeSet) AddNode(path []byte, n *WithPrev) {
if n.IsDeleted() {
set.deletes += 1
} else {
set.updates += 1
}
set.Nodes[string(path)] = n
}
// AddLeaf adds the provided leaf node into set. TODO(rjl493456442) how can
// we get rid of it?
func (set *NodeSet) AddLeaf(parent common.Hash, blob []byte) {
set.Leaves = append(set.Leaves, &leaf{Blob: blob, Parent: parent})
}
// Size returns the number of dirty nodes in set.
func (set *NodeSet) Size() (int, int) {
return set.updates, set.deletes
}
// Hashes returns the hashes of all updated nodes. TODO(rjl493456442) how can
// we get rid of it?
func (set *NodeSet) Hashes() []common.Hash {
var ret []common.Hash
for _, node := range set.Nodes {
ret = append(ret, node.Hash)
}
return ret
}
// Summary returns a string-representation of the NodeSet.
func (set *NodeSet) Summary() string {
var out = new(strings.Builder)
fmt.Fprintf(out, "nodeset owner: %v\n", set.Owner)
if set.Nodes != nil {
for path, n := range set.Nodes {
// Deletion
if n.IsDeleted() {
fmt.Fprintf(out, " [-]: %x prev: %x\n", path, n.Prev)
continue
}
// Insertion
if len(n.Prev) == 0 {
fmt.Fprintf(out, " [+]: %x -> %v\n", path, n.Hash)
continue
}
// Update
fmt.Fprintf(out, " [*]: %x -> %v prev: %x\n", path, n.Hash, n.Prev)
}
}
for _, n := range set.Leaves {
fmt.Fprintf(out, "[leaf]: %v\n", n)
}
return out.String()
}
// MergedNodeSet represents a merged node set for a group of tries.
type MergedNodeSet struct {
Sets map[common.Hash]*NodeSet
}
// NewMergedNodeSet initializes an empty merged set.
func NewMergedNodeSet() *MergedNodeSet {
return &MergedNodeSet{Sets: make(map[common.Hash]*NodeSet)}
}
// NewWithNodeSet constructs a merged nodeset with the provided single set.
func NewWithNodeSet(set *NodeSet) *MergedNodeSet {
merged := NewMergedNodeSet()
merged.Merge(set)
return merged
}
// Merge merges the provided dirty nodes of a trie into the set. The assumption
// is held that no duplicated set belonging to the same trie will be merged twice.
func (set *MergedNodeSet) Merge(other *NodeSet) error {
_, present := set.Sets[other.Owner]
if present {
return fmt.Errorf("duplicate trie for owner %#x", other.Owner)
}
set.Sets[other.Owner] = other
return nil
}