rework iterator state stuff
This commit is contained in:
parent
5b51346a83
commit
100c106ee5
35
builder.go
35
builder.go
@ -148,9 +148,9 @@ func (sdb *builder) WriteStateDiff(
|
||||
func(subdiv uint) {
|
||||
g.Go(func() error {
|
||||
a, b := subitersA[subdiv], subitersB[subdiv]
|
||||
it, aux := utils.NewSymmetricDifferenceIterator(a, b)
|
||||
it := utils.NewSymmetricDifferenceIterator(a, b)
|
||||
return sdb.processAccounts(ctx,
|
||||
it, aux,
|
||||
it, &it.SymmDiffState,
|
||||
params.watchedAddressesLeafPaths,
|
||||
nodeSink, ipldSink, logger,
|
||||
)
|
||||
@ -165,7 +165,7 @@ func (sdb *builder) WriteStateDiffTracked(
|
||||
args Args, params Params,
|
||||
nodeSink sdtypes.StateNodeSink,
|
||||
ipldSink sdtypes.IPLDSink,
|
||||
tracker tracker.Tracker,
|
||||
tracker tracker.IteratorTracker,
|
||||
) error {
|
||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.WriteStateDiffTimer)
|
||||
// Load tries for old and new states
|
||||
@ -178,18 +178,14 @@ func (sdb *builder) WriteStateDiffTracked(
|
||||
return fmt.Errorf("error opening new state trie: %w", err)
|
||||
}
|
||||
|
||||
var subiters []trie.NodeIterator
|
||||
var auxes []*utils.SymmDiffAux
|
||||
var subiters, bases []trie.NodeIterator
|
||||
// Constructor for difference iterator at a specific (recovered) path
|
||||
makeIterator := func(key []byte) trie.NodeIterator {
|
||||
a := triea.NodeIterator(key)
|
||||
b := trieb.NodeIterator(key)
|
||||
diffit, aux := utils.NewSymmetricDifferenceIterator(a, b)
|
||||
// iterators are constructed in-order, so these will align
|
||||
auxes = append(auxes, aux)
|
||||
return diffit
|
||||
return utils.NewSymmetricDifferenceIterator(a, b)
|
||||
}
|
||||
subiters, err = tracker.Restore(makeIterator)
|
||||
subiters, bases, err = tracker.Restore(makeIterator)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error restoring iterators: %w", err)
|
||||
}
|
||||
@ -214,7 +210,7 @@ func (sdb *builder) WriteStateDiffTracked(
|
||||
func(subdiv uint) {
|
||||
g.Go(func() error {
|
||||
return sdb.processAccounts(ctx,
|
||||
subiters[subdiv], auxes[subdiv],
|
||||
subiters[subdiv], &bases[subdiv].(*utils.SymmDiffIterator).SymmDiffState,
|
||||
params.watchedAddressesLeafPaths,
|
||||
nodeSink, ipldSink, logger,
|
||||
)
|
||||
@ -225,9 +221,10 @@ func (sdb *builder) WriteStateDiffTracked(
|
||||
}
|
||||
|
||||
// processAccounts processes account creations, deletions, and updates
|
||||
// the NodeIterator and SymmDiffIterator instances should refer to the same object, will only be used
|
||||
func (sdb *builder) processAccounts(
|
||||
ctx context.Context,
|
||||
it trie.NodeIterator, aux *utils.SymmDiffAux,
|
||||
it trie.NodeIterator, symdiff *utils.SymmDiffState,
|
||||
watchedAddressesLeafPaths [][]byte,
|
||||
nodeSink sdtypes.StateNodeSink, ipldSink sdtypes.IPLDSink,
|
||||
logger log.Logger,
|
||||
@ -250,7 +247,7 @@ func (sdb *builder) processAccounts(
|
||||
if !isWatchedPathPrefix(watchedAddressesLeafPaths, it.Path()) {
|
||||
continue
|
||||
}
|
||||
if aux.FromA() { // Node exists in the old trie
|
||||
if symdiff.FromA() { // Node exists in the old trie
|
||||
if it.Leaf() {
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil {
|
||||
@ -259,7 +256,7 @@ func (sdb *builder) processAccounts(
|
||||
leafKey := make([]byte, len(it.LeafKey()))
|
||||
copy(leafKey, it.LeafKey())
|
||||
|
||||
if aux.CommonPath() {
|
||||
if symdiff.CommonPath() {
|
||||
// If B also contains this leaf node, this is the old state of an updated account.
|
||||
if update, ok := updates[string(leafKey)]; ok {
|
||||
update.oldRoot = account.Root
|
||||
@ -284,7 +281,7 @@ func (sdb *builder) processAccounts(
|
||||
return err
|
||||
}
|
||||
|
||||
if aux.CommonPath() {
|
||||
if symdiff.CommonPath() {
|
||||
// If A also contains this leaf node, this is the new state of an updated account.
|
||||
if update, ok := updates[string(accountW.LeafKey)]; ok {
|
||||
update.new = *accountW
|
||||
@ -354,7 +351,7 @@ func (sdb *builder) processAccounts(
|
||||
}
|
||||
}
|
||||
|
||||
metrics.IndexerMetrics.DifferenceIteratorCounter.Inc(int64(aux.Count()))
|
||||
metrics.IndexerMetrics.DifferenceIteratorCounter.Inc(int64(symdiff.Count()))
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
@ -482,10 +479,10 @@ func (sdb *builder) processStorageUpdates(
|
||||
|
||||
var prevBlob []byte
|
||||
a, b := oldTrie.NodeIterator(nil), newTrie.NodeIterator(nil)
|
||||
it, aux := utils.NewSymmetricDifferenceIterator(a, b)
|
||||
it := utils.NewSymmetricDifferenceIterator(a, b)
|
||||
for it.Next(true) {
|
||||
if aux.FromA() {
|
||||
if it.Leaf() && !aux.CommonPath() {
|
||||
if it.FromA() {
|
||||
if it.Leaf() && !it.CommonPath() {
|
||||
// If this node's leaf key is absent from B, the storage slot was vacated.
|
||||
// In that case, emit an empty "removed" storage node record.
|
||||
if err := storageSink(sdtypes.StorageLeafNode{
|
||||
|
2
go.mod
2
go.mod
@ -124,7 +124,7 @@ require (
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.1.2-0.20230925184550-062eb329435f
|
||||
github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.1.2-0.20230926100620-802551012643
|
||||
github.com/cerc-io/eth-testing => git.vdb.to/cerc-io/eth-testing v0.3.1-0.20230925181540-2ea71042e7e0
|
||||
github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
|
||||
github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
|
||||
|
4
go.sum
4
go.sum
@ -1,6 +1,6 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
git.vdb.to/cerc-io/eth-iterator-utils v0.1.2-0.20230925184550-062eb329435f h1:sIuSkD6U7uYD/FGfvWOBViIuaHd+YhLM0Hln+4BQM10=
|
||||
git.vdb.to/cerc-io/eth-iterator-utils v0.1.2-0.20230925184550-062eb329435f/go.mod h1:Xv+d7Q11qGJcggcfxoj2JEvJJBKj0C66I6PyG5/lz9o=
|
||||
git.vdb.to/cerc-io/eth-iterator-utils v0.1.2-0.20230926100620-802551012643 h1:yJFyJgGVy1RMEJqPrTYyaB7fF1wpfx0Df5Bsunb+Lyg=
|
||||
git.vdb.to/cerc-io/eth-iterator-utils v0.1.2-0.20230926100620-802551012643/go.mod h1:Xv+d7Q11qGJcggcfxoj2JEvJJBKj0C66I6PyG5/lz9o=
|
||||
git.vdb.to/cerc-io/eth-testing v0.3.1-0.20230925181540-2ea71042e7e0 h1:fWAvsSiuDqveuxwnfc8psInfLZhMqHlQnmOpOHsd8Tk=
|
||||
git.vdb.to/cerc-io/eth-testing v0.3.1-0.20230925181540-2ea71042e7e0/go.mod h1:qdvpc/W1xvf2MKx3rMOqvFvYaYIHG77Z1g0lwsmw0Uk=
|
||||
git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1 h1:KLjxHwp9Zp7xhECccmJS00RiL+VwTuUGLU7qeIctg8g=
|
||||
|
@ -7,9 +7,9 @@ import (
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
)
|
||||
|
||||
type symmDiffIterator struct {
|
||||
type SymmDiffIterator struct {
|
||||
a, b iterState // Nodes returned are those in b - a and a - b (keys only)
|
||||
SymmDiffAux
|
||||
SymmDiffState
|
||||
}
|
||||
|
||||
// pairs an iterator with a cache of its valid status
|
||||
@ -18,10 +18,10 @@ type iterState struct {
|
||||
valid bool
|
||||
}
|
||||
|
||||
// SymmDiffAux exposes state specific to symmetric difference iteration, which is not accessible
|
||||
// SymmDiffState exposes state specific to symmetric difference iteration, which is not accessible
|
||||
// from the NodeIterator interface. This includes the number of nodes seen, whether the current key
|
||||
// is common to both A and B, and whether the current node is sourced from A or B.
|
||||
type SymmDiffAux struct {
|
||||
type SymmDiffState struct {
|
||||
yieldFromA bool // Whether next node comes from a
|
||||
count int // Number of nodes scanned on either trie
|
||||
eqPathIndex int // Count index of last pair of equal paths, to detect an updated key
|
||||
@ -30,14 +30,14 @@ type SymmDiffAux struct {
|
||||
// NewSymmetricDifferenceIterator constructs a trie.NodeIterator that iterates over the symmetric difference
|
||||
// of elements in a and b, i.e., the elements in a that are not in b, and vice versa.
|
||||
// Returns the iterator, and a pointer to an auxiliary object for accessing the state not exposed by the NodeIterator interface recording the number of nodes seen.
|
||||
func NewSymmetricDifferenceIterator(a, b trie.NodeIterator) (trie.NodeIterator, *SymmDiffAux) {
|
||||
it := &symmDiffIterator{
|
||||
func NewSymmetricDifferenceIterator(a, b trie.NodeIterator) *SymmDiffIterator {
|
||||
it := &SymmDiffIterator{
|
||||
a: iterState{a, true},
|
||||
b: iterState{b, true},
|
||||
// common paths are detected by a distance <=1 between count and this index, so we start at -2
|
||||
SymmDiffAux: SymmDiffAux{eqPathIndex: -2},
|
||||
SymmDiffState: SymmDiffState{eqPathIndex: -2},
|
||||
}
|
||||
return it, &it.SymmDiffAux
|
||||
return it
|
||||
}
|
||||
|
||||
func (st *iterState) Next(descend bool) bool {
|
||||
@ -46,65 +46,65 @@ func (st *iterState) Next(descend bool) bool {
|
||||
}
|
||||
|
||||
// FromA returns true if the current node is sourced from A.
|
||||
func (it *SymmDiffAux) FromA() bool {
|
||||
func (it *SymmDiffState) FromA() bool {
|
||||
return it.yieldFromA
|
||||
}
|
||||
|
||||
// CommonPath returns true if a node with the current path exists in each sub-iterator - i.e. it
|
||||
// represents an updated node.
|
||||
func (it *SymmDiffAux) CommonPath() bool {
|
||||
func (it *SymmDiffState) CommonPath() bool {
|
||||
return it.count-it.eqPathIndex <= 1
|
||||
}
|
||||
|
||||
// Count returns the number of nodes seen.
|
||||
func (it *SymmDiffAux) Count() int {
|
||||
func (it *SymmDiffState) Count() int {
|
||||
return it.count
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) curr() *iterState {
|
||||
func (it *SymmDiffIterator) curr() *iterState {
|
||||
if it.yieldFromA {
|
||||
return &it.a
|
||||
}
|
||||
return &it.b
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) Hash() common.Hash {
|
||||
func (it *SymmDiffIterator) Hash() common.Hash {
|
||||
return it.curr().Hash()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) Parent() common.Hash {
|
||||
func (it *SymmDiffIterator) Parent() common.Hash {
|
||||
return it.curr().Parent()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) Leaf() bool {
|
||||
func (it *SymmDiffIterator) Leaf() bool {
|
||||
return it.curr().Leaf()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) LeafKey() []byte {
|
||||
func (it *SymmDiffIterator) LeafKey() []byte {
|
||||
return it.curr().LeafKey()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) LeafBlob() []byte {
|
||||
func (it *SymmDiffIterator) LeafBlob() []byte {
|
||||
return it.curr().LeafBlob()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) LeafProof() [][]byte {
|
||||
func (it *SymmDiffIterator) LeafProof() [][]byte {
|
||||
return it.curr().LeafProof()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) Path() []byte {
|
||||
func (it *SymmDiffIterator) Path() []byte {
|
||||
return it.curr().Path()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) NodeBlob() []byte {
|
||||
func (it *SymmDiffIterator) NodeBlob() []byte {
|
||||
return it.curr().NodeBlob()
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) AddResolver(resolver trie.NodeResolver) {
|
||||
func (it *SymmDiffIterator) AddResolver(resolver trie.NodeResolver) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) Next(bool) bool {
|
||||
func (it *SymmDiffIterator) Next(bool) bool {
|
||||
// NodeIterators start in a "pre-valid" state, so the first Next advances to a valid node.
|
||||
if it.count == 0 {
|
||||
if it.a.Next(true) {
|
||||
@ -122,7 +122,7 @@ func (it *symmDiffIterator) Next(bool) bool {
|
||||
return it.a.valid || it.b.valid
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) seek() {
|
||||
func (it *SymmDiffIterator) seek() {
|
||||
// Invariants:
|
||||
// - At the end of the function, the sub-iterator with the lexically lesser path
|
||||
// points to the next element
|
||||
@ -163,7 +163,7 @@ func (it *symmDiffIterator) seek() {
|
||||
}
|
||||
}
|
||||
|
||||
func (it *symmDiffIterator) Error() error {
|
||||
func (it *SymmDiffIterator) Error() error {
|
||||
if err := it.a.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user