consolidate, simplify
This commit is contained in:
parent
2ebd0421e2
commit
06723ee4e0
390
builder.go
390
builder.go
@ -21,7 +21,6 @@ package statediff
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -35,7 +34,6 @@ import (
|
|||||||
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
|
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
|
||||||
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
|
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
|
||||||
"github.com/cerc-io/plugeth-statediff/indexer/shared"
|
"github.com/cerc-io/plugeth-statediff/indexer/shared"
|
||||||
"github.com/cerc-io/plugeth-statediff/trie_helpers"
|
|
||||||
sdtypes "github.com/cerc-io/plugeth-statediff/types"
|
sdtypes "github.com/cerc-io/plugeth-statediff/types"
|
||||||
"github.com/cerc-io/plugeth-statediff/utils"
|
"github.com/cerc-io/plugeth-statediff/utils"
|
||||||
"github.com/cerc-io/plugeth-statediff/utils/log"
|
"github.com/cerc-io/plugeth-statediff/utils/log"
|
||||||
@ -59,25 +57,19 @@ type StateDiffBuilder struct {
|
|||||||
stateCache adapt.StateView
|
stateCache adapt.StateView
|
||||||
}
|
}
|
||||||
|
|
||||||
type IterPair struct {
|
type iterPair struct {
|
||||||
Older, Newer trie.NodeIterator
|
Older, Newer trie.NodeIterator
|
||||||
}
|
}
|
||||||
|
|
||||||
func StateNodeAppender(nodes *[]sdtypes.StateLeafNode) sdtypes.StateNodeSink {
|
type accountUpdate struct {
|
||||||
return func(node sdtypes.StateLeafNode) error {
|
new sdtypes.AccountWrapper
|
||||||
*nodes = append(*nodes, node)
|
oldRoot common.Hash
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
func StorageNodeAppender(nodes *[]sdtypes.StorageLeafNode) sdtypes.StorageNodeSink {
|
type accountUpdateMap map[string]*accountUpdate
|
||||||
return func(node sdtypes.StorageLeafNode) error {
|
|
||||||
*nodes = append(*nodes, node)
|
func appender[T any](to *[]T) func(T) error {
|
||||||
return nil
|
return func(a T) error {
|
||||||
}
|
*to = append(*to, a)
|
||||||
}
|
|
||||||
func IPLDMappingAppender(iplds *[]sdtypes.IPLD) sdtypes.IPLDSink {
|
|
||||||
return func(c sdtypes.IPLD) error {
|
|
||||||
*iplds = append(*iplds, c)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -94,7 +86,7 @@ func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (sdt
|
|||||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildStateDiffObjectTimer)
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildStateDiffObjectTimer)
|
||||||
var stateNodes []sdtypes.StateLeafNode
|
var stateNodes []sdtypes.StateLeafNode
|
||||||
var iplds []sdtypes.IPLD
|
var iplds []sdtypes.IPLD
|
||||||
err := sdb.WriteStateDiff(args, params, StateNodeAppender(&stateNodes), IPLDMappingAppender(&iplds))
|
err := sdb.WriteStateDiff(args, params, appender(&stateNodes), appender(&iplds))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sdtypes.StateObject{}, err
|
return sdtypes.StateObject{}, err
|
||||||
}
|
}
|
||||||
@ -107,147 +99,107 @@ func (sdb *StateDiffBuilder) BuildStateDiffObject(args Args, params Params) (sdt
|
|||||||
}
|
}
|
||||||
|
|
||||||
// WriteStateDiff writes a statediff object to output sinks
|
// WriteStateDiff writes a statediff object to output sinks
|
||||||
func (sdb *StateDiffBuilder) WriteStateDiff(args Args, params Params, nodeSink sdtypes.StateNodeSink,
|
func (sdb *StateDiffBuilder) WriteStateDiff(
|
||||||
ipldSink sdtypes.IPLDSink) error {
|
args Args, params Params,
|
||||||
|
nodeSink sdtypes.StateNodeSink,
|
||||||
|
ipldSink sdtypes.IPLDSink,
|
||||||
|
) error {
|
||||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.WriteStateDiffTimer)
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.WriteStateDiffTimer)
|
||||||
// Load tries for old and new states
|
// Load tries for old and new states
|
||||||
oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
|
oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error creating trie for oldStateRoot: %w", err)
|
return fmt.Errorf("error opening old state trie: %w", err)
|
||||||
}
|
}
|
||||||
newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
|
newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error creating trie for newStateRoot: %w", err)
|
return fmt.Errorf("error opening new state trie: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
iters := IterPair{
|
iters := iterPair{
|
||||||
Older: oldTrie.NodeIterator(nil),
|
Older: oldTrie.NodeIterator(nil),
|
||||||
Newer: newTrie.NodeIterator(nil),
|
Newer: newTrie.NodeIterator(nil),
|
||||||
}
|
}
|
||||||
logger := log.New("hash", args.BlockHash, "number", args.BlockNumber)
|
logger := log.New("hash", args.BlockHash, "number", args.BlockNumber)
|
||||||
return sdb.BuildStateDiff(iters, params, nodeSink, ipldSink, logger)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sdb *StateDiffBuilder) BuildStateDiff(iterPair IterPair, params Params,
|
err = sdb.processAccounts(
|
||||||
nodeSink sdtypes.StateNodeSink, ipldSink sdtypes.IPLDSink, logger log.Logger) error {
|
iters.Older, iters.Newer,
|
||||||
logger.Trace("statediff BEGIN BuildStateDiff")
|
params.watchedAddressesLeafPaths,
|
||||||
defer metrics.ReportAndUpdateDuration("statediff END BuildStateDiff", time.Now(), logger, metrics.IndexerMetrics.BuildStateDiffTimer)
|
nodeSink, ipldSink, logger)
|
||||||
// collect a slice of all the nodes that were touched and exist at B (B-A)
|
|
||||||
// a map of their leafkey to all the accounts that were touched and exist at B
|
|
||||||
// and a slice of all the paths for the nodes in both of the above sets
|
|
||||||
diffAccountsAtB, diffAccountsAtA, err := sdb.createdAndUpdatedState(
|
|
||||||
iterPair.Older, iterPair.Newer, params.watchedAddressesLeafPaths, nodeSink, ipldSink, logger)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error collecting createdAndUpdatedNodes: %w", err)
|
return fmt.Errorf("error collecting createdAndUpdatedNodes: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// collect and sort the leafkey keys for both account mappings into a slice
|
|
||||||
t := time.Now()
|
|
||||||
createKeys := trie_helpers.SortKeys(diffAccountsAtB)
|
|
||||||
deleteKeys := trie_helpers.SortKeys(diffAccountsAtA)
|
|
||||||
logger.Debug("statediff BuildStateDiff sort", "duration", time.Since(t))
|
|
||||||
|
|
||||||
// and then find the intersection of these keys
|
|
||||||
// these are the leafkeys for the accounts which exist at both A and B but are different
|
|
||||||
// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
|
|
||||||
// and leaving the truly created or deleted keys in place
|
|
||||||
t = time.Now()
|
|
||||||
updatedKeys := trie_helpers.FindIntersection(createKeys, deleteKeys)
|
|
||||||
logger.Debug("statediff BuildStateDiff intersection",
|
|
||||||
"count", len(updatedKeys),
|
|
||||||
"duration", time.Since(t))
|
|
||||||
|
|
||||||
// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
|
|
||||||
err = sdb.buildAccountUpdates(diffAccountsAtB, diffAccountsAtA, updatedKeys, nodeSink, ipldSink, logger)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error building diff for updated accounts: %w", err)
|
|
||||||
}
|
|
||||||
// build the diff nodes for created accounts
|
|
||||||
err = sdb.buildAccountCreations(diffAccountsAtB, nodeSink, ipldSink, logger)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error building diff for created accounts: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// createdAndUpdatedState returns
|
// processAccounts processes account creations and deletions, and returns a set of updated
|
||||||
// a slice of all the intermediate nodes that exist in a different state at B than A
|
// existing accounts, indexed by leaf key.
|
||||||
// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
|
func (sdb *StateDiffBuilder) processAccounts(a, b trie.NodeIterator,
|
||||||
// and a slice of the paths for all of the nodes included in both
|
|
||||||
func (sdb *StateDiffBuilder) createdAndUpdatedState(a, b trie.NodeIterator,
|
|
||||||
watchedAddressesLeafPaths [][]byte,
|
watchedAddressesLeafPaths [][]byte,
|
||||||
nodeSink sdtypes.StateNodeSink, ipldSink sdtypes.IPLDSink,
|
nodeSink sdtypes.StateNodeSink, ipldSink sdtypes.IPLDSink,
|
||||||
logger log.Logger,
|
logger log.Logger,
|
||||||
) (sdtypes.AccountMap, sdtypes.AccountMap, error) {
|
) error {
|
||||||
logger.Trace("statediff BEGIN createdAndUpdatedState")
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.CreatedAndUpdatedStateTimer)
|
||||||
defer metrics.ReportAndUpdateDuration("statediff END createdAndUpdatedState", time.Now(), logger, metrics.IndexerMetrics.CreatedAndUpdatedStateTimer)
|
|
||||||
diffAccountsAtB := make(sdtypes.AccountMap)
|
|
||||||
diffAccountsAtA := make(sdtypes.AccountMap)
|
|
||||||
|
|
||||||
// Cache the RLP of the previous node on A and B. When we hit a value node this will be the parent blob.
|
updates := make(accountUpdateMap)
|
||||||
var prevBlobFromA, prevBlobFromB []byte
|
// Cache the RLP of the previous node. When we hit a value node this will be the parent blob.
|
||||||
|
var prevBlob []byte
|
||||||
it, itCount := utils.NewSymmetricDifferenceIterator(a, b)
|
it, itCount := utils.NewSymmetricDifferenceIterator(a, b)
|
||||||
for it.Next(true) {
|
for it.Next(true) {
|
||||||
// ignore node if it is not along paths of interest
|
// ignore node if it is not along paths of interest
|
||||||
if !isWatchedPathPrefix(watchedAddressesLeafPaths, it.Path()) {
|
if !isWatchedPathPrefix(watchedAddressesLeafPaths, it.Path()) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if it.FromA() {
|
if it.FromA() { // Node exists in the old trie
|
||||||
if it.Leaf() {
|
if it.Leaf() {
|
||||||
accountW, err := sdb.processStateValueNode(it, prevBlobFromB)
|
var account types.StateAccount
|
||||||
if err != nil {
|
if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
if accountW == nil {
|
leafKey := make([]byte, len(it.LeafKey()))
|
||||||
continue
|
copy(leafKey, it.LeafKey())
|
||||||
}
|
|
||||||
leafKey := hex.EncodeToString(accountW.LeafKey)
|
|
||||||
diffAccountsAtA[leafKey] = *accountW
|
|
||||||
|
|
||||||
if !it.CommonPath() {
|
if it.CommonPath() {
|
||||||
// if this node is removed, not updated, that means the account was
|
// If B also contains this leaf node, this is the old state of an updated account.
|
||||||
// deleted. in that case, emit an empty "removed" state node record and
|
if update, ok := updates[string(leafKey)]; ok {
|
||||||
// "removed" storage node records for all the storage slots.
|
update.oldRoot = account.Root
|
||||||
diff := sdtypes.StateLeafNode{
|
} else {
|
||||||
AccountWrapper: sdtypes.AccountWrapper{
|
updates[string(leafKey)] = &accountUpdate{oldRoot: account.Root}
|
||||||
LeafKey: accountW.LeafKey,
|
|
||||||
CID: shared.RemovedNodeStateCID,
|
|
||||||
},
|
|
||||||
Removed: true,
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
err = sdb.buildRemovedAccountStorageNodes(accountW.Account.Root, StorageNodeAppender(&diff.StorageDiff))
|
// This node was removed, meaning the account was deleted. Emit empty
|
||||||
|
// "removed" records for the state node and all storage all storage slots.
|
||||||
|
err := sdb.processAccountDeletion(leafKey, account, nodeSink)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("failed building storage diffs for removed state account with key %x\r\nerror: %w", leafKey, err)
|
return err
|
||||||
}
|
|
||||||
if err := nodeSink(diff); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
prevBlobFromB = make([]byte, len(it.NodeBlob()))
|
|
||||||
copy(prevBlobFromB, it.NodeBlob())
|
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// index values by leaf key
|
// Node exists in the new trie
|
||||||
if it.Leaf() {
|
if it.Leaf() {
|
||||||
// if it is a "value" node, we will index the value by leaf key
|
accountW, err := sdb.decodeStateLeaf(it, prevBlob)
|
||||||
accountW, err := sdb.processStateValueNode(it, prevBlobFromA)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
if accountW == nil {
|
|
||||||
continue
|
if it.CommonPath() {
|
||||||
|
// If A also contains this leaf node, this is the new state of an updated account.
|
||||||
|
if update, ok := updates[string(accountW.LeafKey)]; ok {
|
||||||
|
update.new = *accountW
|
||||||
|
} else {
|
||||||
|
updates[string(accountW.LeafKey)] = &accountUpdate{new: *accountW}
|
||||||
|
}
|
||||||
|
} else { // account was created
|
||||||
|
err := sdb.processAccountCreation(accountW, ipldSink, nodeSink)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// for now, just add it to diffAccountsAtB
|
|
||||||
// we will compare to diffAccountsAtA to determine which diffAccountsAtB
|
|
||||||
// were creations and which were updates and also identify accounts that were removed going A->B
|
|
||||||
diffAccountsAtB[hex.EncodeToString(accountW.LeafKey)] = *accountW
|
|
||||||
} else {
|
} else {
|
||||||
// trie nodes will be written to blockstore only
|
// New trie nodes will be written to blockstore only.
|
||||||
// reminder that this includes leaf nodes, since the geth iterator.Leaf() actually
|
// Reminder: this includes leaf nodes, since the geth iterator.Leaf() actually
|
||||||
// signifies a "value" node
|
// signifies a "value" node.
|
||||||
if it.Hash() == zeroHash {
|
if it.Hash() == zeroHash {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -257,42 +209,108 @@ func (sdb *StateDiffBuilder) createdAndUpdatedState(a, b trie.NodeIterator,
|
|||||||
if len(watchedAddressesLeafPaths) > 0 {
|
if len(watchedAddressesLeafPaths) > 0 {
|
||||||
var elements []interface{}
|
var elements []interface{}
|
||||||
if err := rlp.DecodeBytes(nodeVal, &elements); err != nil {
|
if err := rlp.DecodeBytes(nodeVal, &elements); err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
ok, err := isLeaf(elements)
|
ok, err := isLeaf(elements)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
partialPath := utils.CompactToHex(elements[0].([]byte))
|
if ok {
|
||||||
valueNodePath := append(it.Path(), partialPath...)
|
partialPath := utils.CompactToHex(elements[0].([]byte))
|
||||||
if ok && !isWatchedPath(watchedAddressesLeafPaths, valueNodePath) {
|
valueNodePath := append(it.Path(), partialPath...)
|
||||||
continue
|
if !isWatchedPath(watchedAddressesLeafPaths, valueNodePath) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := ipldSink(sdtypes.IPLD{
|
if err := ipldSink(sdtypes.IPLD{
|
||||||
CID: ipld.Keccak256ToCid(ipld.MEthStateTrie, it.Hash().Bytes()).String(),
|
CID: ipld.Keccak256ToCid(ipld.MEthStateTrie, it.Hash().Bytes()).String(),
|
||||||
Content: nodeVal,
|
Content: nodeVal,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, nil, err
|
return err
|
||||||
}
|
}
|
||||||
prevBlobFromA = nodeVal
|
prevBlob = nodeVal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, update := range updates {
|
||||||
|
var storageDiff []sdtypes.StorageLeafNode
|
||||||
|
err := sdb.processUpdatedAccountStorage(
|
||||||
|
update.oldRoot, update.new.Account.Root,
|
||||||
|
appender(&storageDiff), ipldSink,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error processing incremental storage diffs for account with leafkey %x\r\nerror: %w", key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = nodeSink(sdtypes.StateLeafNode{
|
||||||
|
AccountWrapper: update.new,
|
||||||
|
StorageDiff: storageDiff,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug("statediff COUNTS createdAndUpdatedState", "it", itCount, "diffAccountsAtB", len(diffAccountsAtB))
|
|
||||||
metrics.IndexerMetrics.DifferenceIteratorCounter.Inc(int64(*itCount))
|
metrics.IndexerMetrics.DifferenceIteratorCounter.Inc(int64(*itCount))
|
||||||
return diffAccountsAtB, diffAccountsAtA, it.Error()
|
return it.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sdb *StateDiffBuilder) processAccountDeletion(
|
||||||
|
leafKey []byte, account types.StateAccount, nodeSink sdtypes.StateNodeSink,
|
||||||
|
) error {
|
||||||
|
diff := sdtypes.StateLeafNode{
|
||||||
|
AccountWrapper: sdtypes.AccountWrapper{
|
||||||
|
LeafKey: leafKey,
|
||||||
|
CID: shared.RemovedNodeStateCID,
|
||||||
|
},
|
||||||
|
Removed: true,
|
||||||
|
}
|
||||||
|
err := sdb.processRemovedAccountStorage(account.Root, appender(&diff.StorageDiff))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed building storage diffs for removed state account with key %x\r\nerror: %w", leafKey, err)
|
||||||
|
}
|
||||||
|
return nodeSink(diff)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sdb *StateDiffBuilder) processAccountCreation(
|
||||||
|
accountW *sdtypes.AccountWrapper, ipldSink sdtypes.IPLDSink, nodeSink sdtypes.StateNodeSink,
|
||||||
|
) error {
|
||||||
|
diff := sdtypes.StateLeafNode{
|
||||||
|
AccountWrapper: *accountW,
|
||||||
|
}
|
||||||
|
if !bytes.Equal(accountW.Account.CodeHash, nullCodeHash) {
|
||||||
|
// For contract creations, any storage node contained is a diff
|
||||||
|
err := sdb.processCreatedAccountStorage(accountW.Account.Root, appender(&diff.StorageDiff), ipldSink)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed building eventual storage diffs for node with leaf key %x\r\nerror: %w", accountW.LeafKey, err)
|
||||||
|
}
|
||||||
|
// emit codehash => code mappings for contract
|
||||||
|
codeHash := common.BytesToHash(accountW.Account.CodeHash)
|
||||||
|
code, err := sdb.stateCache.ContractCode(codeHash)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %w", codeHash, err)
|
||||||
|
}
|
||||||
|
if err := ipldSink(sdtypes.IPLD{
|
||||||
|
CID: ipld.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes()).String(),
|
||||||
|
Content: code,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nodeSink(diff)
|
||||||
}
|
}
|
||||||
|
|
||||||
// decodes account at leaf and encodes RLP data to CID
|
// decodes account at leaf and encodes RLP data to CID
|
||||||
// reminder: it.Leaf() == true when the iterator is positioned at a "value node" (which is not something
|
// reminder: it.Leaf() == true when the iterator is positioned at a "value node" (which is not something
|
||||||
// that actually exists in an MMPT), therefore we pass the parent node blob as the leaf RLP.
|
// that actually exists in an MMPT), therefore we pass the parent node blob as the leaf RLP.
|
||||||
func (sdb *StateDiffBuilder) processStateValueNode(it trie.NodeIterator, parentBlob []byte) (*sdtypes.AccountWrapper, error) {
|
func (sdb *StateDiffBuilder) decodeStateLeaf(it trie.NodeIterator, parentBlob []byte) (*sdtypes.AccountWrapper, error) {
|
||||||
var account types.StateAccount
|
var account types.StateAccount
|
||||||
if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil {
|
if err := rlp.DecodeBytes(it.LeafBlob(), &account); err != nil {
|
||||||
return nil, fmt.Errorf("error decoding account at leaf key %x: %w", it.LeafKey(), err)
|
return nil, fmt.Errorf("error decoding account at leaf key %x: %w", it.LeafKey(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
leafKey := make([]byte, len(it.LeafKey()))
|
||||||
|
copy(leafKey, it.LeafKey())
|
||||||
return &sdtypes.AccountWrapper{
|
return &sdtypes.AccountWrapper{
|
||||||
LeafKey: it.LeafKey(),
|
LeafKey: it.LeafKey(),
|
||||||
Account: &account,
|
Account: &account,
|
||||||
@ -300,85 +318,23 @@ func (sdb *StateDiffBuilder) processStateValueNode(it trie.NodeIterator, parentB
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
|
// processAccountUpdates emits state and storage node records for all updated existing accounts.
|
||||||
// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
|
func (sdb *StateDiffBuilder) processAccountUpdates(
|
||||||
// needs to be called before building account creations and deletions as this mutates
|
updates accountUpdateMap,
|
||||||
// those account maps to remove the accounts which were updated
|
nodeSink sdtypes.StateNodeSink,
|
||||||
func (sdb *StateDiffBuilder) buildAccountUpdates(creations, deletions sdtypes.AccountMap, updatedKeys []string,
|
ipldSink sdtypes.IPLDSink,
|
||||||
nodeSink sdtypes.StateNodeSink, ipldSink sdtypes.IPLDSink, logger log.Logger) error {
|
logger log.Logger,
|
||||||
logger.Trace("statediff BEGIN buildAccountUpdates",
|
) error {
|
||||||
"creations", len(creations), "deletions", len(deletions), "updated", len(updatedKeys))
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildAccountUpdatesTimer)
|
||||||
defer metrics.ReportAndUpdateDuration("statediff END buildAccountUpdates ",
|
|
||||||
time.Now(), logger, metrics.IndexerMetrics.BuildAccountUpdatesTimer)
|
|
||||||
var err error
|
|
||||||
for _, key := range updatedKeys {
|
|
||||||
createdAcc := creations[key]
|
|
||||||
deletedAcc := deletions[key]
|
|
||||||
var storageDiff []sdtypes.StorageLeafNode
|
|
||||||
if deletedAcc.Account != nil && createdAcc.Account != nil {
|
|
||||||
err = sdb.buildStorageNodesIncremental(
|
|
||||||
deletedAcc.Account.Root, createdAcc.Account.Root,
|
|
||||||
StorageNodeAppender(&storageDiff), ipldSink,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed building incremental storage diffs for account with leafkey %x\r\nerror: %w", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err = nodeSink(sdtypes.StateLeafNode{
|
|
||||||
AccountWrapper: createdAcc,
|
|
||||||
StorageDiff: storageDiff,
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
delete(creations, key)
|
|
||||||
delete(deletions, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
|
// processCreatedAccountStorage processes the storage node records for a newly created account
|
||||||
// it also returns the code and codehash for created contract accounts
|
// i.e. it returns all the storage nodes at this state, since there is no previous state.
|
||||||
func (sdb *StateDiffBuilder) buildAccountCreations(accounts sdtypes.AccountMap, nodeSink sdtypes.StateNodeSink,
|
func (sdb *StateDiffBuilder) processCreatedAccountStorage(
|
||||||
ipldSink sdtypes.IPLDSink, logger log.Logger) error {
|
sr common.Hash, storageSink sdtypes.StorageNodeSink, ipldSink sdtypes.IPLDSink,
|
||||||
logger.Trace("statediff BEGIN buildAccountCreations")
|
) error {
|
||||||
defer metrics.ReportAndUpdateDuration("statediff END buildAccountCreations",
|
|
||||||
time.Now(), logger, metrics.IndexerMetrics.BuildAccountCreationsTimer)
|
|
||||||
for _, val := range accounts {
|
|
||||||
diff := sdtypes.StateLeafNode{
|
|
||||||
AccountWrapper: val,
|
|
||||||
}
|
|
||||||
if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
|
|
||||||
// For contract creations, any storage node contained is a diff
|
|
||||||
err := sdb.buildStorageNodesEventual(val.Account.Root, StorageNodeAppender(&diff.StorageDiff), ipldSink)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed building eventual storage diffs for node with leaf key %x\r\nerror: %w", val.LeafKey, err)
|
|
||||||
}
|
|
||||||
// emit codehash => code mappings for contract
|
|
||||||
codeHash := common.BytesToHash(val.Account.CodeHash)
|
|
||||||
code, err := sdb.stateCache.ContractCode(codeHash)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %w", codeHash, err)
|
|
||||||
}
|
|
||||||
if err := ipldSink(sdtypes.IPLD{
|
|
||||||
CID: ipld.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes()).String(),
|
|
||||||
Content: code,
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := nodeSink(diff); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildStorageNodesEventual builds the storage diff node objects for a created account
|
|
||||||
// i.e. it returns all the storage nodes at this state, since there is no previous state
|
|
||||||
func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, storageSink sdtypes.StorageNodeSink,
|
|
||||||
ipldSink sdtypes.IPLDSink) error {
|
|
||||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildStorageNodesEventualTimer)
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildStorageNodesEventualTimer)
|
||||||
if sr == emptyContractRoot {
|
if sr == emptyContractRoot {
|
||||||
return nil
|
return nil
|
||||||
@ -390,11 +346,11 @@ func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, storageSi
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
it := sTrie.NodeIterator(make([]byte, 0))
|
|
||||||
var prevBlob []byte
|
var prevBlob []byte
|
||||||
|
it := sTrie.NodeIterator(make([]byte, 0))
|
||||||
for it.Next(true) {
|
for it.Next(true) {
|
||||||
if it.Leaf() {
|
if it.Leaf() {
|
||||||
storageLeafNode := sdb.processStorageValueNode(it, prevBlob)
|
storageLeafNode := sdb.decodeStorageLeaf(it, prevBlob)
|
||||||
if err := storageSink(storageLeafNode); err != nil {
|
if err := storageSink(storageLeafNode); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -413,10 +369,10 @@ func (sdb *StateDiffBuilder) buildStorageNodesEventual(sr common.Hash, storageSi
|
|||||||
return it.Error()
|
return it.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
// decodes account at leaf and encodes RLP data to CID
|
// decodes slot at leaf and encodes RLP data to CID
|
||||||
// reminder: it.Leaf() == true when the iterator is positioned at a "value node" (which is not something
|
// reminder: it.Leaf() == true when the iterator is positioned at a "value node" (which is not something
|
||||||
// that actually exists in an MMPT), therefore we pass the parent node blob as the leaf RLP.
|
// that actually exists in an MMPT), therefore we pass the parent node blob as the leaf RLP.
|
||||||
func (sdb *StateDiffBuilder) processStorageValueNode(it trie.NodeIterator, parentBlob []byte) sdtypes.StorageLeafNode {
|
func (sdb *StateDiffBuilder) decodeStorageLeaf(it trie.NodeIterator, parentBlob []byte) sdtypes.StorageLeafNode {
|
||||||
leafKey := make([]byte, len(it.LeafKey()))
|
leafKey := make([]byte, len(it.LeafKey()))
|
||||||
copy(leafKey, it.LeafKey())
|
copy(leafKey, it.LeafKey())
|
||||||
value := make([]byte, len(it.LeafBlob()))
|
value := make([]byte, len(it.LeafBlob()))
|
||||||
@ -429,8 +385,10 @@ func (sdb *StateDiffBuilder) processStorageValueNode(it trie.NodeIterator, paren
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account
|
// processRemovedAccountStorage builds the "removed" diffs for all the storage nodes for a destroyed account
|
||||||
func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, storageSink sdtypes.StorageNodeSink) error {
|
func (sdb *StateDiffBuilder) processRemovedAccountStorage(
|
||||||
|
sr common.Hash, storageSink sdtypes.StorageNodeSink,
|
||||||
|
) error {
|
||||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildRemovedAccountStorageNodesTimer)
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildRemovedAccountStorageNodesTimer)
|
||||||
if sr == emptyContractRoot {
|
if sr == emptyContractRoot {
|
||||||
return nil
|
return nil
|
||||||
@ -442,13 +400,6 @@ func (sdb *StateDiffBuilder) buildRemovedAccountStorageNodes(sr common.Hash, sto
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
it := sTrie.NodeIterator(nil)
|
it := sTrie.NodeIterator(nil)
|
||||||
return sdb.buildRemovedStorageNodesFromTrie(it, storageSink)
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator
|
|
||||||
func (sdb *StateDiffBuilder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, storageSink sdtypes.StorageNodeSink) error {
|
|
||||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildRemovedStorageNodesFromTrieTimer)
|
|
||||||
// it := trie.NewIterator(it)
|
|
||||||
for it.Next(true) {
|
for it.Next(true) {
|
||||||
if it.Leaf() { // only leaf values are indexed, don't need to demarcate removed intermediate nodes
|
if it.Leaf() { // only leaf values are indexed, don't need to demarcate removed intermediate nodes
|
||||||
leafKey := make([]byte, len(it.LeafKey()))
|
leafKey := make([]byte, len(it.LeafKey()))
|
||||||
@ -466,9 +417,12 @@ func (sdb *StateDiffBuilder) buildRemovedStorageNodesFromTrie(it trie.NodeIterat
|
|||||||
return it.Error()
|
return it.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
|
// processUpdatedAccountStorage builds the storage diff node objects for all nodes that exist in a different state at B than A
|
||||||
func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldroot common.Hash, newroot common.Hash, storageSink sdtypes.StorageNodeSink,
|
func (sdb *StateDiffBuilder) processUpdatedAccountStorage(
|
||||||
ipldSink sdtypes.IPLDSink) error {
|
oldroot common.Hash, newroot common.Hash,
|
||||||
|
storageSink sdtypes.StorageNodeSink,
|
||||||
|
ipldSink sdtypes.IPLDSink,
|
||||||
|
) error {
|
||||||
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildStorageNodesIncrementalTimer)
|
defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.BuildStorageNodesIncrementalTimer)
|
||||||
if newroot == oldroot {
|
if newroot == oldroot {
|
||||||
return nil
|
return nil
|
||||||
@ -483,8 +437,8 @@ func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldroot common.Hash, n
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
a, b := oldTrie.NodeIterator(nil), newTrie.NodeIterator(nil)
|
|
||||||
var prevBlob []byte
|
var prevBlob []byte
|
||||||
|
a, b := oldTrie.NodeIterator(nil), newTrie.NodeIterator(nil)
|
||||||
it, _ := utils.NewSymmetricDifferenceIterator(a, b)
|
it, _ := utils.NewSymmetricDifferenceIterator(a, b)
|
||||||
for it.Next(true) {
|
for it.Next(true) {
|
||||||
if it.FromA() {
|
if it.FromA() {
|
||||||
@ -503,7 +457,7 @@ func (sdb *StateDiffBuilder) buildStorageNodesIncremental(oldroot common.Hash, n
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if it.Leaf() {
|
if it.Leaf() {
|
||||||
storageLeafNode := sdb.processStorageValueNode(it, prevBlob)
|
storageLeafNode := sdb.decodeStorageLeaf(it, prevBlob)
|
||||||
if err := storageSink(storageLeafNode); err != nil {
|
if err := storageSink(storageLeafNode); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user