ipld-eth-state-snapshot/pkg/snapshot/util.go

89 lines
2.3 KiB
Go
Raw Normal View History

package snapshot
import (
Account selective snapshot (#46) * snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
2022-08-03 11:35:04 +00:00
"bytes"
"context"
"fmt"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/prom"
file "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/file"
pg "github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot/pg"
snapt "github.com/vulcanize/ipld-eth-state-snapshot/pkg/types"
)
func NewPublisher(mode SnapshotMode, config *Config) (snapt.Publisher, error) {
switch mode {
case PgSnapshot:
driver, err := postgres.NewPGXDriver(context.Background(), config.DB.ConnConfig, config.Eth.NodeInfo)
if err != nil {
return nil, err
}
prom.RegisterDBCollector(config.DB.ConnConfig.DatabaseName, driver)
return pg.NewPublisher(postgres.NewPostgresDB(driver)), nil
case FileSnapshot:
return file.NewPublisher(config.File.OutputDir, config.Eth.NodeInfo)
}
return nil, fmt.Errorf("invalid snapshot mode: %s", mode)
}
// Subtracts 1 from the last byte in a path slice, carrying if needed.
// Does nothing, returning false, for all-zero inputs.
func decrementPath(path []byte) bool {
// check for all zeros
allzero := true
for i := 0; i < len(path); i++ {
allzero = allzero && path[i] == 0
}
if allzero {
return false
}
for i := len(path) - 1; i >= 0; i-- {
val := path[i]
path[i]--
if val == 0 {
path[i] = 0xf
} else {
return true
}
}
return true
}
Account selective snapshot (#46) * snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
2022-08-03 11:35:04 +00:00
// https://github.com/ethereum/go-ethereum/blob/master/trie/encoding.go#L97
func keybytesToHex(str []byte) []byte {
l := len(str)*2 + 1
var nibbles = make([]byte, l)
for i, b := range str {
nibbles[i*2] = b / 16
nibbles[i*2+1] = b % 16
}
nibbles[l-1] = 16
return nibbles
}
func updateSeekedPath(seekedPath *[]byte, nodePath []byte) {
// assumes len(nodePath) <= max len(*seekedPath)
*seekedPath = (*seekedPath)[:len(nodePath)]
copy(*seekedPath, nodePath)
}
// checks that the provided node path is before the end path
func checkUpperPathBound(nodePath, endPath []byte) bool {
// every path is before nil endPath
if endPath == nil {
return true
}
if len(endPath)%2 == 0 {
// in case of even length endpath
// apply open interval filter since the node at endpath will be covered by the next iterator
return bytes.Compare(nodePath, endPath) < 0
}
return bytes.Compare(nodePath, endPath) <= 0
}