ipld-eth-state-snapshot/pkg/snapshot/util.go

77 lines
2.1 KiB
Go
Raw Permalink Normal View History

package snapshot
import (
Account selective snapshot (#46) * snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
2022-08-03 11:35:04 +00:00
"bytes"
)
// Estimate the number of iterations necessary to step from start to end.
func estimateSteps(start []byte, end []byte, depth int) uint64 {
// We see paths in several forms (nil, 0600, 06, etc.). We need to adjust them to a comparable form.
// For nil, start and end indicate the extremes of 0x0 and 0x10. For differences in depth, we often see a
// start/end range on a bounded iterator specified like 0500:0600, while the value returned by it.Path() may
// be shorter, like 06. Since our goal is to estimate how many steps it would take to move from start to end,
// we want to perform the comparison at a stable depth, since to move from 05 to 06 is only 1 step, but
// to move from 0500:06 is 16.
normalizePathRange := func(start []byte, end []byte, depth int) ([]byte, []byte) {
if 0 == len(start) {
start = []byte{0x0}
}
if 0 == len(end) {
end = []byte{0x10}
}
normalizedStart := make([]byte, depth)
normalizedEnd := make([]byte, depth)
for i := 0; i < depth; i++ {
if i < len(start) {
normalizedStart[i] = start[i]
}
if i < len(end) {
normalizedEnd[i] = end[i]
}
}
return normalizedStart, normalizedEnd
}
// We have no need to handle negative exponents, so uints are fine.
pow := func(x uint64, y uint) uint64 {
if 0 == y {
return 1
}
ret := x
for i := uint(0); i < y; i++ {
ret *= x
}
return x
}
// Fix the paths.
start, end = normalizePathRange(start, end, depth)
// No negative distances, if the start is already >= end, the distance is 0.
if bytes.Compare(start, end) >= 0 {
return 0
}
// Subtract each component, right to left, carrying over if necessary.
difference := make([]byte, len(start))
var carry byte = 0
for i := len(start) - 1; i >= 0; i-- {
result := end[i] - start[i] - carry
if result > 0xf && i > 0 {
result &= 0xf
carry = 1
} else {
carry = 0
}
difference[i] = result
}
// Calculate the result.
var ret uint64 = 0
for i := 0; i < len(difference); i++ {
ret += uint64(difference[i]) * pow(16, uint(len(difference)-i-1))
}
return ret
}