fix path rewind

decrement by epsilon, and go up to parent from child 0

iterator.go

@@ -40,9 +40,13 @@ func (it *PrefixBoundIterator) Next(descend bool) bool {
 	if !it.NodeIterator.Next(descend) {
 		return false
 	}
-	// stop if underlying iterator went past upper bound
+	// Stop if underlying iterator went past upper bound.
-	cmp := bytes.Compare(it.Path(), it.EndPath)
+	// Note: this results in a single node of overlap between binned iterators. The more correct
-	return cmp <= 0
+	// behavior would be to make this a strict less-than, so that iterators cover mutually disjoint
+	// subtries. Unfortunately, the NodeIterator constructor takes a compact path, meaning
+	// odd-length paths must be padded with a 0, so e.g. [8] becomes [8, 0], which means we would
+	// skip [8]. So, we use <= here to cover that node for the "next" bin.
+	return bytes.Compare(it.Path(), it.EndPath) <= 0
 }
 
 // NewPrefixBoundIterator returns an iterator with an upper bound value (hex path prefix)

iterator_test.go

@@ -87,12 +87,13 @@ func TestIterator(t *testing.T) {
 			for b := uint(0); b < nbins; b++ {
 				for it := iters[b]; it.Next(true); ix++ {
 					if !bytes.Equal(allPaths[ix], it.Path()) {
-						t.Fatalf("wrong path value\nexpected:\t%v\nactual:\t\t%v",
+						t.Fatalf("wrong path value in bin %d (index %d)\nexpected:\t%v\nactual:\t\t%v",
-							allPaths[ix], it.Path())
+							b, ix, allPaths[ix], it.Path())
 					}
 				}
-				// if the last node path was even-length, it will be duplicated
+				// if the last node path for the previous bin was even-length, the next iterator
-				if len(allPaths[ix-1])&0b1 == 0 {
+				// will seek to the same node and it will be duplicated (see comment in Next()).
+				if len(allPaths[ix-1])&1 == 0 {
 					ix--
 				}
 			}

tracker/tracker.go

@@ -46,8 +46,8 @@ func (tr *Tracker) CaptureSignal(cancelCtx context.CancelFunc) {
 	go func() {
 		sig := <-sigChan
 		log.Error("Signal received (%v), stopping", "signal", sig)
-		// cancel context on receiving a signal
+		// Cancel context on receiving a signal. On cancellation, all tracked iterators complete
-		// on ctx cancellation, all the iterators complete processing of their current node before stopping
+		// processing of their current node before stopping.
 		cancelCtx()
 	}()
 }
@@ -90,8 +90,9 @@ func (tr *Tracker) dump() error {
 	return out.WriteAll(rows)
 }
 
-// Restore attempts to read iterator state from file
+// Restore attempts to read iterator state from the recovery file.
-// if file doesn't exist, returns an empty slice with no error
+// If the file doesn't exist, returns an empty slice with no error.
+// Restored iterators are constructed in the same order as in the returned slice.
 func (tr *Tracker) Restore(makeIterator iter.IteratorConstructor) ([]trie.NodeIterator, error) {
 	file, err := os.Open(tr.recoveryFile)
 	if err != nil {
@@ -128,10 +129,9 @@ func (tr *Tracker) Restore(makeIterator iter.IteratorConstructor) ([]trie.NodeIt
 		}
 
 		// force the lower bound path to an even length (required by geth API/HexToKeyBytes)
-		if len(recoveredPath)&0b1 == 1 {
+		if len(recoveredPath)&1 == 1 {
-			// decrement first to avoid skipped nodes
+			// to avoid skipped nodes, we must rewind by one index
-			decrementPath(recoveredPath)
+			recoveredPath = rewindPath(recoveredPath)
-			recoveredPath = append(recoveredPath, 0)
 		}
 		it := makeIterator(iter.HexToKeyBytes(recoveredPath))
 		boundIt := iter.NewPrefixBoundIterator(it, endPath)
@@ -184,25 +184,22 @@ func (it *Iterator) Next(descend bool) bool {
 	return ret
 }
 
-// Subtracts 1 from the last byte in a path slice, carrying if needed.
+// Rewinds to the path of the previous (pre-order) node:
-// Does nothing, returning false, for all-zero inputs.
+// If the last byte of the path is zero, pops it. Otherwise, decrements it
-func decrementPath(path []byte) bool {
+// and pads with 0xF to 64 bytes (e.g. [1] => [0 f f f ...]).
-	// check for all zeros
+// Returns the passed path (which is also modified in place)
-	allzero := true
+func rewindPath(path []byte) []byte {
-	for i := 0; i < len(path); i++ {
+	if len(path) == 0 {
-		allzero = allzero && path[i] == 0
+		return path
 	}
-	if allzero {
+	if path[len(path)-1] == 0 {
-		return false
+		return path[:len(path)-1]
 	}
-	for i := len(path) - 1; i >= 0; i-- {
+	path[len(path)-1]--
-		val := path[i]
+	padded := make([]byte, 64)
-		path[i]--
+	i := copy(padded, path)
-		if val == 0 {
+	for ; i < len(padded); i++ {
-			path[i] = 0xf
+		padded[i] = 0xf
-		} else {
-			return true
-		}
 	}
-	return true
+	return padded
 }