From 6d20ef7c3b810d34c5a4a3f62053a7813f4a9d8c Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 23 Aug 2020 19:23:16 -0500 Subject: [PATCH 01/12] iterator package * stripped down Nodeiterator interface * prefix-bound iterator for traversing a specific range of keys * visitor over subtries divided at a specific depth --- go.mod | 7 +++ pkg/iterator/iterator.go | 115 +++++++++++++++++++++++++++++++++++++++ pkg/iterator/util.go | 49 +++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 go.mod create mode 100644 pkg/iterator/iterator.go create mode 100644 pkg/iterator/util.go diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..1e656f4a7 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/vulcanize/go-eth-state-node-iterator + +go 1.13 + +require github.com/ethereum/go-ethereum v1.9.11 + +replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.2 diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go new file mode 100644 index 000000000..d68f55535 --- /dev/null +++ b/pkg/iterator/iterator.go @@ -0,0 +1,115 @@ +// +// Copyright © 2020 Vulcanize, Inc +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package iterator + +import ( + // "fmt" + "bytes" + + // "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" +) + +type NodeIterator interface { + Next(bool) bool + Error() error + Hash() common.Hash + Path() []byte + Leaf() bool +} + +type prefixBoundIterator struct { + current trie.NodeIterator + endKey []byte + // endHash common.Hash +} + +func (it *prefixBoundIterator) Next(descend bool) bool { + if it.endKey == nil { // todo: want? + return it.current.Next(descend) + } + cmp := bytes.Compare(it.current.Path(), it.endKey) // stop before endKey + if cmp >= 0 { + return false + } + return it.current.Next(descend) +} + +func (it *prefixBoundIterator) Error() error { + return it.current.Error() +} +func (it *prefixBoundIterator) Hash() common.Hash { + return it.current.Hash() +} +func (it *prefixBoundIterator) Path() []byte { + return it.current.Path() +} +func (it *prefixBoundIterator) Leaf() bool { + return it.current.Leaf() +} + +// Iterator with an upper bound value (hex path prefix) +func NewPrefixBoundIterator(it trie.NodeIterator, to []byte) NodeIterator { + return &prefixBoundIterator{current: it, endKey: to} +} + +// array of 0..f with prefix +func prefixedNibbles(prefix []byte) [][]byte { + var ret [][]byte + for i := byte(0); i < 16; i++ { + elem := make([]byte, len(prefix)) + copy(elem, prefix) + elem = append(elem, i) + ret = append(ret, elem) + } + return ret +} + +// Generates ordered cartesian product of all nibbles of given length, w/ optional prefix +// eg. MakePaths([4], 2) => [[4 0 0] [4 0 1] ... [4 f f]] +func MakePaths(prefix []byte, length int) [][]byte { + paths := [][]byte{prefix} + for depth := 0; depth < length; depth++ { + var newPaths [][]byte + for _, path := range paths { + for _, newPath := range prefixedNibbles(path) { + newPaths = append(newPaths, newPath) + } + } + paths = newPaths + } + return paths +} + +// Apply a function to 16^cutdepth subtries divided by path prefix +func VisitSubtries(tree state.Trie, cutDepth int, callback func(NodeIterator)) { + prefixes := MakePaths(nil, cutDepth) + // pre- and postpend nil to include root & tail + prefixes = append(prefixes, nil) + prefixes = append([][]byte{nil}, prefixes...) + + for i := 0; i < len(prefixes)-1; i++ { + key := prefixes[i] + if len(key) % 2 != 0 { + key = append(key, 0) + } + it := tree.NodeIterator(HexToKeyBytes(key)) + callback(NewPrefixBoundIterator(it, prefixes[i+1])) + } +} diff --git a/pkg/iterator/util.go b/pkg/iterator/util.go new file mode 100644 index 000000000..5dd6601a0 --- /dev/null +++ b/pkg/iterator/util.go @@ -0,0 +1,49 @@ +package iterator + +import ( + "bytes" + "github.com/ethereum/go-ethereum/trie" +) + +func CompareNodes(a, b trie.NodeIterator) int { + if cmp := bytes.Compare(a.Path(), b.Path()); cmp != 0 { + return cmp + } + if a.Leaf() && !b.Leaf() { + return -1 + } else if b.Leaf() && !a.Leaf() { + return 1 + } + if cmp := bytes.Compare(a.Hash().Bytes(), b.Hash().Bytes()); cmp != 0 { + return cmp + } + if a.Leaf() && b.Leaf() { + return bytes.Compare(a.LeafBlob(), b.LeafBlob()) + } + return 0 +} + +// hexToKeyBytes turns hex nibbles into key bytes. +// This can only be used for keys of even length. +func HexToKeyBytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] + } + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") + } + key := make([]byte, len(hex)/2) + decodeNibbles(hex, key) + return key +} + +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] + } +} + +// hasTerm returns whether a hex key has the terminator flag. +func hasTerm(s []byte) bool { + return len(s) > 0 && s[len(s)-1] == 16 +} From f8962c6c581791aa9cbe2ef8e97621ba2a116c9d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 23 Aug 2020 22:11:32 -0500 Subject: [PATCH 02/12] cleanup --- pkg/iterator/iterator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index d68f55535..d2418eec0 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" ) +// Simplified trie node iterator type NodeIterator interface { Next(bool) bool Error() error @@ -37,11 +38,10 @@ type NodeIterator interface { type prefixBoundIterator struct { current trie.NodeIterator endKey []byte - // endHash common.Hash } func (it *prefixBoundIterator) Next(descend bool) bool { - if it.endKey == nil { // todo: want? + if it.endKey == nil { return it.current.Next(descend) } cmp := bytes.Compare(it.current.Path(), it.endKey) // stop before endKey From c3c31959910a363315ade7dd122838a8934a3aa5 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 30 Aug 2020 21:59:15 -0500 Subject: [PATCH 03/12] fmt --- pkg/iterator/iterator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index d2418eec0..c21d23f34 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -21,9 +21,9 @@ import ( "bytes" // "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/trie" ) // Simplified trie node iterator @@ -37,7 +37,7 @@ type NodeIterator interface { type prefixBoundIterator struct { current trie.NodeIterator - endKey []byte + endKey []byte } func (it *prefixBoundIterator) Next(descend bool) bool { @@ -106,7 +106,7 @@ func VisitSubtries(tree state.Trie, cutDepth int, callback func(NodeIterator)) { for i := 0; i < len(prefixes)-1; i++ { key := prefixes[i] - if len(key) % 2 != 0 { + if len(key)%2 != 0 { // zero-pad for odd-length keys key = append(key, 0) } it := tree.NodeIterator(HexToKeyBytes(key)) From d82b49b8ba580144880abd77cdf4d0587ef14e35 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 31 Aug 2020 11:06:03 -0500 Subject: [PATCH 04/12] improve prefix gen use stateful iterator, and allow splitting into powers of 2 rather than 16 --- pkg/iterator/iterator.go | 83 ++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index c21d23f34..87f7db559 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -17,10 +17,9 @@ package iterator import ( - // "fmt" "bytes" + "math/bits" - // "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/trie" @@ -69,37 +68,73 @@ func NewPrefixBoundIterator(it trie.NodeIterator, to []byte) NodeIterator { return &prefixBoundIterator{current: it, endKey: to} } -// array of 0..f with prefix -func prefixedNibbles(prefix []byte) [][]byte { - var ret [][]byte - for i := byte(0); i < 16; i++ { - elem := make([]byte, len(prefix)) - copy(elem, prefix) - elem = append(elem, i) - ret = append(ret, elem) +type prefixGenerator struct { + current []byte + step byte + stepIndex uint +} + +func newPrefixGenerator(nbins uint) prefixGenerator { + if bits.OnesCount(nbins) != 1 { + panic("nbins must be a power of 2") + } + // determine step dist. and path index at which to step + var step byte + var stepIndex uint + for ; nbins != 0; stepIndex++ { + divisor := byte(nbins & 0xf) + if divisor != 0 { + step = 0x10 / divisor + } + nbins = nbins >> 4 + } + return prefixGenerator{ + current: make([]byte, stepIndex), + step: step, + stepIndex: stepIndex-1, + } +} + +func (gen *prefixGenerator) Value() []byte { + return gen.current +} + +func (gen *prefixGenerator) HasNext() bool { + return gen.current[0] <= 0xf +} + +func (gen *prefixGenerator) Next() { + gen.current[gen.stepIndex] += gen.step + overflow := false + for ix := 0; ix < len(gen.current); ix++ { + rix := len(gen.current)-1-ix // reverse + if overflow { + gen.current[rix]++ + overflow = false + } + if rix != 0 && gen.current[rix] > 0xf { + gen.current[rix] = 0 + overflow = true + } } - return ret } // Generates ordered cartesian product of all nibbles of given length, w/ optional prefix // eg. MakePaths([4], 2) => [[4 0 0] [4 0 1] ... [4 f f]] -func MakePaths(prefix []byte, length int) [][]byte { - paths := [][]byte{prefix} - for depth := 0; depth < length; depth++ { - var newPaths [][]byte - for _, path := range paths { - for _, newPath := range prefixedNibbles(path) { - newPaths = append(newPaths, newPath) - } - } - paths = newPaths +func MakePaths(prefix []byte, nbins uint) [][]byte { + var res [][]byte + for it := newPrefixGenerator(nbins); it.HasNext(); it.Next() { + next := make([]byte, len(prefix)) + copy(next, prefix) + next = append(next, it.Value()...) + res = append(res, next) } - return paths + return res } // Apply a function to 16^cutdepth subtries divided by path prefix -func VisitSubtries(tree state.Trie, cutDepth int, callback func(NodeIterator)) { - prefixes := MakePaths(nil, cutDepth) +func VisitSubtries(tree state.Trie, nbins uint, callback func(NodeIterator)) { + prefixes := MakePaths(nil, nbins) // pre- and postpend nil to include root & tail prefixes = append(prefixes, nil) prefixes = append([][]byte{nil}, prefixes...) From a3595be72b83fc002aa856bf5847cb0481434a11 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 31 Aug 2020 22:22:40 -0500 Subject: [PATCH 05/12] doc --- pkg/iterator/iterator.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index 87f7db559..850d91d29 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -119,8 +119,9 @@ func (gen *prefixGenerator) Next() { } } -// Generates ordered cartesian product of all nibbles of given length, w/ optional prefix -// eg. MakePaths([4], 2) => [[4 0 0] [4 0 1] ... [4 f f]] +// Generates paths that cut the trie domain into "nbins" bins, w/ optional prefix +// eg. MakePaths([], 2) => [[0] [8]] +// MakePaths([4], 32) => [[4 0 0] [4 0 8] [4 1 0]... [4 f 8]] func MakePaths(prefix []byte, nbins uint) [][]byte { var res [][]byte for it := newPrefixGenerator(nbins); it.HasNext(); it.Next() { @@ -132,7 +133,7 @@ func MakePaths(prefix []byte, nbins uint) [][]byte { return res } -// Apply a function to 16^cutdepth subtries divided by path prefix +// Apply a function to nbins subtries divided according to path prefix func VisitSubtries(tree state.Trie, nbins uint, callback func(NodeIterator)) { prefixes := MakePaths(nil, nbins) // pre- and postpend nil to include root & tail From 8b8f718505230aed30e1dd320d3e559e1dd7c5da Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 6 Sep 2020 02:28:36 -0500 Subject: [PATCH 06/12] fill nodeiterator interface --- pkg/iterator/iterator.go | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index 850d91d29..efb2d4d55 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -25,21 +25,12 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// Simplified trie node iterator -type NodeIterator interface { - Next(bool) bool - Error() error - Hash() common.Hash - Path() []byte - Leaf() bool -} - -type prefixBoundIterator struct { +type PrefixBoundIterator struct { current trie.NodeIterator endKey []byte } -func (it *prefixBoundIterator) Next(descend bool) bool { +func (it *PrefixBoundIterator) Next(descend bool) bool { if it.endKey == nil { return it.current.Next(descend) } @@ -50,22 +41,34 @@ func (it *prefixBoundIterator) Next(descend bool) bool { return it.current.Next(descend) } -func (it *prefixBoundIterator) Error() error { +func (it *PrefixBoundIterator) Error() error { return it.current.Error() } -func (it *prefixBoundIterator) Hash() common.Hash { +func (it *PrefixBoundIterator) Hash() common.Hash { return it.current.Hash() } -func (it *prefixBoundIterator) Path() []byte { +func (it *PrefixBoundIterator) Path() []byte { return it.current.Path() } -func (it *prefixBoundIterator) Leaf() bool { +func (it *PrefixBoundIterator) Leaf() bool { return it.current.Leaf() } +func (it *PrefixBoundIterator) LeafKey() []byte { + return it.current.LeafKey() +} +func (it *PrefixBoundIterator) LeafBlob() []byte { + return it.current.LeafBlob() +} +func (it *PrefixBoundIterator) LeafProof() [][]byte { + return it.current.LeafProof() +} +func (it *PrefixBoundIterator) Parent() common.Hash { + return it.current.Parent() +} // Iterator with an upper bound value (hex path prefix) -func NewPrefixBoundIterator(it trie.NodeIterator, to []byte) NodeIterator { - return &prefixBoundIterator{current: it, endKey: to} +func NewPrefixBoundIterator(it trie.NodeIterator, to []byte) *PrefixBoundIterator { + return &PrefixBoundIterator{current: it, endKey: to} } type prefixGenerator struct { From 70b374787a724c481a508c3e389db7c21f08f61a Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 3 Sep 2020 23:52:47 -0500 Subject: [PATCH 07/12] iterator slice instead of callback --- pkg/iterator/iterator.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index efb2d4d55..8e24800b7 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -122,7 +122,7 @@ func (gen *prefixGenerator) Next() { } } -// Generates paths that cut the trie domain into "nbins" bins, w/ optional prefix +// Generates paths that cut the trie domain into "nbins" uniform bins, w/ optional prefix // eg. MakePaths([], 2) => [[0] [8]] // MakePaths([4], 32) => [[4 0 0] [4 0 8] [4 1 0]... [4 f 8]] func MakePaths(prefix []byte, nbins uint) [][]byte { @@ -136,19 +136,21 @@ func MakePaths(prefix []byte, nbins uint) [][]byte { return res } -// Apply a function to nbins subtries divided according to path prefix -func VisitSubtries(tree state.Trie, nbins uint, callback func(NodeIterator)) { +// Cut a trie by path prefix, returning `nbins` iterators covering its subtries +func SubtrieIterators(tree state.Trie, nbins uint) []NodeIterator { prefixes := MakePaths(nil, nbins) // pre- and postpend nil to include root & tail prefixes = append(prefixes, nil) prefixes = append([][]byte{nil}, prefixes...) + var iters []NodeIterator for i := 0; i < len(prefixes)-1; i++ { key := prefixes[i] if len(key)%2 != 0 { // zero-pad for odd-length keys key = append(key, 0) } it := tree.NodeIterator(HexToKeyBytes(key)) - callback(NewPrefixBoundIterator(it, prefixes[i+1])) + iters = append(iters, NewPrefixBoundIterator(it, prefixes[i+1])) } + return iters } From deb8b6b71c4167bd907bb0976e58913bd5852e54 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 6 Sep 2020 02:30:02 -0500 Subject: [PATCH 08/12] refactor into slice generator, and factory --- pkg/iterator/iterator.go | 43 +++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index 8e24800b7..e4cc2f3f8 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -136,21 +136,50 @@ func MakePaths(prefix []byte, nbins uint) [][]byte { return res } -// Cut a trie by path prefix, returning `nbins` iterators covering its subtries -func SubtrieIterators(tree state.Trie, nbins uint) []NodeIterator { - prefixes := MakePaths(nil, nbins) +func eachPrefixRange(prefix []byte, nbins uint, callback func ([]byte, []byte)) { + prefixes := MakePaths(prefix, nbins) // pre- and postpend nil to include root & tail prefixes = append(prefixes, nil) prefixes = append([][]byte{nil}, prefixes...) - - var iters []NodeIterator for i := 0; i < len(prefixes)-1; i++ { key := prefixes[i] if len(key)%2 != 0 { // zero-pad for odd-length keys key = append(key, 0) } - it := tree.NodeIterator(HexToKeyBytes(key)) - iters = append(iters, NewPrefixBoundIterator(it, prefixes[i+1])) + callback(key, prefixes[i+1]) } +} + +// Cut a trie by path prefix, returning `nbins` iterators covering its subtries +func SubtrieIterators(tree state.Trie, nbins uint) []trie.NodeIterator { + var iters []trie.NodeIterator + eachPrefixRange(nil, nbins, func (key []byte, endKey []byte) { + it := tree.NodeIterator(HexToKeyBytes(key)) + iters = append(iters, NewPrefixBoundIterator(it, endKey)) + }) return iters } + +// Factory for per-bin subtrie iterators +type SubtrieIteratorFactory struct { + tree state.Trie + startKeys, endKeys [][]byte +} + +func (fac *SubtrieIteratorFactory) Length() int { return len(fac.startKeys) } + +func (fac *SubtrieIteratorFactory) IteratorAt(bin uint) *PrefixBoundIterator { + it := fac.tree.NodeIterator(HexToKeyBytes(fac.startKeys[bin])) + return NewPrefixBoundIterator(it, fac.endKeys[bin]) +} + +// Cut a trie by path prefix, returning `nbins` iterators covering its subtries +func NewSubtrieIteratorFactory(tree state.Trie, nbins uint) SubtrieIteratorFactory { + starts := make([][]byte, 0, nbins) + ends := make([][]byte, 0, nbins) + eachPrefixRange(nil, nbins, func (key []byte, endKey []byte) { + starts = append(starts, key) + ends = append(ends, endKey) + }) + return SubtrieIteratorFactory{tree: tree, startKeys: starts, endKeys: ends} +} From 321f5fe368eb5b2cfc5079e2da38a02d0f02d330 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 6 Sep 2020 12:52:16 -0500 Subject: [PATCH 09/12] don't prepend (nil, 0], make first bin start at nil --- pkg/iterator/iterator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index e4cc2f3f8..e2857aef1 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -138,9 +138,8 @@ func MakePaths(prefix []byte, nbins uint) [][]byte { func eachPrefixRange(prefix []byte, nbins uint, callback func ([]byte, []byte)) { prefixes := MakePaths(prefix, nbins) - // pre- and postpend nil to include root & tail - prefixes = append(prefixes, nil) - prefixes = append([][]byte{nil}, prefixes...) + prefixes = append(prefixes, nil) // include tail + prefixes[0] = nil // set bin 0 left bound to nil to include root for i := 0; i < len(prefixes)-1; i++ { key := prefixes[i] if len(key)%2 != 0 { // zero-pad for odd-length keys From 3715d871612ef071dd54377fd923a20c7060e127 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 6 Sep 2020 02:31:03 -0500 Subject: [PATCH 10/12] Fix termination behavior Boundary must be (a, b] to conform to NodeIterator behavior (stop _at_ next valid key rather than before) --- pkg/iterator/iterator.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index e2857aef1..045f4480c 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -34,11 +34,13 @@ func (it *PrefixBoundIterator) Next(descend bool) bool { if it.endKey == nil { return it.current.Next(descend) } - cmp := bytes.Compare(it.current.Path(), it.endKey) // stop before endKey - if cmp >= 0 { + if !it.current.Next(descend) { return false } - return it.current.Next(descend) + p := it.current.Path(); if len(p) >= 4 { p = p[:4] } + // stop if underlying iterator went past endKey + cmp := bytes.Compare(it.current.Path(), it.endKey) + return cmp <= 0 } func (it *PrefixBoundIterator) Error() error { From fb1768b7a9f77ec64ef98260186277497a949f6e Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 6 Sep 2020 12:55:48 -0500 Subject: [PATCH 11/12] go fmt, cleanup --- pkg/iterator/iterator.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/pkg/iterator/iterator.go b/pkg/iterator/iterator.go index 045f4480c..0cc94bc75 100644 --- a/pkg/iterator/iterator.go +++ b/pkg/iterator/iterator.go @@ -37,7 +37,6 @@ func (it *PrefixBoundIterator) Next(descend bool) bool { if !it.current.Next(descend) { return false } - p := it.current.Path(); if len(p) >= 4 { p = p[:4] } // stop if underlying iterator went past endKey cmp := bytes.Compare(it.current.Path(), it.endKey) return cmp <= 0 @@ -61,7 +60,7 @@ func (it *PrefixBoundIterator) LeafKey() []byte { func (it *PrefixBoundIterator) LeafBlob() []byte { return it.current.LeafBlob() } -func (it *PrefixBoundIterator) LeafProof() [][]byte { +func (it *PrefixBoundIterator) LeafProof() [][]byte { return it.current.LeafProof() } func (it *PrefixBoundIterator) Parent() common.Hash { @@ -74,8 +73,8 @@ func NewPrefixBoundIterator(it trie.NodeIterator, to []byte) *PrefixBoundIterato } type prefixGenerator struct { - current []byte - step byte + current []byte + step byte stepIndex uint } @@ -94,9 +93,9 @@ func newPrefixGenerator(nbins uint) prefixGenerator { nbins = nbins >> 4 } return prefixGenerator{ - current: make([]byte, stepIndex), - step: step, - stepIndex: stepIndex-1, + current: make([]byte, stepIndex), + step: step, + stepIndex: stepIndex - 1, } } @@ -112,7 +111,7 @@ func (gen *prefixGenerator) Next() { gen.current[gen.stepIndex] += gen.step overflow := false for ix := 0; ix < len(gen.current); ix++ { - rix := len(gen.current)-1-ix // reverse + rix := len(gen.current) - 1 - ix // reverse if overflow { gen.current[rix]++ overflow = false @@ -124,7 +123,7 @@ func (gen *prefixGenerator) Next() { } } -// Generates paths that cut the trie domain into "nbins" uniform bins, w/ optional prefix +// Generates paths that cut trie domain into "nbins" uniform conterminous bins (w/ opt. prefix) // eg. MakePaths([], 2) => [[0] [8]] // MakePaths([4], 32) => [[4 0 0] [4 0 8] [4 1 0]... [4 f 8]] func MakePaths(prefix []byte, nbins uint) [][]byte { @@ -138,13 +137,13 @@ func MakePaths(prefix []byte, nbins uint) [][]byte { return res } -func eachPrefixRange(prefix []byte, nbins uint, callback func ([]byte, []byte)) { +func eachPrefixRange(prefix []byte, nbins uint, callback func([]byte, []byte)) { prefixes := MakePaths(prefix, nbins) prefixes = append(prefixes, nil) // include tail - prefixes[0] = nil // set bin 0 left bound to nil to include root + prefixes[0] = nil // set bin 0 left bound to nil to include root for i := 0; i < len(prefixes)-1; i++ { key := prefixes[i] - if len(key)%2 != 0 { // zero-pad for odd-length keys + if len(key)%2 != 0 { // zero-pad for odd-length keys key = append(key, 0) } callback(key, prefixes[i+1]) @@ -154,7 +153,7 @@ func eachPrefixRange(prefix []byte, nbins uint, callback func ([]byte, []byte)) // Cut a trie by path prefix, returning `nbins` iterators covering its subtries func SubtrieIterators(tree state.Trie, nbins uint) []trie.NodeIterator { var iters []trie.NodeIterator - eachPrefixRange(nil, nbins, func (key []byte, endKey []byte) { + eachPrefixRange(nil, nbins, func(key []byte, endKey []byte) { it := tree.NodeIterator(HexToKeyBytes(key)) iters = append(iters, NewPrefixBoundIterator(it, endKey)) }) @@ -163,7 +162,7 @@ func SubtrieIterators(tree state.Trie, nbins uint) []trie.NodeIterator { // Factory for per-bin subtrie iterators type SubtrieIteratorFactory struct { - tree state.Trie + tree state.Trie startKeys, endKeys [][]byte } @@ -178,7 +177,7 @@ func (fac *SubtrieIteratorFactory) IteratorAt(bin uint) *PrefixBoundIterator { func NewSubtrieIteratorFactory(tree state.Trie, nbins uint) SubtrieIteratorFactory { starts := make([][]byte, 0, nbins) ends := make([][]byte, 0, nbins) - eachPrefixRange(nil, nbins, func (key []byte, endKey []byte) { + eachPrefixRange(nil, nbins, func(key []byte, endKey []byte) { starts = append(starts, key) ends = append(ends, endKey) }) From c6a1534e7f9d6bbd1c09911b0fff3c479421ec58 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Sun, 6 Sep 2020 12:59:32 -0500 Subject: [PATCH 12/12] move to root path --- pkg/iterator/iterator.go => iterator.go | 2 +- pkg/iterator/util.go => util.go | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename pkg/iterator/iterator.go => iterator.go (98%) rename pkg/iterator/util.go => util.go (100%) diff --git a/pkg/iterator/iterator.go b/iterator.go similarity index 98% rename from pkg/iterator/iterator.go rename to iterator.go index 0cc94bc75..08c975d98 100644 --- a/pkg/iterator/iterator.go +++ b/iterator.go @@ -123,7 +123,7 @@ func (gen *prefixGenerator) Next() { } } -// Generates paths that cut trie domain into "nbins" uniform conterminous bins (w/ opt. prefix) +// Generates paths that cut trie domain into `nbins` uniform conterminous bins (w/ opt. prefix) // eg. MakePaths([], 2) => [[0] [8]] // MakePaths([4], 32) => [[4 0 0] [4 0 8] [4 1 0]... [4 f 8]] func MakePaths(prefix []byte, nbins uint) [][]byte { diff --git a/pkg/iterator/util.go b/util.go similarity index 100% rename from pkg/iterator/util.go rename to util.go