deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

update version; minor refactoring

Browse Source
This commit is contained in:
Ian Norden
2020-05-04 12:14:57 -05:00
parent 9b0ded692b
commit dfb66eb67b
16 changed files with 127 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/storage/backfiller.go
+2 -1
View File
@@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
)
const (
@@ -63,7 +64,7 @@ func (bf *backFiller) BackFill(startingBlock, endingBlock uint64, backFill chan
logrus.Infof("going to fill in gap from %d to %d", startingBlock, endingBlock)
// break the range up into bins of smaller ranges
blockRangeBins, err := utils.GetBlockHeightBins(startingBlock, endingBlock, bf.batchSize)
blockRangeBins, err := utilities.GetBlockHeightBins(startingBlock, endingBlock, bf.batchSize)
if err != nil {
return err
}
libraries/shared/storage/utils/bins.go
-72
View File
@@ -1,72 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package utils
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
func GetBlockHeightBins(startingBlock, endingBlock, batchSize uint64) ([][]uint64, error) {
if endingBlock < startingBlock {
return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
}
if batchSize == 0 {
return nil, errors.New("backfill: batchsize needs to be greater than zero")
}
length := endingBlock - startingBlock + 1
numberOfBins := length / batchSize
remainder := length % batchSize
if remainder != 0 {
numberOfBins++
}
blockRangeBins := make([][]uint64, numberOfBins)
for i := range blockRangeBins {
nextBinStart := startingBlock + batchSize
blockRange := make([]uint64, 0, nextBinStart-startingBlock+1)
for j := startingBlock; j < nextBinStart && j <= endingBlock; j++ {
blockRange = append(blockRange, j)
}
startingBlock = nextBinStart
blockRangeBins[i] = blockRange
}
return blockRangeBins, nil
}
func MissingHeightsToGaps(heights []uint64) []shared.Gap {
validationGaps := make([]shared.Gap, 0)
start := heights[0]
lastHeight := start
for i, height := range heights[1:] {
if height != lastHeight+1 {
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: lastHeight,
})
start = height
}
if i+2 == len(heights) {
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: height,
})
}
lastHeight = height
}
return validationGaps
}
libraries/shared/utilities/utilities_suite_test.go
+36
View File
@@ -0,0 +1,36 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package utilities_test
import (
"io/ioutil"
"testing"
"github.com/sirupsen/logrus"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Utilities Suite")
}
var _ = BeforeSuite(func() {
logrus.SetOutput(ioutil.Discard)
})
libraries/shared/utilities/utils.go
+57 -4
View File
@@ -16,9 +16,62 @@
package utilities
func NullToZero(str string) string {
if str == "" {
return "0"
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
// GetBlockHeightBins splits a block range up into bins of block heights of the given batch size
func GetBlockHeightBins(startingBlock, endingBlock, batchSize uint64) ([][]uint64, error) {
if endingBlock < startingBlock {
return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
}
return str
if batchSize == 0 {
return nil, errors.New("backfill: batchsize needs to be greater than zero")
}
length := endingBlock - startingBlock + 1
numberOfBins := length / batchSize
remainder := length % batchSize
if remainder != 0 {
numberOfBins++
}
blockRangeBins := make([][]uint64, numberOfBins)
for i := range blockRangeBins {
nextBinStart := startingBlock + batchSize
blockRange := make([]uint64, 0, nextBinStart-startingBlock+1)
for j := startingBlock; j < nextBinStart && j <= endingBlock; j++ {
blockRange = append(blockRange, j)
}
startingBlock = nextBinStart
blockRangeBins[i] = blockRange
}
return blockRangeBins, nil
}
// MissingHeightsToGaps returns a slice of gaps from a slice of missing block heights
func MissingHeightsToGaps(heights []uint64) []shared.Gap {
if len(heights) == 0 {
return nil
}
validationGaps := make([]shared.Gap, 0)
start := heights[0]
lastHeight := start
for i, height := range heights[1:] {
if height != lastHeight+1 {
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: lastHeight,
})
start = height
}
if i+2 == len(heights) {
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: height,
})
}
lastHeight = height
}
return validationGaps
}
libraries/shared/storage/utils/bins_test.go → libraries/shared/utilities/utils_test.go
+2 -2
View File
@@ -14,13 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package utils_test
package utilities_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
utils "github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
)
var _ = Describe("GetBlockHeightBins", func() {