deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

update version; minor refactoring

Browse Source
This commit is contained in:
Ian Norden
2020-05-04 12:14:57 -05:00
parent 9b0ded692b
commit dfb66eb67b
16 changed files with 127 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/storage/backfiller.go
+2 -1
View File
@@ -27,6 +27,7 @@ import (
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/utilities"
)
const (
@@ -63,7 +64,7 @@ func (bf *backFiller) BackFill(startingBlock, endingBlock uint64, backFill chan
logrus.Infof("going to fill in gap from %d to %d", startingBlock, endingBlock)
// break the range up into bins of smaller ranges
blockRangeBins, err := utils.GetBlockHeightBins(startingBlock, endingBlock, bf.batchSize)
blockRangeBins, err := utilities.GetBlockHeightBins(startingBlock, endingBlock, bf.batchSize)
if err != nil {
return err
}
libraries/shared/storage/utils/bins.go
-72
View File
@@ -1,72 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package utils
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
func GetBlockHeightBins(startingBlock, endingBlock, batchSize uint64) ([][]uint64, error) {
if endingBlock < startingBlock {
return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
}
if batchSize == 0 {
return nil, errors.New("backfill: batchsize needs to be greater than zero")
}
length := endingBlock - startingBlock + 1
numberOfBins := length / batchSize
remainder := length % batchSize
if remainder != 0 {
numberOfBins++
}
blockRangeBins := make([][]uint64, numberOfBins)
for i := range blockRangeBins {
nextBinStart := startingBlock + batchSize
blockRange := make([]uint64, 0, nextBinStart-startingBlock+1)
for j := startingBlock; j < nextBinStart && j <= endingBlock; j++ {
blockRange = append(blockRange, j)
}
startingBlock = nextBinStart
blockRangeBins[i] = blockRange
}
return blockRangeBins, nil
}
func MissingHeightsToGaps(heights []uint64) []shared.Gap {
validationGaps := make([]shared.Gap, 0)
start := heights[0]
lastHeight := start
for i, height := range heights[1:] {
if height != lastHeight+1 {
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: lastHeight,
})
start = height
}
if i+2 == len(heights) {
validationGaps = append(validationGaps, shared.Gap{
Start: start,
Stop: height,
})
}
lastHeight = height
}
return validationGaps
}
libraries/shared/storage/utils/bins_test.go
-74
View File
@@ -1,74 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package utils_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
var _ = Describe("GetBlockHeightBins", func() {
It("splits a block range up into bins", func() {
var startingBlock uint64 = 1
var endingBlock uint64 = 10101
var batchSize uint64 = 100
blockRangeBins, err := utils.GetBlockHeightBins(startingBlock, endingBlock, batchSize)
Expect(err).ToNot(HaveOccurred())
Expect(len(blockRangeBins)).To(Equal(102))
Expect(blockRangeBins[101]).To(Equal([]uint64{10101}))
startingBlock = 101
endingBlock = 10100
batchSize = 100
lastBin := make([]uint64, 0)
for i := 10001; i <= 10100; i++ {
lastBin = append(lastBin, uint64(i))
}
blockRangeBins, err = utils.GetBlockHeightBins(startingBlock, endingBlock, batchSize)
Expect(err).ToNot(HaveOccurred())
Expect(len(blockRangeBins)).To(Equal(100))
Expect(blockRangeBins[99]).To(Equal(lastBin))
startingBlock = 1
endingBlock = 1
batchSize = 100
blockRangeBins, err = utils.GetBlockHeightBins(startingBlock, endingBlock, batchSize)
Expect(err).ToNot(HaveOccurred())
Expect(len(blockRangeBins)).To(Equal(1))
Expect(blockRangeBins[0]).To(Equal([]uint64{1}))
})
It("throws an error if the starting block is higher than the ending block", func() {
var startingBlock uint64 = 10102
var endingBlock uint64 = 10101
var batchSize uint64 = 100
_, err := utils.GetBlockHeightBins(startingBlock, endingBlock, batchSize)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("ending block number needs to be greater than starting block number"))
})
It("throws an error if the batch size is zero", func() {
var startingBlock uint64 = 1
var endingBlock uint64 = 10101
var batchSize uint64 = 0
_, err := utils.GetBlockHeightBins(startingBlock, endingBlock, batchSize)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("batchsize needs to be greater than zero"))
})
})