cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

split backfill range up into smaller bins and process them concurrently; improve tests; review fixes

Browse Source
This commit is contained in:
Ian Norden
2019-12-02 11:20:49 -06:00
parent 563832422c
commit 8562abd180
18 changed files with 744 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/storage/backfill.go
-87
View File
@@ -1,87 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
// BackFiller is the backfilling interface
type BackFiller interface {
BackFill(startingBlock, endingBlock uint64) ([]utils.StorageDiff, error)
}
// backFiller is the backfilling struct
type backFiller struct {
fetcher fetcher.StateDiffFetcher
}
// NewStorageBackFiller returns a BackFiller
func NewStorageBackFiller(fetcher fetcher.StateDiffFetcher) BackFiller {
return &backFiller{
fetcher: fetcher,
}
}
// BackFill uses the provided config to fetch and return the state diff at the specified blocknumber
// StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
func (bf *backFiller) BackFill(startingBlock, endingBlock uint64) ([]utils.StorageDiff, error) {
results := make([]utils.StorageDiff, 0)
if endingBlock < startingBlock {
return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
}
blockHeights := make([]uint64, 0, endingBlock-startingBlock+1)
for i := startingBlock; i <= endingBlock; i++ {
blockHeights = append(blockHeights, i)
}
payloads, err := bf.fetcher.FetchStateDiffsAt(blockHeights)
if err != nil {
return nil, err
}
for _, payload := range payloads {
stateDiff := new(statediff.StateDiff)
stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff)
if stateDiffDecodeErr != nil {
return nil, stateDiffDecodeErr
}
accounts := utils.GetAccountsFromDiff(*stateDiff)
for _, account := range accounts {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
for _, storage := range account.Storage {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
return nil, formatErr
}
logrus.Trace("adding storage diff to results",
"keccak of address: ", diff.HashedAddress.Hex(),
"block height: ", diff.BlockHeight,
"storage key: ", diff.StorageKey.Hex(),
"storage value: ", diff.StorageValue.Hex())
results = append(results, diff)
}
}
}
return results, nil
}
libraries/shared/storage/backfill_test.go
-75
View File
@@ -1,75 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_test
import (
"bytes"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
)
var _ = Describe("BackFiller", func() {
Describe("BackFill", func() {
var (
fetcher *mocks.StateDiffFetcher
backFiller storage.BackFiller
)
BeforeEach(func() {
fetcher = new(mocks.StateDiffFetcher)
fetcher.SetPayloadsToReturn(map[uint64]*statediff.Payload{
test_data.BlockNumber.Uint64(): &test_data.MockStatediffPayload,
test_data.BlockNumber2.Uint64(): &test_data.MockStatediffPayload2,
})
backFiller = storage.NewStorageBackFiller(fetcher)
})
It("Batch calls statediff_stateDiffAt", func() {
backFillStorage, backFillErr := backFiller.BackFill(test_data.BlockNumber.Uint64(), test_data.BlockNumber2.Uint64())
Expect(backFillErr).ToNot(HaveOccurred())
Expect(len(backFillStorage)).To(Equal(4))
// Can only rlp encode the slice of diffs as part of a struct
// Rlp encoding allows us to compare content of the slices when the order in the slice may vary
expectedDiffStruct := struct {
diffs []utils.StorageDiff
}{
[]utils.StorageDiff{
test_data.CreatedExpectedStorageDiff,
test_data.UpdatedExpectedStorageDiff,
test_data.UpdatedExpectedStorageDiff2,
test_data.DeletedExpectedStorageDiff,
},
}
expectedDiffBytes, rlpErr1 := rlp.EncodeToBytes(expectedDiffStruct)
Expect(rlpErr1).ToNot(HaveOccurred())
receivedDiffStruct := struct {
diffs []utils.StorageDiff
}{
backFillStorage,
}
receivedDiffBytes, rlpErr2 := rlp.EncodeToBytes(receivedDiffStruct)
Expect(rlpErr2).ToNot(HaveOccurred())
Expect(bytes.Equal(expectedDiffBytes, receivedDiffBytes)).To(BeTrue())
})
})
})
libraries/shared/storage/backfiller.go
+158
View File
@@ -0,0 +1,158 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"errors"
"fmt"
"sync/atomic"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
const (
DefaultMaxBatchSize uint64 = 5000
defaultMaxBatchNumber int64 = 100
)
// BackFiller is the backfilling interface
type BackFiller interface {
BackFill(endingBlock uint64, backFill chan utils.StorageDiff, errChan chan error, done chan bool) error
}
// backFiller is the backfilling struct
type backFiller struct {
fetcher fetcher.StateDiffFetcher
batchSize uint64
startingBlock uint64
}
// NewStorageBackFiller returns a BackFiller
func NewStorageBackFiller(fetcher fetcher.StateDiffFetcher, startingBlock, batchSize uint64) BackFiller {
if batchSize == 0 {
batchSize = DefaultMaxBatchSize
}
return &backFiller{
fetcher: fetcher,
batchSize: batchSize,
startingBlock: startingBlock,
}
}
// BackFill fetches, processes, and returns utils.StorageDiffs over a range of blocks
// It splits a large range up into smaller chunks, batch fetching and processing those chunks concurrently
func (bf *backFiller) BackFill(endingBlock uint64, backFill chan utils.StorageDiff, errChan chan error, done chan bool) error {
if endingBlock < bf.startingBlock {
return errors.New("backfill: ending block number needs to be greater than starting block number")
}
// break the range up into bins of smaller ranges
length := endingBlock - bf.startingBlock + 1
numberOfBins := length / bf.batchSize
remainder := length % bf.batchSize
if remainder != 0 {
numberOfBins++
}
blockRangeBins := make([][]uint64, numberOfBins)
for i := range blockRangeBins {
nextBinStart := bf.startingBlock + uint64(bf.batchSize)
if nextBinStart > endingBlock {
nextBinStart = endingBlock + 1
}
blockRange := make([]uint64, 0, nextBinStart-bf.startingBlock+1)
for j := bf.startingBlock; j < nextBinStart; j++ {
blockRange = append(blockRange, j)
}
bf.startingBlock = nextBinStart
blockRangeBins[i] = blockRange
}
// int64 for atomic incrementing and decrementing to track the number of active processing goroutines we have
var activeCount int64
// channel for processing goroutines to signal when they are done
processingDone := make(chan bool)
// for each block range bin spin up a goroutine to batch fetch and process state diffs for that range
go func() {
for _, blockHeights := range blockRangeBins {
// if we have reached our limit of active goroutines
// wait for one to finish before starting the next
if atomic.AddInt64(&activeCount, 1) > defaultMaxBatchNumber {
// this blocks until a process signals it has finished
// immediately forwards the signal to the normal listener so that it keeps the correct count
processingDone <- <-processingDone
}
go func(blockHeights []uint64) {
payloads, fetchErr := bf.fetcher.FetchStateDiffsAt(blockHeights)
if fetchErr != nil {
errChan <- fetchErr
}
for _, payload := range payloads {
stateDiff := new(statediff.StateDiff)
stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff)
if stateDiffDecodeErr != nil {
errChan <- stateDiffDecodeErr
continue
}
accounts := utils.GetAccountsFromDiff(*stateDiff)
for _, account := range accounts {
logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
for _, storage := range account.Storage {
diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
if formatErr != nil {
errChan <- formatErr
continue
}
logrus.Trace("adding storage diff to results",
"keccak of address: ", diff.HashedAddress.Hex(),
"block height: ", diff.BlockHeight,
"storage key: ", diff.StorageKey.Hex(),
"storage value: ", diff.StorageValue.Hex())
backFill <- diff
}
}
}
// when this goroutine is done, send out a signal
processingDone <- true
}(blockHeights)
}
}()
// goroutine that listens on the processingDone chan
// keeps track of the number of processing goroutines that have finished
// when they have all finished, sends the final signal out
go func() {
goroutinesFinished := 0
for {
select {
case <-processingDone:
atomic.AddInt64(&activeCount, -1)
goroutinesFinished++
if goroutinesFinished == int(numberOfBins) {
done <- true
return
}
}
}
}()
return nil
}
libraries/shared/storage/backfiller_test.go
+232
View File
@@ -0,0 +1,232 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storage_test
import (
"errors"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
)
var _ = Describe("BackFiller", func() {
Describe("BackFill", func() {
var (
mockFetcher *mocks.StateDiffFetcher
backFiller storage.BackFiller
)
BeforeEach(func() {
mockFetcher = new(mocks.StateDiffFetcher)
mockFetcher.PayloadsToReturn = map[uint64]statediff.Payload{
test_data.BlockNumber.Uint64(): test_data.MockStatediffPayload,
test_data.BlockNumber2.Uint64(): test_data.MockStatediffPayload2,
}
})
It("batch calls statediff_stateDiffAt", func() {
backFiller = storage.NewStorageBackFiller(mockFetcher, test_data.BlockNumber.Uint64(), 100)
backFill := make(chan utils.StorageDiff)
done := make(chan bool)
errChan := make(chan error)
backFillInitErr := backFiller.BackFill(
test_data.BlockNumber2.Uint64(),
backFill,
errChan,
done)
Expect(backFillInitErr).ToNot(HaveOccurred())
var diffs []utils.StorageDiff
for {
select {
case diff := <-backFill:
diffs = append(diffs, diff)
continue
case err := <-errChan:
Expect(err).ToNot(HaveOccurred())
continue
case <-done:
break
}
break
}
Expect(mockFetcher.CalledTimes).To(Equal(int64(1)))
Expect(len(diffs)).To(Equal(4))
Expect(containsDiff(diffs, test_data.CreatedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.UpdatedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.DeletedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.UpdatedExpectedStorageDiff2)).To(BeTrue())
})
It("has a configurable batch size", func() {
backFiller = storage.NewStorageBackFiller(mockFetcher, test_data.BlockNumber.Uint64(), 1)
backFill := make(chan utils.StorageDiff)
done := make(chan bool)
errChan := make(chan error)
backFillInitErr := backFiller.BackFill(
test_data.BlockNumber2.Uint64(),
backFill,
errChan,
done)
Expect(backFillInitErr).ToNot(HaveOccurred())
var diffs []utils.StorageDiff
for {
select {
case diff := <-backFill:
diffs = append(diffs, diff)
continue
case err := <-errChan:
Expect(err).ToNot(HaveOccurred())
continue
case <-done:
break
}
break
}
Expect(mockFetcher.CalledTimes).To(Equal(int64(2)))
Expect(len(diffs)).To(Equal(4))
Expect(containsDiff(diffs, test_data.CreatedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.UpdatedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.DeletedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.UpdatedExpectedStorageDiff2)).To(BeTrue())
})
It("handles bin numbers in excess of the goroutine limit (100)", func() {
payloadsToReturn := make(map[uint64]statediff.Payload, 1001)
for i := test_data.BlockNumber.Uint64(); i <= test_data.BlockNumber.Uint64()+1000; i++ {
payloadsToReturn[i] = test_data.MockStatediffPayload
}
mockFetcher.PayloadsToReturn = payloadsToReturn
// batch size of 2 with 1001 block range => 501 bins
backFiller = storage.NewStorageBackFiller(mockFetcher, test_data.BlockNumber.Uint64(), 2)
backFill := make(chan utils.StorageDiff)
done := make(chan bool)
errChan := make(chan error)
backFillInitErr := backFiller.BackFill(
test_data.BlockNumber.Uint64()+1000,
backFill,
errChan,
done)
Expect(backFillInitErr).ToNot(HaveOccurred())
var diffs []utils.StorageDiff
for {
select {
case diff := <-backFill:
diffs = append(diffs, diff)
continue
case err := <-errChan:
Expect(err).ToNot(HaveOccurred())
continue
case <-done:
break
}
break
}
Expect(mockFetcher.CalledTimes).To(Equal(int64(501)))
Expect(len(diffs)).To(Equal(3003))
Expect(containsDiff(diffs, test_data.CreatedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.UpdatedExpectedStorageDiff)).To(BeTrue())
Expect(containsDiff(diffs, test_data.DeletedExpectedStorageDiff)).To(BeTrue())
})
It("passes fetcher errors forward", func() {
mockFetcher.FetchErrs = map[uint64]error{
test_data.BlockNumber.Uint64(): errors.New("mock fetcher error"),
}
backFiller = storage.NewStorageBackFiller(mockFetcher, test_data.BlockNumber.Uint64(), 1)
backFill := make(chan utils.StorageDiff)
done := make(chan bool)
errChan := make(chan error)
backFillInitErr := backFiller.BackFill(
test_data.BlockNumber2.Uint64(),
backFill,
errChan,
done)
Expect(backFillInitErr).ToNot(HaveOccurred())
var numOfErrs int
var diffs []utils.StorageDiff
for {
select {
case diff := <-backFill:
diffs = append(diffs, diff)
continue
case err := <-errChan:
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("mock fetcher error"))
numOfErrs++
continue
case <-done:
break
}
break
}
Expect(mockFetcher.CalledTimes).To(Equal(int64(2)))
Expect(numOfErrs).To(Equal(1))
Expect(len(diffs)).To(Equal(1))
Expect(containsDiff(diffs, test_data.UpdatedExpectedStorageDiff2)).To(BeTrue())
mockFetcher.FetchErrs = map[uint64]error{
test_data.BlockNumber.Uint64(): errors.New("mock fetcher error"),
test_data.BlockNumber2.Uint64(): errors.New("mock fetcher error"),
}
mockFetcher.CalledTimes = 0
backFiller = storage.NewStorageBackFiller(mockFetcher, test_data.BlockNumber.Uint64(), 1)
backFill = make(chan utils.StorageDiff)
done = make(chan bool)
errChan = make(chan error)
backFillInitErr = backFiller.BackFill(
test_data.BlockNumber2.Uint64(),
backFill,
errChan,
done)
Expect(backFillInitErr).ToNot(HaveOccurred())
numOfErrs = 0
diffs = []utils.StorageDiff{}
for {
select {
case diff := <-backFill:
diffs = append(diffs, diff)
continue
case err := <-errChan:
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("mock fetcher error"))
numOfErrs++
continue
case <-done:
break
}
break
}
Expect(mockFetcher.CalledTimes).To(Equal(int64(2)))
Expect(numOfErrs).To(Equal(2))
Expect(len(diffs)).To(Equal(0))
})
})
})
func containsDiff(diffs []utils.StorageDiff, diff utils.StorageDiff) bool {
for _, d := range diffs {
if d == diff {
return true
}
}
return false
}