forked from cerc-io/ipld-eth-server
major refactor pt 3
This commit is contained in:
Ian Norden
@@ -0,0 +1,149 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package historical
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/node"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/postgres"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/utils"
|
||||
)
|
||||
|
||||
// Env variables
|
||||
const (
|
||||
SUPERNODE_CHAIN = "SUPERNODE_CHAIN"
|
||||
SUPERNODE_FREQUENCY = "SUPERNODE_FREQUENCY"
|
||||
SUPERNODE_BATCH_SIZE = "SUPERNODE_BATCH_SIZE"
|
||||
SUPERNODE_BATCH_NUMBER = "SUPERNODE_BATCH_NUMBER"
|
||||
SUPERNODE_VALIDATION_LEVEL = "SUPERNODE_VALIDATION_LEVEL"
|
||||
|
||||
BACKFILL_MAX_IDLE_CONNECTIONS = "BACKFILL_MAX_IDLE_CONNECTIONS"
|
||||
BACKFILL_MAX_OPEN_CONNECTIONS = "BACKFILL_MAX_OPEN_CONNECTIONS"
|
||||
BACKFILL_MAX_CONN_LIFETIME = "BACKFILL_MAX_CONN_LIFETIME"
|
||||
)
|
||||
|
||||
// Config struct
|
||||
type Config struct {
|
||||
Chain shared.ChainType
|
||||
IPFSPath string
|
||||
IPFSMode shared.IPFSMode
|
||||
DBConfig config.Database
|
||||
|
||||
DB *postgres.DB
|
||||
HTTPClient interface{}
|
||||
Frequency time.Duration
|
||||
BatchSize uint64
|
||||
BatchNumber uint64
|
||||
ValidationLevel int
|
||||
Timeout time.Duration // HTTP connection timeout in seconds
|
||||
NodeInfo node.Node
|
||||
}
|
||||
|
||||
// NewConfig is used to initialize a historical config from a .toml file
|
||||
func NewConfig() (*Config, error) {
|
||||
c := new(Config)
|
||||
var err error
|
||||
|
||||
viper.BindEnv("superNode.chain", SUPERNODE_CHAIN)
|
||||
chain := viper.GetString("superNode.chain")
|
||||
c.Chain, err = shared.NewChainType(chain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.IPFSMode, err = shared.GetIPFSMode()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.IPFSMode == shared.LocalInterface || c.IPFSMode == shared.RemoteClient {
|
||||
c.IPFSPath, err = shared.GetIPFSPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
c.DBConfig.Init()
|
||||
|
||||
if err := c.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Config) init() error {
|
||||
var err error
|
||||
|
||||
viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
|
||||
viper.BindEnv("bitcoin.httpPath", shared.BTC_HTTP_PATH)
|
||||
viper.BindEnv("superNode.frequency", SUPERNODE_FREQUENCY)
|
||||
viper.BindEnv("superNode.batchSize", SUPERNODE_BATCH_SIZE)
|
||||
viper.BindEnv("superNode.batchNumber", SUPERNODE_BATCH_NUMBER)
|
||||
viper.BindEnv("superNode.validationLevel", SUPERNODE_VALIDATION_LEVEL)
|
||||
viper.BindEnv("superNode.timeout", shared.HTTP_TIMEOUT)
|
||||
|
||||
timeout := viper.GetInt("superNode.timeout")
|
||||
if timeout < 15 {
|
||||
timeout = 15
|
||||
}
|
||||
c.Timeout = time.Second * time.Duration(timeout)
|
||||
|
||||
switch c.Chain {
|
||||
case shared.Ethereum:
|
||||
ethHTTP := viper.GetString("ethereum.httpPath")
|
||||
c.NodeInfo, c.HTTPClient, err = shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case shared.Bitcoin:
|
||||
btcHTTP := viper.GetString("bitcoin.httpPath")
|
||||
c.NodeInfo, c.HTTPClient = shared.GetBtcNodeAndClient(btcHTTP)
|
||||
}
|
||||
|
||||
freq := viper.GetInt("superNode.frequency")
|
||||
var frequency time.Duration
|
||||
if freq <= 0 {
|
||||
frequency = time.Second * 30
|
||||
} else {
|
||||
frequency = time.Second * time.Duration(freq)
|
||||
}
|
||||
c.Frequency = frequency
|
||||
c.BatchSize = uint64(viper.GetInt64("superNode.batchSize"))
|
||||
c.BatchNumber = uint64(viper.GetInt64("superNode.batchNumber"))
|
||||
c.ValidationLevel = viper.GetInt("superNode.validationLevel")
|
||||
|
||||
dbConn := overrideDBConnConfig(c.DBConfig)
|
||||
db := utils.LoadPostgres(dbConn, c.NodeInfo)
|
||||
c.DB = &db
|
||||
return nil
|
||||
}
|
||||
|
||||
func overrideDBConnConfig(con config.Database) config.Database {
|
||||
viper.BindEnv("database.backFill.maxIdle", BACKFILL_MAX_IDLE_CONNECTIONS)
|
||||
viper.BindEnv("database.backFill.maxOpen", BACKFILL_MAX_OPEN_CONNECTIONS)
|
||||
viper.BindEnv("database.backFill.maxLifetime", BACKFILL_MAX_CONN_LIFETIME)
|
||||
con.MaxIdle = viper.GetInt("database.backFill.maxIdle")
|
||||
con.MaxOpen = viper.GetInt("database.backFill.maxOpen")
|
||||
con.MaxLifetime = viper.GetInt("database.backFill.maxLifetime")
|
||||
return con
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package historical_test
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TestIPFSWatcher(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "IPFS Watcher Historical Suite Test")
|
||||
}
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
logrus.SetOutput(ioutil.Discard)
|
||||
})
|
||||
@@ -0,0 +1,207 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package historical
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/builders"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/utils"
|
||||
)
|
||||
|
||||
// BackFillInterface for filling in gaps in the ipfs-blockchain-watcher db
|
||||
type BackFillInterface interface {
|
||||
// Method for the watcher to periodically check for and fill in gaps in its data using an archival node
|
||||
BackFill(wg *sync.WaitGroup)
|
||||
Stop() error
|
||||
}
|
||||
|
||||
// BackFillService for filling in gaps in the watcher
|
||||
type BackFillService struct {
|
||||
// Interface for converting payloads into IPLD object payloads
|
||||
Converter shared.PayloadConverter
|
||||
// Interface for publishing the IPLD payloads to IPFS
|
||||
Publisher shared.IPLDPublisher
|
||||
// Interface for indexing the CIDs of the published IPLDs in Postgres
|
||||
Indexer shared.CIDIndexer
|
||||
// Interface for searching and retrieving CIDs from Postgres index
|
||||
Retriever shared.CIDRetriever
|
||||
// Interface for fetching payloads over at historical blocks; over http
|
||||
Fetcher shared.PayloadFetcher
|
||||
// Channel for forwarding backfill payloads to the ScreenAndServe process
|
||||
ScreenAndServeChan chan shared.ConvertedData
|
||||
// Check frequency
|
||||
GapCheckFrequency time.Duration
|
||||
// Size of batch fetches
|
||||
BatchSize uint64
|
||||
// Number of goroutines
|
||||
BatchNumber int64
|
||||
// Channel for receiving quit signal
|
||||
QuitChan chan bool
|
||||
// Chain type
|
||||
chain shared.ChainType
|
||||
// Headers with times_validated lower than this will be resynced
|
||||
validationLevel int
|
||||
}
|
||||
|
||||
// NewBackFillService returns a new BackFillInterface
|
||||
func NewBackFillService(settings *Config, screenAndServeChan chan shared.ConvertedData) (BackFillInterface, error) {
|
||||
publisher, err := builders.NewIPLDPublisher(settings.Chain, settings.IPFSPath, settings.DB, settings.IPFSMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indexer, err := builders.NewCIDIndexer(settings.Chain, settings.DB, settings.IPFSMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
converter, err := builders.NewPayloadConverter(settings.Chain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retriever, err := builders.NewCIDRetriever(settings.Chain, settings.DB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fetcher, err := builders.NewPaylaodFetcher(settings.Chain, settings.HTTPClient, settings.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
batchSize := settings.BatchSize
|
||||
if batchSize == 0 {
|
||||
batchSize = shared.DefaultMaxBatchSize
|
||||
}
|
||||
batchNumber := int64(settings.BatchNumber)
|
||||
if batchNumber == 0 {
|
||||
batchNumber = shared.DefaultMaxBatchNumber
|
||||
}
|
||||
return &BackFillService{
|
||||
Indexer: indexer,
|
||||
Converter: converter,
|
||||
Publisher: publisher,
|
||||
Retriever: retriever,
|
||||
Fetcher: fetcher,
|
||||
GapCheckFrequency: settings.Frequency,
|
||||
BatchSize: batchSize,
|
||||
BatchNumber: int64(batchNumber),
|
||||
ScreenAndServeChan: screenAndServeChan,
|
||||
QuitChan: make(chan bool),
|
||||
chain: settings.Chain,
|
||||
validationLevel: settings.ValidationLevel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// BackFill periodically checks for and fills in gaps in the watcher db
|
||||
func (bfs *BackFillService) BackFill(wg *sync.WaitGroup) {
|
||||
ticker := time.NewTicker(bfs.GapCheckFrequency)
|
||||
go func() {
|
||||
wg.Add(1)
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-bfs.QuitChan:
|
||||
log.Infof("quiting %s FillGapsInSuperNode process", bfs.chain.String())
|
||||
return
|
||||
case <-ticker.C:
|
||||
gaps, err := bfs.Retriever.RetrieveGapsInData(bfs.validationLevel)
|
||||
if err != nil {
|
||||
log.Errorf("%s watcher db backFill RetrieveGapsInData error: %v", bfs.chain.String(), err)
|
||||
continue
|
||||
}
|
||||
// spin up worker goroutines for this search pass
|
||||
// we start and kill a new batch of workers for each pass
|
||||
// so that we know each of the previous workers is done before we search for new gaps
|
||||
heightsChan := make(chan []uint64)
|
||||
for i := 1; i <= int(bfs.BatchNumber); i++ {
|
||||
go bfs.backFill(wg, i, heightsChan)
|
||||
}
|
||||
for _, gap := range gaps {
|
||||
log.Infof("backFilling %s data from %d to %d", bfs.chain.String(), gap.Start, gap.Stop)
|
||||
blockRangeBins, err := utils.GetBlockHeightBins(gap.Start, gap.Stop, bfs.BatchSize)
|
||||
if err != nil {
|
||||
log.Errorf("%s watcher db backFill GetBlockHeightBins error: %v", bfs.chain.String(), err)
|
||||
continue
|
||||
}
|
||||
for _, heights := range blockRangeBins {
|
||||
select {
|
||||
case <-bfs.QuitChan:
|
||||
log.Infof("quiting %s BackFill process", bfs.chain.String())
|
||||
return
|
||||
default:
|
||||
heightsChan <- heights
|
||||
}
|
||||
}
|
||||
}
|
||||
// send a quit signal to each worker
|
||||
// this blocks until each worker has finished its current task and is free to receive from the quit channel
|
||||
for i := 1; i <= int(bfs.BatchNumber); i++ {
|
||||
bfs.QuitChan <- true
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Infof("%s BackFill goroutine successfully spun up", bfs.chain.String())
|
||||
}
|
||||
|
||||
func (bfs *BackFillService) backFill(wg *sync.WaitGroup, id int, heightChan chan []uint64) {
|
||||
wg.Add(1)
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case heights := <-heightChan:
|
||||
log.Debugf("%s backFill worker %d processing section from %d to %d", bfs.chain.String(), id, heights[0], heights[len(heights)-1])
|
||||
payloads, err := bfs.Fetcher.FetchAt(heights)
|
||||
if err != nil {
|
||||
log.Errorf("%s backFill worker %d fetcher error: %s", bfs.chain.String(), id, err.Error())
|
||||
}
|
||||
for _, payload := range payloads {
|
||||
ipldPayload, err := bfs.Converter.Convert(payload)
|
||||
if err != nil {
|
||||
log.Errorf("%s backFill worker %d converter error: %s", bfs.chain.String(), id, err.Error())
|
||||
}
|
||||
// If there is a ScreenAndServe process listening, forward converted payload to it
|
||||
select {
|
||||
case bfs.ScreenAndServeChan <- ipldPayload:
|
||||
log.Debugf("%s backFill worker %d forwarded converted payload to server", bfs.chain.String(), id)
|
||||
default:
|
||||
log.Debugf("%s backFill worker %d unable to forward converted payload to server; no channel ready to receive", bfs.chain.String(), id)
|
||||
}
|
||||
cidPayload, err := bfs.Publisher.Publish(ipldPayload)
|
||||
if err != nil {
|
||||
log.Errorf("%s backFill worker %d publisher error: %s", bfs.chain.String(), id, err.Error())
|
||||
continue
|
||||
}
|
||||
if err := bfs.Indexer.Index(cidPayload); err != nil {
|
||||
log.Errorf("%s backFill worker %d indexer error: %s", bfs.chain.String(), id, err.Error())
|
||||
}
|
||||
}
|
||||
log.Infof("%s backFill worker %d finished section from %d to %d", bfs.chain.String(), id, heights[0], heights[len(heights)-1])
|
||||
case <-bfs.QuitChan:
|
||||
log.Infof("%s backFill worker %d shutting down", bfs.chain.String(), id)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (bfs *BackFillService) Stop() error {
|
||||
log.Infof("Stopping %s backFill service", bfs.chain.String())
|
||||
close(bfs.QuitChan)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package historical_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/eth/mocks"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/historical"
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
|
||||
mocks2 "github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared/mocks"
|
||||
)
|
||||
|
||||
var _ = Describe("BackFiller", func() {
|
||||
Describe("FillGaps", func() {
|
||||
It("Periodically checks for and fills in gaps in the watcher's data", func() {
|
||||
mockCidRepo := &mocks.CIDIndexer{
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockPublisher := &mocks.IterativeIPLDPublisher{
|
||||
ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockConverter := &mocks.IterativePayloadConverter{
|
||||
ReturnIPLDPayload: []eth.ConvertedPayload{mocks.MockConvertedPayload, mocks.MockConvertedPayload},
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockRetriever := &mocks2.CIDRetriever{
|
||||
FirstBlockNumberToReturn: 0,
|
||||
GapsToRetrieve: []shared.Gap{
|
||||
{
|
||||
Start: 100, Stop: 101,
|
||||
},
|
||||
},
|
||||
}
|
||||
mockFetcher := &mocks2.PayloadFetcher{
|
||||
PayloadsToReturn: map[uint64]shared.RawChainData{
|
||||
100: mocks.MockStateDiffPayload,
|
||||
101: mocks.MockStateDiffPayload,
|
||||
},
|
||||
}
|
||||
quitChan := make(chan bool, 1)
|
||||
backfiller := &historical.BackFillService{
|
||||
Indexer: mockCidRepo,
|
||||
Publisher: mockPublisher,
|
||||
Converter: mockConverter,
|
||||
Fetcher: mockFetcher,
|
||||
Retriever: mockRetriever,
|
||||
GapCheckFrequency: time.Second * 2,
|
||||
BatchSize: shared.DefaultMaxBatchSize,
|
||||
BatchNumber: shared.DefaultMaxBatchNumber,
|
||||
QuitChan: quitChan,
|
||||
}
|
||||
wg := &sync.WaitGroup{}
|
||||
backfiller.BackFill(wg)
|
||||
time.Sleep(time.Second * 3)
|
||||
quitChan <- true
|
||||
Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(2))
|
||||
Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
|
||||
Expect(mockCidRepo.PassedCIDPayload[1]).To(Equal(mocks.MockCIDPayload))
|
||||
Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2))
|
||||
Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload))
|
||||
Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockConvertedPayload))
|
||||
Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(2))
|
||||
Expect(mockConverter.PassedStatediffPayload[0]).To(Equal(mocks.MockStateDiffPayload))
|
||||
Expect(mockConverter.PassedStatediffPayload[1]).To(Equal(mocks.MockStateDiffPayload))
|
||||
Expect(mockRetriever.CalledTimes).To(Equal(1))
|
||||
Expect(len(mockFetcher.CalledAtBlockHeights)).To(Equal(1))
|
||||
Expect(mockFetcher.CalledAtBlockHeights[0]).To(Equal([]uint64{100, 101}))
|
||||
})
|
||||
|
||||
It("Works for single block `ranges`", func() {
|
||||
mockCidRepo := &mocks.CIDIndexer{
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockPublisher := &mocks.IterativeIPLDPublisher{
|
||||
ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload},
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockConverter := &mocks.IterativePayloadConverter{
|
||||
ReturnIPLDPayload: []eth.ConvertedPayload{mocks.MockConvertedPayload},
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockRetriever := &mocks2.CIDRetriever{
|
||||
FirstBlockNumberToReturn: 0,
|
||||
GapsToRetrieve: []shared.Gap{
|
||||
{
|
||||
Start: 100, Stop: 100,
|
||||
},
|
||||
},
|
||||
}
|
||||
mockFetcher := &mocks2.PayloadFetcher{
|
||||
PayloadsToReturn: map[uint64]shared.RawChainData{
|
||||
100: mocks.MockStateDiffPayload,
|
||||
},
|
||||
}
|
||||
quitChan := make(chan bool, 1)
|
||||
backfiller := &historical.BackFillService{
|
||||
Indexer: mockCidRepo,
|
||||
Publisher: mockPublisher,
|
||||
Converter: mockConverter,
|
||||
Fetcher: mockFetcher,
|
||||
Retriever: mockRetriever,
|
||||
GapCheckFrequency: time.Second * 2,
|
||||
BatchSize: shared.DefaultMaxBatchSize,
|
||||
BatchNumber: shared.DefaultMaxBatchNumber,
|
||||
QuitChan: quitChan,
|
||||
}
|
||||
wg := &sync.WaitGroup{}
|
||||
backfiller.BackFill(wg)
|
||||
time.Sleep(time.Second * 3)
|
||||
quitChan <- true
|
||||
Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(1))
|
||||
Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
|
||||
Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(1))
|
||||
Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload))
|
||||
Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(1))
|
||||
Expect(mockConverter.PassedStatediffPayload[0]).To(Equal(mocks.MockStateDiffPayload))
|
||||
Expect(mockRetriever.CalledTimes).To(Equal(1))
|
||||
Expect(len(mockFetcher.CalledAtBlockHeights)).To(Equal(1))
|
||||
Expect(mockFetcher.CalledAtBlockHeights[0]).To(Equal([]uint64{100}))
|
||||
})
|
||||
|
||||
It("Finds beginning gap", func() {
|
||||
mockCidRepo := &mocks.CIDIndexer{
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockPublisher := &mocks.IterativeIPLDPublisher{
|
||||
ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockConverter := &mocks.IterativePayloadConverter{
|
||||
ReturnIPLDPayload: []eth.ConvertedPayload{mocks.MockConvertedPayload, mocks.MockConvertedPayload},
|
||||
ReturnErr: nil,
|
||||
}
|
||||
mockRetriever := &mocks2.CIDRetriever{
|
||||
FirstBlockNumberToReturn: 3,
|
||||
GapsToRetrieve: []shared.Gap{
|
||||
{
|
||||
Start: 0,
|
||||
Stop: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
mockFetcher := &mocks2.PayloadFetcher{
|
||||
PayloadsToReturn: map[uint64]shared.RawChainData{
|
||||
1: mocks.MockStateDiffPayload,
|
||||
2: mocks.MockStateDiffPayload,
|
||||
},
|
||||
}
|
||||
quitChan := make(chan bool, 1)
|
||||
backfiller := &historical.BackFillService{
|
||||
Indexer: mockCidRepo,
|
||||
Publisher: mockPublisher,
|
||||
Converter: mockConverter,
|
||||
Fetcher: mockFetcher,
|
||||
Retriever: mockRetriever,
|
||||
GapCheckFrequency: time.Second * 2,
|
||||
BatchSize: shared.DefaultMaxBatchSize,
|
||||
BatchNumber: shared.DefaultMaxBatchNumber,
|
||||
QuitChan: quitChan,
|
||||
}
|
||||
wg := &sync.WaitGroup{}
|
||||
backfiller.BackFill(wg)
|
||||
time.Sleep(time.Second * 3)
|
||||
quitChan <- true
|
||||
Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(2))
|
||||
Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
|
||||
Expect(mockCidRepo.PassedCIDPayload[1]).To(Equal(mocks.MockCIDPayload))
|
||||
Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2))
|
||||
Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockConvertedPayload))
|
||||
Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockConvertedPayload))
|
||||
Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(2))
|
||||
Expect(mockConverter.PassedStatediffPayload[0]).To(Equal(mocks.MockStateDiffPayload))
|
||||
Expect(mockConverter.PassedStatediffPayload[1]).To(Equal(mocks.MockStateDiffPayload))
|
||||
Expect(mockRetriever.CalledTimes).To(Equal(1))
|
||||
Expect(len(mockFetcher.CalledAtBlockHeights)).To(Equal(1))
|
||||
Expect(mockFetcher.CalledAtBlockHeights[0]).To(Equal([]uint64{0, 1, 2}))
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user