seed => super; port 80 => port 8080; backfill process for the super_node

2019-12-02 13:24:58 -06:00
parent 83fd76bc8a
commit 40c3aff597
33 changed files with 843 additions and 373 deletions
@@ -0,0 +1,135 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package super_node
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/params"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+)
+
+// BackFillInterface for filling in gaps in the super node
+type BackFillInterface interface {
+	// Method for the super node to periodically check for and fill in gaps in its data using an archival node
+	FillGaps(wg *sync.WaitGroup, quitChan <-chan bool)
+}
+
+// BackFillService for filling in gaps in the super node
+type BackFillService struct {
+	// Interface for converting statediff payloads into ETH-IPLD object payloads
+	Converter ipfs.PayloadConverter
+	// Interface for publishing the ETH-IPLD payloads to IPFS
+	Publisher ipfs.IPLDPublisher
+	// Interface for indexing the CIDs of the published ETH-IPLDs in Postgres
+	Repository CIDRepository
+	// Interface for searching and retrieving CIDs from Postgres index
+	Retriever CIDRetriever
+	// State-diff fetcher; needs to be configured with an archival core.RpcClient
+	StateDiffFetcher fetcher.IStateDiffFetcher
+	// Check frequency
+	GapCheckFrequency time.Duration
+}
+
+// NewBackFillService returns a new BackFillInterface
+func NewBackFillService(ipfsPublisher ipfs.IPLDPublisher, db *postgres.DB, archivalNodeRpcClient core.RpcClient, freq time.Duration) BackFillInterface {
+	return &BackFillService{
+		Repository:        NewCIDRepository(db),
+		Converter:         ipfs.NewPayloadConverter(params.MainnetChainConfig),
+		Publisher:         ipfsPublisher,
+		Retriever:         NewCIDRetriever(db),
+		StateDiffFetcher:  fetcher.NewStateDiffFetcher(archivalNodeRpcClient),
+		GapCheckFrequency: freq,
+	}
+}
+
+// FillGaps periodically checks for and fills in gaps in the super node db
+// this requires a core.RpcClient that is pointed at an archival node with the StateDiffAt method exposed
+func (bfs *BackFillService) FillGaps(wg *sync.WaitGroup, quitChan <-chan bool) {
+	ticker := time.NewTicker(bfs.GapCheckFrequency)
+	wg.Add(1)
+
+	go func() {
+		for {
+			select {
+			case <-quitChan:
+				log.Info("quiting FillGaps process")
+				wg.Done()
+				return
+			case <-ticker.C:
+				log.Info("searching for gaps in the super node database")
+				startingBlock, firstBlockErr := bfs.Retriever.RetrieveFirstBlockNumber()
+				if firstBlockErr != nil {
+					log.Error(firstBlockErr)
+					continue
+				}
+				if startingBlock != 1 {
+					startingGap := [2]int64{
+						1,
+						startingBlock - 1,
+					}
+					log.Info("found gap at the beginning of the sync")
+					bfs.fillGaps(startingGap)
+				}
+
+				gaps, gapErr := bfs.Retriever.RetrieveGapsInData()
+				if gapErr != nil {
+					log.Error(gapErr)
+					continue
+				}
+				for _, gap := range gaps {
+					bfs.fillGaps(gap)
+				}
+			}
+		}
+	}()
+}
+
+func (bfs *BackFillService) fillGaps(gap [2]int64) {
+	log.Infof("filling in gap from block %d to block %d", gap[0], gap[1])
+	blockHeights := make([]uint64, 0, gap[1]-gap[0]+1)
+	for i := gap[0]; i <= gap[1]; i++ {
+		blockHeights = append(blockHeights, uint64(i))
+	}
+	payloads, fetchErr := bfs.StateDiffFetcher.FetchStateDiffsAt(blockHeights)
+	if fetchErr != nil {
+		log.Error(fetchErr)
+		return
+	}
+	for _, payload := range payloads {
+		ipldPayload, convertErr := bfs.Converter.Convert(*payload)
+		if convertErr != nil {
+			log.Error(convertErr)
+			continue
+		}
+		cidPayload, publishErr := bfs.Publisher.Publish(ipldPayload)
+		if publishErr != nil {
+			log.Error(publishErr)
+			continue
+		}
+		indexErr := bfs.Repository.Index(cidPayload)
+		if indexErr != nil {
+			log.Error(indexErr)
+		}
+	}
+}