backfill based on validation level

pkg/super_node/eth/retriever.go

@@ -445,7 +445,8 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
 }
 
 // RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db
-func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
+// it finds the union of heights where no data exists and where the times_validated is lower than the validation level
+func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap, error) {
 	pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM eth.header_cids
 				LEFT JOIN eth.header_cids r on eth.header_cids.block_number = r.block_number - 1
 				LEFT JOIN eth.header_cids fr on eth.header_cids.block_number < fr.block_number
@@ -455,18 +456,45 @@ func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
 		Start uint64 `db:"start"`
 		Stop  uint64 `db:"stop"`
 	}, 0)
-	err := ecr.db.Select(&results, pgStr)
-	if err != nil {
+	if err := ecr.db.Select(&results, pgStr); err != nil {
 		return nil, err
 	}
-	gaps := make([]shared.Gap, len(results))
+	emptyGaps := make([]shared.Gap, len(results))
 	for i, res := range results {
-		gaps[i] = shared.Gap{
+		emptyGaps[i] = shared.Gap{
 			Start: res.Start,
 			Stop:  res.Stop,
 		}
 	}
-	return gaps, nil
+
+	// Find sections of blocks where we are below the validation level
+	// There will be no overlap between these "gaps" and the ones above
+	pgStr = `SELECT block_number FROM eth.header_cids
+			WHERE times_validated < $1
+			ORDER BY block_number`
+	var heights []uint64
+	if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil {
+		return nil, err
+	}
+	if len(heights) == 0 {
+		return emptyGaps, nil
+	}
+	validationGaps := make([]shared.Gap, 0)
+	start := heights[0]
+	lastHeight := start
+	for _, height := range heights[1:] {
+		if height == lastHeight+1 {
+			lastHeight = height
+			continue
+		}
+		validationGaps = append(validationGaps, shared.Gap{
+			Start: start,
+			Stop:  lastHeight,
+		})
+		start = height
+		lastHeight = start
+	}
+	return append(emptyGaps, validationGaps...), nil
 }
 
 // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash

pkg/super_node/eth/retriever_test.go

@@ -484,7 +484,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload2)
 			Expect(err).ToNot(HaveOccurred())
-			gaps, err := retriever.RetrieveGapsInData()
+			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(gaps)).To(Equal(0))
 		})
@@ -494,11 +494,29 @@ var _ = Describe("Retriever", func() {
 			payload.HeaderCID.BlockNumber = "5"
 			err := repo.Index(&payload)
 			Expect(err).ToNot(HaveOccurred())
-			gaps, err := retriever.RetrieveGapsInData()
+			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(gaps)).To(Equal(0))
 		})
 
+		It("Can handle single block gaps", func() {
+			payload1 := *mocks.MockCIDPayload
+			payload1.HeaderCID.BlockNumber = "2"
+			payload2 := payload1
+			payload2.HeaderCID.BlockNumber = "4"
+			err := repo.Index(mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload2)
+			Expect(err).ToNot(HaveOccurred())
+			gaps, err := retriever.RetrieveGapsInData(1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(gaps)).To(Equal(1))
+			Expect(gaps[0].Start).To(Equal(uint64(3)))
+			Expect(gaps[0].Stop).To(Equal(uint64(3)))
+		})
+
 		It("Finds gap between two entries", func() {
 			payload1 := *mocks.MockCIDPayload
 			payload1.HeaderCID.BlockNumber = "1010101"
@@ -508,7 +526,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload2)
 			Expect(err).ToNot(HaveOccurred())
-			gaps, err := retriever.RetrieveGapsInData()
+			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(gaps)).To(Equal(1))
 			Expect(gaps[0].Start).To(Equal(uint64(6)))
@@ -540,7 +558,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload6)
 			Expect(err).ToNot(HaveOccurred())
-			gaps, err := retriever.RetrieveGapsInData()
+			gaps, err := retriever.RetrieveGapsInData(1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(gaps)).To(Equal(3))
 			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 6, Stop: 99})).To(BeTrue())