backfill based on validation level

pkg/super_node/btc/retriever.go

@@ -161,7 +161,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID
 }
 
 // RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db
-func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
+func (ecr *CIDRetriever) RetrieveGapsInData(validationLevel int) ([]shared.Gap, error) {
 	pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM btc.header_cids
 				LEFT JOIN btc.header_cids r on btc.header_cids.block_number = r.block_number - 1
 				LEFT JOIN btc.header_cids fr on btc.header_cids.block_number < fr.block_number
@@ -171,18 +171,45 @@ func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
 		Start uint64 `db:"start"`
 		Stop  uint64 `db:"stop"`
 	}, 0)
-	err := ecr.db.Select(&results, pgStr)
-	if err != nil {
+	if err := ecr.db.Select(&results, pgStr); err != nil {
 		return nil, err
 	}
-	gaps := make([]shared.Gap, len(results))
+	emptyGaps := make([]shared.Gap, len(results))
 	for i, res := range results {
-		gaps[i] = shared.Gap{
+		emptyGaps[i] = shared.Gap{
 			Start: res.Start,
 			Stop:  res.Stop,
 		}
 	}
-	return gaps, nil
+
+	// Find sections of blocks where we are below the validation level
+	// There will be no overlap between these "gaps" and the ones above
+	pgStr = `SELECT block_number FROM btc.header_cids
+			WHERE times_validated < $1
+			ORDER BY block_number`
+	var heights []uint64
+	if err := ecr.db.Select(&heights, pgStr, validationLevel); err != nil {
+		return nil, err
+	}
+	if len(heights) == 0 {
+		return emptyGaps, nil
+	}
+	validationGaps := make([]shared.Gap, 0)
+	start := heights[0]
+	lastHeight := start
+	for _, height := range heights[1:] {
+		if height == lastHeight+1 {
+			lastHeight = height
+			continue
+		}
+		validationGaps = append(validationGaps, shared.Gap{
+			Start: start,
+			Stop:  lastHeight,
+		})
+		start = height
+		lastHeight = start
+	}
+	return append(emptyGaps, validationGaps...), nil
 }
 
 // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash