292: Backfill gaps in the recent past on startup when tracking head. (#395)

* Backfill gaps in the recent past when statediffing head.

indexer/database/sql/indexer.go

@@ -221,6 +221,16 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	return blockTx, err
 }
 
+// CurrentBlock returns the HeaderModel of the highest existing block in the database.
+func (sdi *StateDiffIndexer) CurrentBlock() (*models.HeaderModel, error) {
+	return sdi.dbWriter.maxHeader()
+}
+
+// DetectGaps returns a list of gaps in the database found within the specified block range.
+func (sdi *StateDiffIndexer) DetectGaps(beginBlockNumber uint64, endBlockNumber uint64) ([]*interfaces.BlockGap, error) {
+	return sdi.dbWriter.detectGaps(beginBlockNumber, endBlockNumber)
+}
+
 // processHeader publishes and indexes a header IPLD in Postgres
 // it returns the headerID
 func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode ipld.IPLD, reward, td *big.Int) (string, error) {

indexer/database/sql/interfaces.go

@@ -45,6 +45,8 @@ type Driver interface {
 
 // Statements interface to accommodate different SQL query syntax
 type Statements interface {
+	DetectGapsStm() string
+	MaxHeaderStm() string
 	ExistsHeaderStm() string
 	InsertHeaderStm() string
 	InsertUncleStm() string

indexer/database/sql/postgres/database.go

@@ -41,8 +41,19 @@ type DB struct {
 	sql.Driver
 }
 
+// MaxHeaderStm satisfies the sql.Statements interface
+func (db *DB) MaxHeaderStm() string {
+	return fmt.Sprintf("SELECT block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase FROM %s ORDER BY block_number DESC LIMIT 1", schema.TableHeader.Name)
+}
+
+// ExistsHeaderStm satisfies the sql.Statements interface
 func (db *DB) ExistsHeaderStm() string {
-	return fmt.Sprintf("SELECT EXISTS(SELECT 1 from %s WHERE block_number = $1 AND block_hash = $2 LIMIT 1)", schema.TableHeader.Name)
+	return fmt.Sprintf("SELECT EXISTS(SELECT 1 from %s WHERE block_number = $1::BIGINT AND block_hash = $2::TEXT LIMIT 1)", schema.TableHeader.Name)
+}
+
+// DetectGapsStm satisfies the sql.Statements interface
+func (db *DB) DetectGapsStm() string {
+	return fmt.Sprintf("SELECT block_number + 1 AS first_missing, (next_bn - 1) AS last_missing FROM (SELECT block_number, LEAD(block_number) OVER (ORDER BY block_number) AS next_bn FROM %s WHERE block_number >= $1::BIGINT AND block_number <= $2::BIGINT) h WHERE next_bn > block_number + 1", schema.TableHeader.Name)
 }
 
 // InsertHeaderStm satisfies the sql.Statements interface

indexer/database/sql/writer.go

@@ -27,6 +27,7 @@ import (
 	"github.com/shopspring/decimal"
 
 	"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
 	"github.com/cerc-io/plugeth-statediff/indexer/models"
 )
 
@@ -47,11 +48,55 @@ func (w *Writer) Close() error {
 	return w.db.Close()
 }
 
+// hasHeader returns true if a matching hash+number record exists in the database, else false.
 func (w *Writer) hasHeader(blockHash common.Hash, blockNumber uint64) (exists bool, err error) {
-	err = w.db.QueryRow(w.db.Context(), w.db.ExistsHeaderStm(), blockNumber, blockHash.String()).Scan(&exists)
+	// pgx misdetects the parameter OIDs and selects int8, which can overflow.
+	// unfortunately there is no good place to override it, so it is safer to pass the uint64s as text
+	// and let PG handle the cast
+	err = w.db.QueryRow(w.db.Context(), w.db.ExistsHeaderStm(), strconv.FormatUint(blockNumber, 10), blockHash.String()).Scan(&exists)
 	return exists, err
 }
 
+// detectGaps returns a list of BlockGaps detected within the specified block range
+// For example, if the database contains blocks the overall range 1000:2000, but is missing blocks 1110:1230 and 1380
+// it would return [{FirstMissing: 1110, LastMissing: 1230}, {FirstMissing: 1380, LastMissing: 1380}]
+func (w *Writer) detectGaps(beginBlockNumber uint64, endBlockNumber uint64) ([]*interfaces.BlockGap, error) {
+	var gaps []*interfaces.BlockGap
+	// pgx misdetects the parameter OIDs and selects int8, which can overflow.
+	// unfortunately there is no good place to override it, so it is safer to pass the uint64s as text
+	// and let PG handle the cast
+	err := w.db.Select(w.db.Context(), &gaps, w.db.DetectGapsStm(), strconv.FormatUint(beginBlockNumber, 10), strconv.FormatUint(endBlockNumber, 10))
+	return gaps, err
+}
+
+// maxHeader returns the header for the highest block number in the database.
+// SELECT block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase FROM %s ORDER BY block_number DESC LIMIT 1
+func (w *Writer) maxHeader() (*models.HeaderModel, error) {
+	var model models.HeaderModel
+	var err error
+	var number, td, reward uint64
+	err = w.db.QueryRow(w.db.Context(), w.db.MaxHeaderStm()).Scan(
+		&number,
+		&model.BlockHash,
+		&model.ParentHash,
+		&model.CID,
+		&td,
+		&model.NodeIDs,
+		&reward,
+		&model.StateRoot,
+		&model.TxRoot,
+		&model.RctRoot,
+		&model.UnclesHash,
+		&model.Bloom,
+		&model.Timestamp,
+		&model.Coinbase,
+	)
+	model.BlockNumber = strconv.FormatUint(number, 10)
+	model.TotalDifficulty = strconv.FormatUint(td, 10)
+	model.Reward = strconv.FormatUint(reward, 10)
+	return &model, err
+}
+
 /*
 INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)