Statediff missing parent blocks automatically (#8)

Port of https://github.com/cerc-io/go-ethereum/pull/399 and https://github.com/cerc-io/go-ethereum/pull/397.

Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Reviewed-on: #8

indexer/database/dump/indexer.go

@@ -187,6 +187,7 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
 		UnclesHash:      header.UncleHash.String(),
 		Timestamp:       header.Time,
 		Coinbase:        header.Coinbase.String(),
+		Canonical:       true,
 	}
 	_, err := fmt.Fprintf(sdi.dump, "%+v\r\n", mod)
 	return headerID, err

indexer/database/file/csv_writer.go

@@ -231,7 +231,8 @@ func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
 	var values []interface{}
 	values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
 		header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot,
-		header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase)
+		header.RctRoot, header.UnclesHash, header.Bloom, strconv.FormatUint(header.Timestamp, 10), header.Coinbase,
+		header.Canonical)
 	csw.rows <- tableRow{schema.TableHeader, values}
 	metrics.IndexerMetrics.BlocksCounter.Inc(1)
 }

indexer/database/file/indexer.go

@@ -244,6 +244,7 @@ func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode ipld
 		UnclesHash:      header.UncleHash.String(),
 		Timestamp:       header.Time,
 		Coinbase:        header.Coinbase.String(),
+		Canonical:       true,
 	})
 	return headerID
 }

indexer/database/file/sql_writer.go

@@ -140,8 +140,8 @@ const (
 	ipldInsert = "INSERT INTO ipld.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
 
 	headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, " +
-		"state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) VALUES " +
-		"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s');\n"
+		"state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase, canonical) VALUES " +
+		"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %t);\n"
 
 	uncleInsert = "INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES " +
 		"('%s', '%s', '%s', '%s', '%s', '%s', %d);\n"
@@ -189,7 +189,7 @@ func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
 func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
 	stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
 		header.TotalDifficulty, formatPostgresStringArray(header.NodeIDs), header.Reward, header.StateRoot, header.TxRoot,
-		header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase)
+		header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase, header.Canonical)
 	sqw.stmts <- []byte(stmt)
 	metrics.IndexerMetrics.BlocksCounter.Inc(1)
 }

indexer/database/sql/indexer.go

@@ -257,6 +257,7 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
 		UnclesHash:      header.UncleHash.String(),
 		Timestamp:       header.Time,
 		Coinbase:        header.Coinbase.String(),
+		Canonical:       true,
 	})
 }
 

indexer/database/sql/interfaces.go

@@ -49,6 +49,7 @@ type Statements interface {
 	MaxHeaderStm() string
 	ExistsHeaderStm() string
 	InsertHeaderStm() string
+	SetCanonicalHeaderStm() string
 	InsertUncleStm() string
 	InsertTxStm() string
 	InsertRctStm() string

indexer/database/sql/postgres/database.go

@@ -62,6 +62,12 @@ func (db *DB) InsertHeaderStm() string {
 	return schema.TableHeader.ToInsertStatement(db.upsert)
 }
 
+// SetCanonicalHeaderStm satisfies the sql.Statements interface
+// Stm == Statement
+func (db *DB) SetCanonicalHeaderStm() string {
+	return fmt.Sprintf("UPDATE %s SET canonical = false WHERE block_number = $1::BIGINT AND block_hash <> $2::TEXT AND canonical = true", schema.TableHeader.Name)
+}
+
 // InsertUncleStm satisfies the sql.Statements interface
 func (db *DB) InsertUncleStm() string {
 	return schema.TableUncle.ToInsertStatement(db.upsert)

indexer/database/sql/writer.go

@@ -90,6 +90,7 @@ func (w *Writer) maxHeader() (*models.HeaderModel, error) {
 		&model.Bloom,
 		&model.Timestamp,
 		&model.Coinbase,
+		&model.Canonical,
 	)
 	model.BlockNumber = strconv.FormatUint(number, 10)
 	model.TotalDifficulty = strconv.FormatUint(td, 10)
@@ -118,11 +119,22 @@ func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
 		header.UnclesHash,
 		header.Bloom,
 		header.Timestamp,
-		header.Coinbase)
+		header.Coinbase,
+		header.Canonical,
+	)
 	if err != nil {
 		return insertError{"eth.header_cids", err, w.db.InsertHeaderStm(), header}
 	}
 	metrics.IndexerMetrics.BlocksCounter.Inc(1)
+
+	_, err = tx.Exec(w.db.Context(), w.db.SetCanonicalHeaderStm(),
+		header.BlockNumber,
+		header.BlockHash,
+	)
+	if err != nil {
+		return insertError{"eth.header_cids", err, w.db.SetCanonicalHeaderStm(), header}
+	}
+
 	return nil
 }