Add new job queue for historical blocks processing (#442)

* Add TODOs for historical blocks processing

* Add new job for historical blocks processing

* Handle historical job completion

* Fetch latest block in chain and start historical block processing

* Fix starting realtime block processing from latest canonical block

* Refactor historical block processing method and add logs

* Add dummy indexer methods in graph-node to pass test

* Changes in codegen for historical processing in generated watcher

packages/codegen/src/templates/database-template.handlebars

@@ -253,6 +253,12 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async forceUpdateSyncStatus (queryRunner: QueryRunner, blockHash: string, blockNumber: number): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.forceUpdateSyncStatus(repo, blockHash, blockNumber);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 
@@ -271,6 +277,12 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.getBlocksAtHeight(repo, height, isPruned);
   }
 
+  async getLatestProcessedBlockProgress (isPruned: boolean): Promise<BlockProgress | undefined> {
+    const repo = this._conn.getRepository(BlockProgress);
+
+    return this._baseDatabase.getLatestProcessedBlockProgress(repo, isPruned);
+  }
+
   async markBlocksAsPruned (queryRunner: QueryRunner, blocks: BlockProgress[]): Promise<void> {
     const repo = queryRunner.manager.getRepository(BlockProgress);
 

packages/codegen/src/templates/indexer-template.handlebars

@@ -509,7 +509,7 @@ export class Indexer implements IndexerInterface {
     if (!this._serverConfig.enableState) {
       return;
     }
-    
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -637,6 +637,10 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async forceUpdateSyncStatus (blockHash: string, blockNumber: number): Promise<SyncStatus> {
+    return this._baseIndexer.forceUpdateSyncStatus(blockHash, blockNumber);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -653,6 +657,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
+  async getLatestProcessedBlockProgress (isPruned: boolean): Promise<BlockProgress | undefined> {
+    return this._db.getLatestProcessedBlockProgress(isPruned);
+  }
+
   async fetchEventsAndSaveBlocks (blocks: DeepPartial<BlockProgress>[]): Promise<{ blockProgress: BlockProgress, events: DeepPartial<Event>[] }[]> {
     return this._baseIndexer.fetchEventsAndSaveBlocks(blocks, this._eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
   }

packages/codegen/src/templates/job-runner-template.handlebars

@@ -34,6 +34,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();