Reset watcher to previous indexed block on start (#207)

* Reset watcher to previous indexed block before start

* Implement changes in other watchers

* Save successfully fetched blocks and events to prefetch cache

* Add unique query for transaction table

* Check db for blocks before fetching from eth-server

* Show all mismatches at a block

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>

packages/codegen/src/indexer.ts

@@ -45,6 +45,7 @@ export class Indexer {
 
     const queryObject = {
       name,
+      entityName: '',
       getQueryName: '',
       saveQueryName: '',
       params: _.cloneDeep(params),
@@ -56,10 +57,12 @@ export class Indexer {
 
     if (name.charAt(0) === '_') {
       const capitalizedName = `${name.charAt(1).toUpperCase()}${name.slice(2)}`;
+      queryObject.entityName = `_${capitalizedName}`;
       queryObject.getQueryName = `_get${capitalizedName}`;
       queryObject.saveQueryName = `_save${capitalizedName}`;
     } else {
       const capitalizedName = `${name.charAt(0).toUpperCase()}${name.slice(1)}`;
+      queryObject.entityName = capitalizedName;
       queryObject.getQueryName = `get${capitalizedName}`;
       queryObject.saveQueryName = `save${capitalizedName}`;
     }

packages/codegen/src/templates/database-template.handlebars

@@ -3,7 +3,7 @@
 //
 
 import assert from 'assert';
-import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, FindManyOptions } from 'typeorm';
+import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, FindManyOptions, EntityTarget } from 'typeorm';
 import path from 'path';
 
 import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, Where } from '@cerc-io/util';
@@ -265,6 +265,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.removeEntities(queryRunner, entity, findConditions);
   }
 
+  async deleteEntitiesByConditions<Entity> (queryRunner: QueryRunner, entity: EntityTarget<Entity>, findConditions: FindConditions<Entity>): Promise<void> {
+    await this._baseDatabase.deleteEntitiesByConditions(queryRunner, entity, findConditions);
+  }
+
   async getAncestorAtDepth (blockHash: string, depth: number): Promise<string> {
     return this._baseDatabase.getAncestorAtDepth(blockHash, depth);
   }

packages/codegen/src/templates/indexer-template.handlebars

@@ -50,6 +50,9 @@ import { StateSyncStatus } from './entity/StateSyncStatus';
 import { BlockProgress } from './entity/BlockProgress';
 import { State } from './entity/State';
 
+{{#each queries as | query |}}
+import { {{query.entityName}} } from './entity/{{query.entityName}}';
+{{/each}}
 {{#each subgraphEntities as | subgraphEntity |}}
 import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.className}}';
 {{/each}}
@@ -537,8 +540,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async fetchBlockWithEvents (block: DeepPartial<BlockProgress>): Promise<BlockProgress> {
-    return this._baseIndexer.fetchBlockWithEvents(block, this._fetchAndSaveEvents.bind(this));
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+    return this._baseIndexer.saveBlockAndFetchEvents(block, this._saveBlockAndFetchEvents.bind(this));
   }
 
   async getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise<Array<Event>> {
@@ -594,6 +597,18 @@ export class Indexer implements IndexerInterface {
     this._subgraphStateMap.clear();
   }
 
+  async resetWatcherToBlock (blockNumber: number): Promise<void> {
+    const entities = [
+      {{#each queries as | query |}}
+      {{query.entityName}},
+      {{/each}}
+      {{#each subgraphEntities as | subgraphEntity |}}
+      {{subgraphEntity.className}},
+      {{/each}}
+    ];
+    await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
+  }
+
   _populateEntityTypesMap (): void {
   {{#each subgraphEntities as | subgraphEntity |}}
     this._entityTypesMap.set('{{subgraphEntity.className}}', {

packages/codegen/src/templates/job-runner-template.handlebars

@@ -49,6 +49,8 @@ export class JobRunner {
   }
 
   async start (): Promise<void> {
+    await this._jobQueue.deleteAllJobs();
+    await this._baseJobRunner.resetToPrevIndexedBlock();
     await this.subscribeBlockProcessingQueue();
     await this.subscribeEventProcessingQueue();
     await this.subscribeBlockCheckpointQueue();

packages/codegen/src/templates/reset-watcher-template.handlebars

@@ -67,58 +67,6 @@ export const handler = async (argv: any): Promise<void> => {
   await graphWatcher.init();
   {{/if}}
 
-  const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false);
-  assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`);
-  assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`);
-  const [blockProgress] = blockProgresses;
-
-  const dbTx = await db.createTransactionRunner();
-
-  try {
-    const entities = [BlockProgress
-    {{~#each queries as | query |~}}
-    , {{query.entityName}}
-    {{~/each~}}
-    ];
-
-    const removeEntitiesPromise = entities.map(async entityClass => {
-      return db.removeEntities<any>(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) });
-    });
-
-    await Promise.all(removeEntitiesPromise);
-
-    const syncStatus = await indexer.getSyncStatus();
-    assert(syncStatus, 'Missing syncStatus');
-
-    if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
-      await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
-    }
-
-    if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) {
-      await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true);
-    }
-
-    const stateSyncStatus = await indexer.getStateSyncStatus();
-
-    if (stateSyncStatus) {
-      if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) {
-        await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true);
-      }
-
-      if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) {
-        await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true);
-      }
-    }
-
-    await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true);
-
-    dbTx.commitTransaction();
-  } catch (error) {
-    await dbTx.rollbackTransaction();
-    throw error;
-  } finally {
-    await dbTx.release();
-  }
-
+  await indexer.resetWatcherToBlock(argv.blockNumber);
   log('Reset watcher successfully');
 };

packages/codegen/src/templates/server-template.handlebars

@@ -76,6 +76,8 @@ export const main = async (): Promise<any> => {
 
   if (watcherKind === KIND_ACTIVE) {
     await jobQueue.start();
+    // Delete jobs to prevent creating jobs after completion of processing previous block.
+    await jobQueue.deleteAllJobs();
     await eventWatcher.start();
   }