Update watcher import CLI to create entities from checkpoint (#186)

* Import entities from checkpoint

* Fix IPLD state when updating subgraph entity

* Changes in codegen and other watchers

* Update IPLD state with all Block entities

* Add verify and create sub commands to checkpoint

* Add option for specifying snapshot block in export state CLI

packages/codegen/src/templates/export-state-template.handlebars

@@ -42,11 +42,6 @@ const main = async (): Promise<void> => {
       describe: 'Configuration file path (toml)',
       default: DEFAULT_CONFIG_PATH
     },
-    exportFile: {
-      alias: 'o',
-      type: 'string',
-      describe: 'Export file path'
-    },
     {{#if (subgraphPath)}}
     verify: {
       alias: 'v',
@@ -55,11 +50,10 @@ const main = async (): Promise<void> => {
       default: true
     },
     {{/if}}
-    createCheckpoint: {
-      alias: 'c',
-      type: 'boolean',
-      describe: 'Create new checkpoint',
-      default: false
+    exportFile: {
+      alias: 'o',
+      type: 'string',
+      describe: 'Export file path'
     }
   }).argv;
 
@@ -122,10 +116,7 @@ const main = async (): Promise<void> => {
 
     // Create and export checkpoint if checkpointing is on for the contract.
     if (contract.checkpoint) {
-      if (argv.createCheckpoint) {
-        log(`Creating checkpoint at block ${block.blockNumber}`);
-        await indexer.createCheckpoint(contract.address, block.blockHash);
-      }
+      await indexer.createCheckpoint(contract.address, block.blockHash);
 
       const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
       assert(ipldBlock);

packages/codegen/src/templates/import-state-template.handlebars

@@ -91,6 +91,7 @@ export const main = async (): Promise<any> => {
     eventWatcher,
     config.upstream.ethServer.blockDelayInMilliSecs,
     {
+      prefetch: true,
       startBlock: importData.snapshotBlock.blockNumber,
       endBlock: importData.snapshotBlock.blockNumber
     }
@@ -114,12 +115,23 @@ export const main = async (): Promise<any> => {
 
     ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
 
-    await indexer.saveOrUpdateIPLDBlock(ipldBlock);
+    ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
+    {{#if (subgraphPath)}}
+    await graphWatcher.updateEntitiesFromIPLDState(ipldBlock);
+    {{/if}}
   }
 
+  // Mark snapshot block as completely processed.
+  block.isComplete = true;
+  await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
+  await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
+  await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
+
   // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
   await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);
   await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged);
+
+  log(`Import completed for snapshot block at height ${block.blockNumber}`);
 };
 
 main().catch(err => {

packages/codegen/src/templates/indexer-template.handlebars

@@ -57,12 +57,12 @@ const JSONbigNative = JSONbig({ useNativeBigInt: true });
 
 {{#each contracts as | contract |}}
 const KIND_{{capitalize contract.contractName}} = '{{contract.contractKind}}';
-{{/each}}
 
+{{/each}}
 {{#each uniqueEvents as | event |}}
 const {{capitalize event}}_EVENT = '{{event}}';
-{{/each}}
 
+{{/each}}
 export type ResultEvent = {
   block: {
     cid: string;
@@ -443,8 +443,6 @@ export class Indexer implements IPLDIndexerInterface {
 
   {{#if (subgraphPath)}}
   async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
-    const relations = this._relationsMap.get(entity) || {};
-
     const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
 
     return data;

packages/codegen/src/templates/readme-template.handlebars

@@ -150,7 +150,7 @@ GQL console: http://localhost:{{port}}/graphql
       yarn export-state --export-file [export-file-path]
       ```
 
-      * `export-file`: Path of JSON file to which to export the watcher data.
+      * `export-file`: Path of file to which to export the watcher data.
 
     * In target watcher, run job-runner:
 
@@ -164,15 +164,7 @@ GQL console: http://localhost:{{port}}/graphql
       yarn import-state --import-file <import-file-path>
       ```
 
-      * `import-file`: Path of JSON file from which to import the watcher data.
-
-    * Run fill:
-
-      ```bash
-      yarn fill --start-block <snapshot-block> --end-block <to-block>
-      ```
-
-      * `snapshot-block`: Block number at which the watcher state was exported.
+      * `import-file`: Path of file from which to import the watcher data.
 
     * Run server: