Implement eden-watcher changes in other watchers and codegen (#192)

* Implement eden-watcher changes in other watchers and codegen

* Use node space size only for eden-watcher

packages/codegen/src/checkpoint.ts

@@ -7,15 +7,29 @@ import path from 'path';
 import Handlebars from 'handlebars';
 import { Writable } from 'stream';
 
-const TEMPLATE_FILE = './templates/checkpoint-template.handlebars';
+const CHECKPOINT_TEMPLATE_FILE = './templates/checkpoint-template.handlebars';
+const CREATE_TEMPLATE_FILE = './templates/checkpoint-create-template.handlebars';
+const VERIFY_TEMPLATE_FILE = './templates/checkpoint-verify-template.handlebars';
 
 /**
  * Writes the checkpoint file generated from a template to a stream.
  * @param outStream A writable output stream to write the checkpoint file to.
  */
-export function exportCheckpoint (outStream: Writable, subgraphPath: string): void {
+export function exportCheckpoint (checkpointOutStream: Writable, checkpointCreateOutStream: Writable, checkpointVerifyOutStream: Writable | undefined, subgraphPath: string): void {
-  const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString();
+  const checkpointTemplateString = fs.readFileSync(path.resolve(__dirname, CHECKPOINT_TEMPLATE_FILE)).toString();
-  const template = Handlebars.compile(templateString);
+  const checkpointTemplate = Handlebars.compile(checkpointTemplateString);
-  const checkpoint = template({ subgraphPath });
+  const checkpoint = checkpointTemplate({ subgraphPath });
-  outStream.write(checkpoint);
+  checkpointOutStream.write(checkpoint);
+
+  const createCheckpointTemplateString = fs.readFileSync(path.resolve(__dirname, CREATE_TEMPLATE_FILE)).toString();
+  const createCheckpointTemplate = Handlebars.compile(createCheckpointTemplateString);
+  const createCheckpoint = createCheckpointTemplate({ subgraphPath });
+  checkpointCreateOutStream.write(createCheckpoint);
+
+  if (checkpointVerifyOutStream) {
+    const verifyCheckpointTemplateString = fs.readFileSync(path.resolve(__dirname, VERIFY_TEMPLATE_FILE)).toString();
+    const verifyCheckpointTemplate = Handlebars.compile(verifyCheckpointTemplateString);
+    const verifyCheckpointString = verifyCheckpointTemplate({});
+    checkpointVerifyOutStream.write(verifyCheckpointString);
+  }
 }

packages/codegen/src/generate-code.ts

@@ -153,6 +153,9 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
 
     const resetCmdsFolder = path.join(outputDir, 'src/cli/reset-cmds');
     if (!fs.existsSync(resetCmdsFolder)) fs.mkdirSync(resetCmdsFolder, { recursive: true });
+
+    const checkpointCmdsFolder = path.join(outputDir, 'src/cli/checkpoint-cmds');
+    if (!fs.existsSync(checkpointCmdsFolder)) fs.mkdirSync(checkpointCmdsFolder, { recursive: true });
   }
 
   let outStream: Writable;
@@ -236,10 +239,23 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
     : process.stdout;
   exportWatchContract(outStream, config.subgraphPath);
 
-  outStream = outputDir
+  let checkpointOutStream, checkpointCreateOutStream, checkpointVerifyOutStream;
-    ? fs.createWriteStream(path.join(outputDir, 'src/cli/checkpoint.ts'))
+
-    : process.stdout;
+  if (outputDir) {
-  exportCheckpoint(outStream, config.subgraphPath);
+    checkpointOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/checkpoint.ts'));
+    checkpointCreateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/checkpoint-cmds/create.ts'));
+    if (config.subgraphPath) {
+      checkpointVerifyOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/checkpoint-cmds/verify.ts'));
+    }
+  } else {
+    checkpointOutStream = process.stdout;
+    checkpointCreateOutStream = process.stdout;
+    if (config.subgraphPath) {
+      checkpointVerifyOutStream = process.stdout;
+    }
+  }
+
+  exportCheckpoint(checkpointOutStream, checkpointCreateOutStream, checkpointVerifyOutStream, config.subgraphPath);
 
   outStream = outputDir
     ? fs.createWriteStream(path.join(outputDir, 'src/hooks.ts'))

packages/codegen/src/templates/checkpoint-create-template.handlebars

@@ -0,0 +1,75 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+{{#if (subgraphPath)}}
+import path from 'path';
+{{/if}}
+import debug from 'debug';
+import assert from 'assert';
+
+import { getConfig, initClients, JobQueue, Config } from '@cerc-io/util';
+{{#if (subgraphPath)}}
+import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
+{{/if}}
+
+import { Database } from '../../database';
+import { Indexer } from '../../indexer';
+
+const log = debug('vulcanize:checkpoint-create');
+
+export const command = 'create';
+
+export const desc = 'Create checkpoint';
+
+export const builder = {
+  address: {
+    type: 'string',
+    require: true,
+    demandOption: true,
+    describe: 'Contract address to create the checkpoint for.'
+  },
+  blockHash: {
+    type: 'string',
+    describe: 'Blockhash at which to create the checkpoint.'
+  }
+};
+
+export const handler = async (argv: any): Promise<void> => {
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, ethProvider } = await initClients(config);
+
+  const db = new Database(config.database);
+  await db.init();
+
+  {{#if (subgraphPath)}}
+
+  const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
+  await graphDb.init();
+
+  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
+  {{/if}}
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
+
+  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
+  assert(dbConnectionString, 'Missing job queue db connection string');
+
+  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
+  await jobQueue.start();
+
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
+  await indexer.init();
+  {{#if (subgraphPath)}}
+
+  graphWatcher.setIndexer(indexer);
+  await graphWatcher.init();
+  {{/if}}
+
+  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
+
+  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
+
+  await db.close();
+};

packages/codegen/src/templates/checkpoint-template.handlebars

@@ -2,26 +2,19 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-{{#if (subgraphPath)}}
-import path from 'path';
-{{/if}}
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import assert from 'assert';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@cerc-io/util';
+import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
-{{#if (subgraphPath)}}
-import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
-{{/if}}
 
-import { Database } from '../database';
+import { hideBin } from 'yargs/helpers';
-import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:checkpoint');
 
-const main = async (): Promise<void> => {
+const main = async () => {
-  const argv = await yargs.parserConfiguration({
+  return yargs(hideBin(process.argv))
+    .parserConfiguration({
       'parse-numbers': false
     }).options({
       configFile: {
@@ -29,60 +22,18 @@ const main = async (): Promise<void> => {
         type: 'string',
         require: true,
         demandOption: true,
-      describe: 'Configuration file path (toml)',
+        describe: 'configuration file path (toml)',
         default: DEFAULT_CONFIG_PATH
-    },
-    address: {
-      type: 'string',
-      require: true,
-      demandOption: true,
-      describe: 'Contract address to create the checkpoint for.'
-    },
-    blockHash: {
-      type: 'string',
-      describe: 'Blockhash at which to create the checkpoint.'
       }
-  }).argv;
+    })
-
+    .commandDir('checkpoint-cmds', { extensions: ['ts', 'js'], exclude: /([a-zA-Z0-9\s_\\.\-:])+(.d.ts)$/ })
-  const config: Config = await getConfig(argv.configFile);
+    .demandCommand(1)
-  const { ethClient, ethProvider } = await initClients(config);
+    .help()
-
+    .argv;
-  const db = new Database(config.database);
-  await db.init();
-  {{#if (subgraphPath)}}
-
-  const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../entity/*'));
-  await graphDb.init();
-
-  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
-  {{/if}}
-
-  const jobQueueConfig = config.jobQueue;
-  assert(jobQueueConfig, 'Missing job queue config');
-
-  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
-  assert(dbConnectionString, 'Missing job queue db connection string');
-
-  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
-  await jobQueue.start();
-
-  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
-  await indexer.init();
-  {{#if (subgraphPath)}}
-
-  graphWatcher.setIndexer(indexer);
-  await graphWatcher.init();
-  {{/if}}
-
-  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
-
-  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
-
-  await db.close();
 };
 
-main().catch(err => {
+main().then(() => {
+  process.exit();
+}).catch(err => {
   log(err);
-}).finally(() => {
-  process.exit(0);
 });

packages/codegen/src/templates/checkpoint-verify-template.handlebars

@@ -0,0 +1,66 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import path from 'path';
+import debug from 'debug';
+import assert from 'assert';
+
+import { getConfig, initClients, JobQueue, Config, verifyCheckpointData } from '@cerc-io/util';
+import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
+
+import { Database } from '../../database';
+import { Indexer } from '../../indexer';
+
+const log = debug('vulcanize:checkpoint-verify');
+
+export const command = 'verify';
+
+export const desc = 'Verify checkpoint';
+
+export const builder = {
+  cid: {
+    alias: 'c',
+    type: 'string',
+    demandOption: true,
+    describe: 'Checkpoint CID to be verified'
+  }
+};
+
+export const handler = async (argv: any): Promise<void> => {
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, ethProvider } = await initClients(config);
+
+  const db = new Database(config.database);
+  await db.init();
+
+  const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
+  await graphDb.init();
+
+  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
+
+  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
+  assert(dbConnectionString, 'Missing job queue db connection string');
+
+  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
+  await jobQueue.start();
+
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
+  await indexer.init();
+
+  graphWatcher.setIndexer(indexer);
+  await graphWatcher.init();
+
+  const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
+  assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
+  const data = indexer.getIPLDData(ipldBlock);
+
+  log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`);
+  await verifyCheckpointData(graphDb, ipldBlock.block, data);
+  log('Checkpoint data verified');
+
+  await db.close();
+};

packages/codegen/src/templates/export-state-template.handlebars

@@ -16,7 +16,6 @@ import {
   initClients,
   JobQueue,
   {{#if (subgraphPath)}}
-  verifyCheckpointData,
   {{/if}}
   StateKind
 } from '@cerc-io/util';
@@ -42,18 +41,14 @@ const main = async (): Promise<void> => {
       describe: 'Configuration file path (toml)',
       default: DEFAULT_CONFIG_PATH
     },
-    {{#if (subgraphPath)}}
-    verify: {
-      alias: 'v',
-      type: 'boolean',
-      describe: 'Verify checkpoint',
-      default: true
-    },
-    {{/if}}
     exportFile: {
       alias: 'o',
       type: 'string',
       describe: 'Export file path'
+    },
+    blockNumber: {
+      type: 'number',
+      describe: 'Block number to create snapshot at'
     }
   }).argv;
 
@@ -96,9 +91,25 @@ const main = async (): Promise<void> => {
   const contracts = await db.getContracts();
 
   // Get latest block with hooks processed.
-  const block = await indexer.getLatestHooksProcessedBlock();
+  let block = await indexer.getLatestHooksProcessedBlock();
   assert(block);
 
+  if (argv.blockNumber) {
+    if (argv.blockNumber > block.blockNumber) {
+      throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
+    }
+
+    const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
+
+    if (!blocksAtSnapshotHeight.length) {
+      throw new Error(`No blocks at snapshot height ${argv.blockNumber}`);
+    }
+
+    block = blocksAtSnapshotHeight[0];
+  }
+
+  log(`Creating export snapshot at block height ${block.blockNumber}`);
+
   // Export snapshot block.
   exportData.snapshotBlock = {
     blockNumber: block.blockNumber,
@@ -123,14 +134,6 @@ const main = async (): Promise<void> => {
 
       const data = indexer.getIPLDData(ipldBlock);
 
-      {{#if (subgraphPath)}}
-      if (argv.verify) {
-        log(`Verifying checkpoint data for contract ${contract.address}`);
-        await verifyCheckpointData(graphDb, ipldBlock.block, data);
-        log('Checkpoint data verified');
-      }
-
-      {{/if}}
       if (indexer.isIPFSConfigured()) {
         await indexer.pushToIPFS(data);
       }

packages/codegen/src/templates/import-state-template.handlebars

@@ -126,6 +126,8 @@ export const main = async (): Promise<any> => {
   await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
   await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
   await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
+  await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
+  await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
 
   // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
   await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);

packages/codegen/src/templates/package-template.handlebars

@@ -11,17 +11,20 @@
     "copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
     "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
     "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
-    "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
+    "job-runner": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true node --enable-source-maps dist/job-runner.js",
-    "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
+    "job-runner:dev": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true ts-node src/job-runner.ts",
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
     {{#if (subgraphPath)}}
     "fill:state": "DEBUG=vulcanize:* ts-node src/fill.ts --state",
     {{/if}}
     "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
-    "checkpoint": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
+    "checkpoint": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/checkpoint.js",
-    "export-state": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "checkpoint:dev": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
-    "import-state": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
+    "export-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/export-state.js",
+    "export-state:dev": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "import-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/import-state.js",
+    "import-state:dev": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
     "inspect-cid": "DEBUG=vulcanize:* ts-node src/cli/inspect-cid.ts",
     "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts"
   },

packages/codegen/src/templates/readme-template.handlebars

@@ -120,12 +120,22 @@ GQL console: http://localhost:{{port}}/graphql
   * To create a checkpoint for a contract:
 
     ```bash
-    yarn checkpoint --address <contract-address> --block-hash [block-hash]
+    yarn checkpoint create --address <contract-address> --block-hash [block-hash]
     ```
 
     * `address`: Address or identifier of the contract for which to create a checkpoint.
     * `block-hash`: Hash of a block (in the pruned region) at which to create the checkpoint (default: latest canonical block hash).
 
+  {{#if (subgraphPath)}}
+  * To verify a checkpoint:
+
+    ```bash
+    yarn checkpoint verify --cid <checkpoint-cid>
+    ```
+
+    `cid`: CID of the checkpoint for which to verify.
+  
+  {{/if}}
   * To reset the watcher to a previous block number:
 
     * Reset state:
@@ -147,10 +157,11 @@ GQL console: http://localhost:{{port}}/graphql
     * In source watcher, export watcher state:
 
       ```bash
-      yarn export-state --export-file [export-file-path]
+      yarn export-state --export-file [export-file-path] --block-number [snapshot-block-height]
       ```
 
       * `export-file`: Path of file to which to export the watcher data.
+      * `block-number`: Block height at which to take snapshot for export.
 
     * In target watcher, run job-runner:
 

packages/eden-watcher/package.json

@@ -11,7 +11,7 @@
     "copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
     "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
     "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
-    "job-runner": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true node --enable-source-maps dist/job-runner.js",
+    "job-runner": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true node --max-old-space-size=3072 --enable-source-maps dist/job-runner.js",
     "job-runner:dev": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true ts-node src/job-runner.ts",
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",

packages/eden-watcher/src/hooks.ts

@@ -85,9 +85,6 @@ export async function createStateCheckpoint (indexer: Indexer, contractAddress:
     diffStartBlockNumber = initBlock.block.blockNumber - 1;
   }
 
-  // Fetching all diff blocks after the latest 'checkpoint' | 'init'.
-  const diffBlocks = await indexer.getDiffIPLDBlocksInRange(contractAddress, diffStartBlockNumber, block.blockNumber);
-
   const prevNonDiffBlockData = codec.decode(Buffer.from(prevNonDiffBlock.data)) as any;
   const data = {
     state: prevNonDiffBlockData.state
@@ -95,6 +92,7 @@ export async function createStateCheckpoint (indexer: Indexer, contractAddress:
 
   console.time('time:hooks#createStateCheckpoint');
 
+  // Fetching and merging all diff blocks after the latest 'checkpoint' | 'init' in batch.
   for (let i = diffStartBlockNumber; i < block.blockNumber;) {
     const endBlockHeight = Math.min(i + IPLD_BATCH_BLOCKS, block.blockNumber);
     console.time(`time:hooks#createStateCheckpoint-batch-merge-diff-${i}-${endBlockHeight}`);

packages/erc721-watcher/README.md

@@ -128,7 +128,7 @@ GQL console: http://localhost:3006/graphql
   * To create a checkpoint for a contract:
 
     ```bash
-    yarn checkpoint --address <contract-address> --block-hash [block-hash]
+    yarn checkpoint create --address <contract-address> --block-hash [block-hash]
     ```
 
     * `address`: Address or identifier of the contract for which to create a checkpoint.
@@ -155,10 +155,11 @@ GQL console: http://localhost:3006/graphql
     * In source watcher, export watcher state:
 
       ```bash
-      yarn export-state --export-file [export-file-path]
+      yarn export-state --export-file [export-file-path] --block-number [snapshot-block-height]
       ```
 
       * `export-file`: Path of file to which to export the watcher data.
+      * `block-number`: Block height at which to take snapshot for export.
 
     * In target watcher, run job-runner:
 

packages/erc721-watcher/demo.md

@@ -373,7 +373,7 @@
 * After the `diff` block has been created (can check if event block number pruned in yarn server log), create a checkpoint using CLI in `packages/erc721-watcher`:
 
   ```bash
-  yarn checkpoint --address $NFT_ADDRESS
+  yarn checkpoint create --address $NFT_ADDRESS
   ```
 
   * Run the `getState` query again with the output blockHash and kind `checkpoint` at the endpoint.

packages/erc721-watcher/package.json

@@ -16,9 +16,12 @@
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
     "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
-    "checkpoint": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
+    "checkpoint": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/checkpoint.js",
-    "export-state": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "checkpoint:dev": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
-    "import-state": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
+    "export-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/export-state.js",
+    "export-state:dev": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "import-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/import-state.js",
+    "import-state:dev": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
     "inspect-cid": "DEBUG=vulcanize:* ts-node src/cli/inspect-cid.ts",
     "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts",
     "nft:deploy": "hardhat --network localhost nft-deploy",

packages/erc721-watcher/src/cli/checkpoint-cmds/create.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import debug from 'debug';
+import assert from 'assert';
+
+import { getConfig, initClients, JobQueue, Config } from '@cerc-io/util';
+
+import { Database } from '../../database';
+import { Indexer } from '../../indexer';
+
+const log = debug('vulcanize:checkpoint-create');
+
+export const command = 'create';
+
+export const desc = 'Create checkpoint';
+
+export const builder = {
+  address: {
+    type: 'string',
+    require: true,
+    demandOption: true,
+    describe: 'Contract address to create the checkpoint for.'
+  },
+  blockHash: {
+    type: 'string',
+    describe: 'Blockhash at which to create the checkpoint.'
+  }
+};
+
+export const handler = async (argv: any): Promise<void> => {
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, ethProvider } = await initClients(config);
+
+  const db = new Database(config.database);
+  await db.init();
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
+
+  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
+  assert(dbConnectionString, 'Missing job queue db connection string');
+
+  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
+  await jobQueue.start();
+
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
+  await indexer.init();
+
+  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
+
+  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
+
+  await db.close();
+};

packages/erc721-watcher/src/cli/checkpoint.ts

@@ -5,17 +5,16 @@
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import assert from 'assert';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@cerc-io/util';
+import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
 
-import { Database } from '../database';
+import { hideBin } from 'yargs/helpers';
-import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:checkpoint');
 
-const main = async (): Promise<void> => {
+const main = async () => {
-  const argv = await yargs.parserConfiguration({
+  return yargs(hideBin(process.argv))
+    .parserConfiguration({
       'parse-numbers': false
     }).options({
       configFile: {
@@ -23,48 +22,18 @@ const main = async (): Promise<void> => {
         type: 'string',
         require: true,
         demandOption: true,
-      describe: 'Configuration file path (toml)',
+        describe: 'configuration file path (toml)',
         default: DEFAULT_CONFIG_PATH
-    },
-    address: {
-      type: 'string',
-      require: true,
-      demandOption: true,
-      describe: 'Contract address to create the checkpoint for.'
-    },
-    blockHash: {
-      type: 'string',
-      describe: 'Blockhash at which to create the checkpoint.'
       }
-  }).argv;
+    })
-
+    .commandDir('checkpoint-cmds', { extensions: ['ts', 'js'], exclude: /([a-zA-Z0-9\s_\\.\-:])+(.d.ts)$/ })
-  const config: Config = await getConfig(argv.configFile);
+    .demandCommand(1)
-  const { ethClient, ethProvider } = await initClients(config);
+    .help()
-
+    .argv;
-  const db = new Database(config.database);
-  await db.init();
-
-  const jobQueueConfig = config.jobQueue;
-  assert(jobQueueConfig, 'Missing job queue config');
-
-  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
-  assert(dbConnectionString, 'Missing job queue db connection string');
-
-  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
-  await jobQueue.start();
-
-  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
-  await indexer.init();
-
-  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
-
-  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
-
-  await db.close();
 };
 
-main().catch(err => {
+main().then(() => {
+  process.exit();
+}).catch(err => {
   log(err);
-}).finally(() => {
-  process.exit(0);
 });

packages/erc721-watcher/src/cli/export-state.ts

@@ -33,6 +33,10 @@ const main = async (): Promise<void> => {
       alias: 'o',
       type: 'string',
       describe: 'Export file path'
+    },
+    blockNumber: {
+      type: 'number',
+      describe: 'Block number to create snapshot at'
     }
   }).argv;
 
@@ -61,11 +65,25 @@ const main = async (): Promise<void> => {
   };
 
   const contracts = await db.getContracts();
-
+  let block = await indexer.getLatestHooksProcessedBlock();
-  // Get latest block with hooks processed.
-  const block = await indexer.getLatestHooksProcessedBlock();
   assert(block);
 
+  if (argv.blockNumber) {
+    if (argv.blockNumber > block.blockNumber) {
+      throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
+    }
+
+    const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
+
+    if (!blocksAtSnapshotHeight.length) {
+      throw new Error(`No blocks at snapshot height ${argv.blockNumber}`);
+    }
+
+    block = blocksAtSnapshotHeight[0];
+  }
+
+  log(`Creating export snapshot at block height ${block.blockNumber}`);
+
   // Export snapshot block.
   exportData.snapshotBlock = {
     blockNumber: block.blockNumber,

packages/erc721-watcher/src/cli/import-state.ts

@@ -100,7 +100,7 @@ export const main = async (): Promise<any> => {
 
     ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
 
-    await indexer.saveOrUpdateIPLDBlock(ipldBlock);
+    ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
   }
 
   // Mark snapshot block as completely processed.
@@ -108,6 +108,8 @@ export const main = async (): Promise<any> => {
   await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
   await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
   await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
+  await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
+  await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
 
   // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
   await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);

packages/graph-test-watcher/README.md

@@ -122,12 +122,20 @@ GQL console: http://localhost:3008/graphql
   * To create a checkpoint for a contract:
 
     ```bash
-    yarn checkpoint --address <contract-address> --block-hash [block-hash]
+    yarn checkpoint create --address <contract-address> --block-hash [block-hash]
     ```
 
     * `address`: Address or identifier of the contract for which to create a checkpoint.
     * `block-hash`: Hash of a block (in the pruned region) at which to create the checkpoint (default: latest canonical block hash).
 
+  * To verify a checkpoint:
+
+    ```bash
+    yarn checkpoint verify --cid <checkpoint-cid>
+    ```
+
+    `cid`: CID of the checkpoint for which to verify.
+
   * To reset the watcher to a previous block number:
 
     * Reset state:
@@ -149,10 +157,11 @@ GQL console: http://localhost:3008/graphql
     * In source watcher, export watcher state:
 
       ```bash
-      yarn export-state --export-file [export-file-path]
+      yarn export-state --export-file [export-file-path] --block-number [snapshot-block-height]
       ```
 
       * `export-file`: Path of file to which to export the watcher data.
+      * `block-number`: Block height at which to take snapshot for export.
 
     * In target watcher, run job-runner:
 

packages/graph-test-watcher/package.json

@@ -15,10 +15,14 @@
     "job-runner:dev": "DEBUG=vulcanize:* YARN_CHILD_PROCESS=true ts-node src/job-runner.ts",
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
+    "fill:state": "DEBUG=vulcanize:* ts-node src/fill.ts --state",
     "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
-    "checkpoint": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
+    "checkpoint": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/checkpoint.js",
-    "export-state": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "checkpoint:dev": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
-    "import-state": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
+    "export-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/export-state.js",
+    "export-state:dev": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "import-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/import-state.js",
+    "import-state:dev": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
     "inspect-cid": "DEBUG=vulcanize:* ts-node src/cli/inspect-cid.ts",
     "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts"
   },

packages/graph-test-watcher/src/cli/checkpoint-cmds/create.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import path from 'path';
+import debug from 'debug';
+import assert from 'assert';
+
+import { getConfig, initClients, JobQueue, Config } from '@cerc-io/util';
+import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
+
+import { Database } from '../../database';
+import { Indexer } from '../../indexer';
+
+const log = debug('vulcanize:checkpoint-create');
+
+export const command = 'create';
+
+export const desc = 'Create checkpoint';
+
+export const builder = {
+  address: {
+    type: 'string',
+    require: true,
+    demandOption: true,
+    describe: 'Contract address to create the checkpoint for.'
+  },
+  blockHash: {
+    type: 'string',
+    describe: 'Blockhash at which to create the checkpoint.'
+  }
+};
+
+export const handler = async (argv: any): Promise<void> => {
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, ethProvider } = await initClients(config);
+
+  const db = new Database(config.database);
+  await db.init();
+
+  const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
+  await graphDb.init();
+
+  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
+
+  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
+  assert(dbConnectionString, 'Missing job queue db connection string');
+
+  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
+  await jobQueue.start();
+
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
+  await indexer.init();
+
+  graphWatcher.setIndexer(indexer);
+  await graphWatcher.init();
+
+  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
+
+  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
+
+  await db.close();
+};

packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import path from 'path';
+import debug from 'debug';
+import assert from 'assert';
+
+import { getConfig, initClients, JobQueue, Config, verifyCheckpointData } from '@cerc-io/util';
+import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
+
+import { Database } from '../../database';
+import { Indexer } from '../../indexer';
+
+const log = debug('vulcanize:checkpoint-verify');
+
+export const command = 'verify';
+
+export const desc = 'Verify checkpoint';
+
+export const builder = {
+  cid: {
+    alias: 'c',
+    type: 'string',
+    demandOption: true,
+    describe: 'Checkpoint CID to be verified'
+  }
+};
+
+export const handler = async (argv: any): Promise<void> => {
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, ethProvider } = await initClients(config);
+
+  const db = new Database(config.database);
+  await db.init();
+
+  const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*'));
+  await graphDb.init();
+
+  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
+
+  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
+  assert(dbConnectionString, 'Missing job queue db connection string');
+
+  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
+  await jobQueue.start();
+
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
+  await indexer.init();
+
+  graphWatcher.setIndexer(indexer);
+  await graphWatcher.init();
+
+  const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
+  assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');
+  const data = indexer.getIPLDData(ipldBlock);
+
+  log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`);
+  await verifyCheckpointData(graphDb, ipldBlock.block, data);
+  log('Checkpoint data verified');
+
+  await db.close();
+};

packages/graph-test-watcher/src/cli/checkpoint.ts

@@ -2,22 +2,19 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import path from 'path';
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import assert from 'assert';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@cerc-io/util';
+import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
-import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 
-import { Database } from '../database';
+import { hideBin } from 'yargs/helpers';
-import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:checkpoint');
 
-const main = async (): Promise<void> => {
+const main = async () => {
-  const argv = await yargs.parserConfiguration({
+  return yargs(hideBin(process.argv))
+    .parserConfiguration({
       'parse-numbers': false
     }).options({
       configFile: {
@@ -25,56 +22,18 @@ const main = async (): Promise<void> => {
         type: 'string',
         require: true,
         demandOption: true,
-      describe: 'Configuration file path (toml)',
+        describe: 'configuration file path (toml)',
         default: DEFAULT_CONFIG_PATH
-    },
-    address: {
-      type: 'string',
-      require: true,
-      demandOption: true,
-      describe: 'Contract address to create the checkpoint for.'
-    },
-    blockHash: {
-      type: 'string',
-      describe: 'Blockhash at which to create the checkpoint.'
       }
-  }).argv;
+    })
-
+    .commandDir('checkpoint-cmds', { extensions: ['ts', 'js'], exclude: /([a-zA-Z0-9\s_\\.\-:])+(.d.ts)$/ })
-  const config: Config = await getConfig(argv.configFile);
+    .demandCommand(1)
-  const { ethClient, ethProvider } = await initClients(config);
+    .help()
-
+    .argv;
-  const db = new Database(config.database);
-  await db.init();
-
-  const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../entity/*'));
-  await graphDb.init();
-
-  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
-
-  const jobQueueConfig = config.jobQueue;
-  assert(jobQueueConfig, 'Missing job queue config');
-
-  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
-  assert(dbConnectionString, 'Missing job queue db connection string');
-
-  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
-  await jobQueue.start();
-
-  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
-  await indexer.init();
-
-  graphWatcher.setIndexer(indexer);
-  await graphWatcher.init();
-
-  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
-
-  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
-
-  await db.close();
 };
 
-main().catch(err => {
+main().then(() => {
+  process.exit();
+}).catch(err => {
   log(err);
-}).finally(() => {
-  process.exit(0);
 });

packages/graph-test-watcher/src/cli/export-state.ts

@@ -9,7 +9,7 @@ import debug from 'debug';
 import fs from 'fs';
 import path from 'path';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind, verifyCheckpointData } from '@cerc-io/util';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util';
 import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 import * as codec from '@ipld/dag-cbor';
 
@@ -35,11 +35,9 @@ const main = async (): Promise<void> => {
       type: 'string',
       describe: 'Export file path'
     },
-    verify: {
+    blockNumber: {
-      alias: 'v',
+      type: 'number',
-      type: 'boolean',
+      describe: 'Block number to create snapshot at'
-      describe: 'Verify checkpoint',
-      default: true
     }
   }).argv;
 
@@ -76,11 +74,25 @@ const main = async (): Promise<void> => {
   };
 
   const contracts = await db.getContracts();
-
+  let block = await indexer.getLatestHooksProcessedBlock();
-  // Get latest block with hooks processed.
-  const block = await indexer.getLatestHooksProcessedBlock();
   assert(block);
 
+  if (argv.blockNumber) {
+    if (argv.blockNumber > block.blockNumber) {
+      throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
+    }
+
+    const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
+
+    if (!blocksAtSnapshotHeight.length) {
+      throw new Error(`No blocks at snapshot height ${argv.blockNumber}`);
+    }
+
+    block = blocksAtSnapshotHeight[0];
+  }
+
+  log(`Creating export snapshot at block height ${block.blockNumber}`);
+
   // Export snapshot block.
   exportData.snapshotBlock = {
     blockNumber: block.blockNumber,
@@ -105,12 +117,6 @@ const main = async (): Promise<void> => {
 
       const data = indexer.getIPLDData(ipldBlock);
 
-      if (argv.verify) {
-        log(`Verifying checkpoint data for contract ${contract.address}`);
-        await verifyCheckpointData(graphDb, ipldBlock.block, data);
-        log('Checkpoint data verified');
-      }
-
       if (indexer.isIPFSConfigured()) {
         await indexer.pushToIPFS(data);
       }

packages/graph-test-watcher/src/cli/import-state.ts

@@ -118,6 +118,8 @@ export const main = async (): Promise<any> => {
   await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
   await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
   await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
+  await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
+  await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
 
   // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
   await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);

packages/mobymask-watcher/README.md

@@ -122,7 +122,7 @@ GQL console: http://localhost:3010/graphql
   * To create a checkpoint for a contract:
 
     ```bash
-    yarn checkpoint --address <contract-address> --block-hash [block-hash]
+    yarn checkpoint create --address <contract-address> --block-hash [block-hash]
     ```
 
     * `address`: Address or identifier of the contract for which to create a checkpoint.
@@ -149,10 +149,11 @@ GQL console: http://localhost:3010/graphql
     * In source watcher, export watcher state:
 
       ```bash
-      yarn export-state --export-file [export-file-path]
+      yarn export-state --export-file [export-file-path] --block-number [snapshot-block-height]
       ```
 
       * `export-file`: Path of file to which to export the watcher data.
+      * `block-number`: Block height at which to take snapshot for export.
 
     * In target watcher, run job-runner:
 

packages/mobymask-watcher/package.json

@@ -16,9 +16,12 @@
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
     "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
-    "checkpoint": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
+    "checkpoint": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/checkpoint.js",
-    "export-state": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "checkpoint:dev": "DEBUG=vulcanize:* ts-node src/cli/checkpoint.ts",
-    "import-state": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
+    "export-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/export-state.js",
+    "export-state:dev": "DEBUG=vulcanize:* ts-node src/cli/export-state.ts",
+    "import-state": "DEBUG=vulcanize:* node --enable-source-maps dist/cli/import-state.js",
+    "import-state:dev": "DEBUG=vulcanize:* ts-node src/cli/import-state.ts",
     "inspect-cid": "DEBUG=vulcanize:* ts-node src/cli/inspect-cid.ts",
     "index-block": "DEBUG=vulcanize:* ts-node src/cli/index-block.ts"
   },

packages/mobymask-watcher/src/cli/checkpoint-cmds/create.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import debug from 'debug';
+import assert from 'assert';
+
+import { getConfig, initClients, JobQueue, Config } from '@cerc-io/util';
+
+import { Database } from '../../database';
+import { Indexer } from '../../indexer';
+
+const log = debug('vulcanize:checkpoint-create');
+
+export const command = 'create';
+
+export const desc = 'Create checkpoint';
+
+export const builder = {
+  address: {
+    type: 'string',
+    require: true,
+    demandOption: true,
+    describe: 'Contract address to create the checkpoint for.'
+  },
+  blockHash: {
+    type: 'string',
+    describe: 'Blockhash at which to create the checkpoint.'
+  }
+};
+
+export const handler = async (argv: any): Promise<void> => {
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, ethProvider } = await initClients(config);
+
+  const db = new Database(config.database);
+  await db.init();
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
+
+  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
+  assert(dbConnectionString, 'Missing job queue db connection string');
+
+  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
+  await jobQueue.start();
+
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
+  await indexer.init();
+
+  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
+
+  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
+
+  await db.close();
+};

packages/mobymask-watcher/src/cli/checkpoint.ts

@@ -5,17 +5,16 @@
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import assert from 'assert';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@cerc-io/util';
+import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
 
-import { Database } from '../database';
+import { hideBin } from 'yargs/helpers';
-import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:checkpoint');
 
-const main = async (): Promise<void> => {
+const main = async () => {
-  const argv = await yargs.parserConfiguration({
+  return yargs(hideBin(process.argv))
+    .parserConfiguration({
       'parse-numbers': false
     }).options({
       configFile: {
@@ -23,48 +22,18 @@ const main = async (): Promise<void> => {
         type: 'string',
         require: true,
         demandOption: true,
-      describe: 'Configuration file path (toml)',
+        describe: 'configuration file path (toml)',
         default: DEFAULT_CONFIG_PATH
-    },
-    address: {
-      type: 'string',
-      require: true,
-      demandOption: true,
-      describe: 'Contract address to create the checkpoint for.'
-    },
-    blockHash: {
-      type: 'string',
-      describe: 'Blockhash at which to create the checkpoint.'
       }
-  }).argv;
+    })
-
+    .commandDir('checkpoint-cmds', { extensions: ['ts', 'js'], exclude: /([a-zA-Z0-9\s_\\.\-:])+(.d.ts)$/ })
-  const config: Config = await getConfig(argv.configFile);
+    .demandCommand(1)
-  const { ethClient, ethProvider } = await initClients(config);
+    .help()
-
+    .argv;
-  const db = new Database(config.database);
-  await db.init();
-
-  const jobQueueConfig = config.jobQueue;
-  assert(jobQueueConfig, 'Missing job queue config');
-
-  const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
-  assert(dbConnectionString, 'Missing job queue db connection string');
-
-  const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
-  await jobQueue.start();
-
-  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
-  await indexer.init();
-
-  const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
-
-  log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);
-
-  await db.close();
 };
 
-main().catch(err => {
+main().then(() => {
+  process.exit();
+}).catch(err => {
   log(err);
-}).finally(() => {
-  process.exit(0);
 });

packages/mobymask-watcher/src/cli/export-state.ts

@@ -33,6 +33,10 @@ const main = async (): Promise<void> => {
       alias: 'o',
       type: 'string',
       describe: 'Export file path'
+    },
+    blockNumber: {
+      type: 'number',
+      describe: 'Block number to create snapshot at'
     }
   }).argv;
 
@@ -61,11 +65,25 @@ const main = async (): Promise<void> => {
   };
 
   const contracts = await db.getContracts();
-
+  let block = await indexer.getLatestHooksProcessedBlock();
-  // Get latest block with hooks processed.
-  const block = await indexer.getLatestHooksProcessedBlock();
   assert(block);
 
+  if (argv.blockNumber) {
+    if (argv.blockNumber > block.blockNumber) {
+      throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`);
+    }
+
+    const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false);
+
+    if (!blocksAtSnapshotHeight.length) {
+      throw new Error(`No blocks at snapshot height ${argv.blockNumber}`);
+    }
+
+    block = blocksAtSnapshotHeight[0];
+  }
+
+  log(`Creating export snapshot at block height ${block.blockNumber}`);
+
   // Export snapshot block.
   exportData.snapshotBlock = {
     blockNumber: block.blockNumber,

packages/mobymask-watcher/src/cli/import-state.ts

@@ -100,7 +100,7 @@ export const main = async (): Promise<any> => {
 
     ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data));
 
-    await indexer.saveOrUpdateIPLDBlock(ipldBlock);
+    ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock);
   }
 
   // Mark snapshot block as completely processed.
@@ -108,6 +108,8 @@ export const main = async (): Promise<any> => {
   await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
   await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber);
   await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber);
+  await indexer.updateIPLDStatusHooksBlock(block.blockNumber);
+  await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber);
 
   // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block.
   await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init);

packages/util/src/block-size-cache.ts

@@ -45,9 +45,15 @@ const cacheBlockSizesAsync = async (provider: providers.JsonRpcProvider, blockNu
     // Start prefetching blocks after latest height in blockSizeMap.
     for (let i = startBlockHeight; i <= endBlockHeight; i++) {
       console.time(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
-      const { size, hash } = await provider.send('eth_getBlockByNumber', [utils.hexStripZeros(utils.hexlify(i)), false]);
+      const block = await provider.send('eth_getBlockByNumber', [utils.hexStripZeros(utils.hexlify(i)), false]);
-      console.timeEnd(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
+
+      if (block) {
+        const { size, hash } = block;
         blockSizeMap.set(hash, { size, blockNumber: i });
+      } else {
+        log(`No block found at height ${i}`);
+      }
+      console.timeEnd(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
     }
   }