Refactor fill and fill-state CLIs to cli package (#257)

* Refactor fill CLI to cli package

* Refactor method to fill-state to graph-node

* Refactor fill-state CLI to cli package

* Move subgraph state utils to a separate file

* Refactor subgraph state helper methods to graph-node

* Update mock indexer

* Move watcher job-runner to util

* Remove mock server and data from erc20-watcher

* Import watcher job-runner from util

packages/cli/src/fill.ts

@@ -0,0 +1,159 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import assert from 'assert';
+import 'reflect-metadata';
+import debug from 'debug';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { ConnectionOptions } from 'typeorm';
+import { PubSub } from 'graphql-subscriptions';
+
+import { JsonRpcProvider } from '@ethersproject/providers';
+import { GraphWatcher, fillState } from '@cerc-io/graph-node';
+import { EthClient } from '@cerc-io/ipld-eth-client';
+import {
+  DEFAULT_CONFIG_PATH,
+  JobQueue,
+  DatabaseInterface,
+  IndexerInterface,
+  ServerConfig,
+  Clients,
+  EventWatcherInterface,
+  fillBlocks
+} from '@cerc-io/util';
+
+import { BaseCmd } from './base';
+
+const log = debug('vulcanize:fill');
+
+interface Arguments {
+  configFile: string;
+  startBlock: number;
+  endBlock: number;
+  prefetch: boolean;
+  batchBlocks: number;
+  state: boolean;
+}
+
+export class FillCmd {
+  _argv?: Arguments
+  _baseCmd: BaseCmd;
+
+  constructor () {
+    this._baseCmd = new BaseCmd();
+  }
+
+  get indexer (): IndexerInterface | undefined {
+    return this._baseCmd.indexer;
+  }
+
+  async initConfig<ConfigType> (): Promise<ConfigType> {
+    this._argv = this._getArgv();
+    assert(this._argv);
+
+    return this._baseCmd.initConfig(this._argv.configFile);
+  }
+
+  async init (
+    Database: new (config: ConnectionOptions,
+      serverConfig?: ServerConfig
+    ) => DatabaseInterface,
+    Indexer: new (
+      serverConfig: ServerConfig,
+      db: DatabaseInterface,
+      clients: Clients,
+      ethProvider: JsonRpcProvider,
+      jobQueue: JobQueue,
+      graphWatcher?: GraphWatcher
+    ) => IndexerInterface,
+    EventWatcher: new(
+      ethClient: EthClient,
+      indexer: IndexerInterface,
+      pubsub: PubSub,
+      jobQueue: JobQueue
+    ) => EventWatcherInterface,
+    clients: { [key: string]: any } = {},
+    entityQueryTypeMap?: Map<any, any>,
+    entityToLatestEntityMap?: Map<any, any>
+  ): Promise<void> {
+    await this.initConfig();
+
+    await this._baseCmd.init(Database, Indexer, clients, entityQueryTypeMap, entityToLatestEntityMap);
+    await this._baseCmd.initEventWatcher(EventWatcher);
+  }
+
+  async exec (contractEntitiesMap: Map<string, string[]> = new Map()): Promise<void> {
+    assert(this._argv);
+
+    const config = this._baseCmd.config;
+    const jobQueue = this._baseCmd.jobQueue;
+    const database = this._baseCmd.database;
+    const indexer = this._baseCmd.indexer;
+    const eventWatcher = this._baseCmd.eventWatcher;
+
+    assert(config);
+    assert(jobQueue);
+    assert(database);
+    assert(indexer);
+    assert(eventWatcher);
+
+    if (this._argv.state) {
+      assert(config.server.enableState, 'State creation disabled');
+
+      const { startBlock, endBlock } = this._argv;
+
+      // NOTE: Assuming all blocks in the given range are in the pruned region
+      log(`Filling state for subgraph entities in range: [${startBlock}, ${endBlock}]`);
+      await fillState(indexer, contractEntitiesMap, this._argv);
+      log(`Filled state for subgraph entities in range: [${startBlock}, ${endBlock}]`);
+    } else {
+      await fillBlocks(jobQueue, indexer, eventWatcher, config.jobQueue.blockDelayInMilliSecs, this._argv);
+    }
+
+    await database.close();
+  }
+
+  _getArgv (): any {
+    return yargs(hideBin(process.argv)).parserConfiguration({
+      'parse-numbers': false
+    }).env(
+      'FILL'
+    ).options({
+      configFile: {
+        alias: 'f',
+        type: 'string',
+        require: true,
+        demandOption: true,
+        describe: 'Configuration file path (toml)',
+        default: DEFAULT_CONFIG_PATH
+      },
+      startBlock: {
+        type: 'number',
+        demandOption: true,
+        describe: 'Block number to start processing at'
+      },
+      endBlock: {
+        type: 'number',
+        demandOption: true,
+        describe: 'Block number to stop processing at'
+      },
+      prefetch: {
+        type: 'boolean',
+        default: false,
+        describe: 'Block and events prefetch mode'
+      },
+      batchBlocks: {
+        type: 'number',
+        default: 10,
+        describe: 'Number of blocks prefetched in batch'
+      },
+      state: {
+        type: 'boolean',
+        default: false,
+        describe: 'Fill state for subgraph entities'
+      }
+    }).argv;
+  }
+}

packages/cli/src/index.ts

@@ -13,3 +13,4 @@ export * from './export-state';
 export * from './server';
 export * from './job-runner';
 export * from './index-block';
+export * from './fill';

packages/cli/src/job-runner.ts

@@ -17,12 +17,7 @@ import {
   IndexerInterface,
   ServerConfig,
   Clients,
-  JobRunner as BaseJobRunner,
-  JobQueueConfig,
-  QUEUE_BLOCK_PROCESSING,
-  QUEUE_EVENT_PROCESSING,
-  QUEUE_BLOCK_CHECKPOINT,
-  QUEUE_HOOKS,
+  WatcherJobRunner as JobRunner,
   startMetricsServer
 } from '@cerc-io/util';
 
@@ -112,41 +107,3 @@ export class JobRunnerCmd {
       .argv;
   }
 }
-
-export class JobRunner {
-  jobQueue: JobQueue
-  baseJobRunner: BaseJobRunner
-  _indexer: IndexerInterface
-  _jobQueueConfig: JobQueueConfig
-
-  constructor (jobQueueConfig: JobQueueConfig, indexer: IndexerInterface, jobQueue: JobQueue) {
-    this._jobQueueConfig = jobQueueConfig;
-    this._indexer = indexer;
-    this.jobQueue = jobQueue;
-    this.baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this.jobQueue);
-  }
-
-  async subscribeBlockProcessingQueue (): Promise<void> {
-    await this.jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => {
-      await this.baseJobRunner.processBlock(job);
-    });
-  }
-
-  async subscribeEventProcessingQueue (): Promise<void> {
-    await this.jobQueue.subscribe(QUEUE_EVENT_PROCESSING, async (job) => {
-      await this.baseJobRunner.processEvent(job);
-    });
-  }
-
-  async subscribeHooksQueue (): Promise<void> {
-    await this.jobQueue.subscribe(QUEUE_HOOKS, async (job) => {
-      await this.baseJobRunner.processHooks(job);
-    });
-  }
-
-  async subscribeBlockCheckpointQueue (): Promise<void> {
-    await this.jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => {
-      await this.baseJobRunner.processCheckpoint(job);
-    });
-  }
-}