Refactor graph-node database and move to util (#259)

* Move graph-database from graph-node to util

* Refactor and remove graph-node dependency from cli package

* Modify dependencies using depcheck

* Implement CLI refactoring changes in other watchers

* Review changes to remove eden comment and fix local import in util

* Import GraphDatabase from util instead of graph-node

* Move graph-node non assemblyscript code to util package

* Implement CLI refactoring changes in codegen

* Fix graph-node tests after refactoring

* Move fromStateEntityValues to graph state utils

packages/codegen/src/templates/checkpoint-create-template.handlebars

@@ -3,8 +3,11 @@
 //
 
 import { CreateCheckpointCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../../database';
 import { Indexer } from '../../indexer';
 
 export const command = 'create';
@@ -26,7 +29,19 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const createCheckpointCmd = new CreateCheckpointCmd();
-  await createCheckpointCmd.init(argv, Database, Indexer);
+  await createCheckpointCmd.init(argv, Database);
 
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    createCheckpointCmd.config!.server,
+    createCheckpointCmd.clients!.ethClient,
+    createCheckpointCmd.ethProvider!,
+    createCheckpointCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await createCheckpointCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
   await createCheckpointCmd.exec();
 };

packages/codegen/src/templates/checkpoint-verify-template.handlebars

@@ -3,8 +3,9 @@
 //
 
 import { VerifyCheckpointCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../../database';
 import { Indexer } from '../../indexer';
 
 export const command = 'verify';
@@ -22,7 +23,17 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const verifyCheckpointCmd = new VerifyCheckpointCmd();
-  await verifyCheckpointCmd.init(argv, Database, Indexer);
+  await verifyCheckpointCmd.init(argv, Database);
 
-  await verifyCheckpointCmd.exec();
+  const { graphWatcher, graphDb } = await getGraphDbAndWatcher(
+    verifyCheckpointCmd.config!.server,
+    verifyCheckpointCmd.clients!.ethClient,
+    verifyCheckpointCmd.ethProvider!,
+    verifyCheckpointCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await verifyCheckpointCmd.initIndexer(Indexer, graphWatcher);
+  await verifyCheckpointCmd.exec(graphDb);
 };

packages/codegen/src/templates/database-template.handlebars

@@ -6,10 +6,16 @@ import assert from 'assert';
 import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, FindManyOptions, EntityTarget } from 'typeorm';
 import path from 'path';
 
-import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, Where } from '@cerc-io/util';
-{{#if (subgraphPath)}}
-import { ENTITY_QUERY_TYPE } from '@cerc-io/graph-node';
-{{/if}}
+import {
+  {{#if (subgraphPath)}}
+  ENTITY_QUERY_TYPE,
+  {{/if}}
+  Database as BaseDatabase,
+  DatabaseInterface,
+  QueryOptions,
+  StateKind,
+  Where
+} from '@cerc-io/util';
 
 import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';

packages/codegen/src/templates/export-state-template.handlebars

@@ -6,16 +6,31 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { ExportStateCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:export-state');
 
 const main = async (): Promise<void> => {
   const exportStateCmd = new ExportStateCmd();
-  await exportStateCmd.init(Database, Indexer);
+  await exportStateCmd.init(Database);
 
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    exportStateCmd.config!.server,
+    exportStateCmd.clients!.ethClient,
+    exportStateCmd.ethProvider!,
+    exportStateCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await exportStateCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
   await exportStateCmd.exec();
 };
 

packages/codegen/src/templates/fill-template.handlebars

@@ -2,15 +2,14 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-{{#if (subgraphPath)}}
 import assert from 'assert';
-{{/if}}
 import 'reflect-metadata';
 import debug from 'debug';
 
 import { FillCmd } from '@cerc-io/cli';
 {{#if (subgraphPath)}}
-import { getContractEntitiesMap } from '@cerc-io/graph-node';
+import { getContractEntitiesMap } from '@cerc-io/util';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 {{/if}}
 
 import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database';
@@ -21,15 +20,25 @@ const log = debug('vulcanize:fill');
 
 export const main = async (): Promise<any> => {
   const fillCmd = new FillCmd();
-  await fillCmd.init(Database, Indexer, EventWatcher{{#if (subgraphPath)}}, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}});
+  await fillCmd.init(Database);
 
   {{#if (subgraphPath)}}
-  const indexer = fillCmd.indexer as Indexer;
-  assert(indexer);
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    fillCmd.config!.server,
+    fillCmd.clients!.ethClient,
+    fillCmd.ethProvider!,
+    fillCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
 
+  {{/if}}
+  await fillCmd.initIndexer(Indexer, EventWatcher{{#if (subgraphPath)}}, graphWatcher{{/if}});
+
+  {{#if (subgraphPath)}}
   // Get contractEntitiesMap required for fill-state
   // NOTE: Assuming each entity type is only mapped to a single contract
-  const contractEntitiesMap = getContractEntitiesMap(indexer.graphWatcher.dataSources);
+  const contractEntitiesMap = getContractEntitiesMap(graphWatcher.dataSources);
 
   {{/if}}
   await fillCmd.exec({{#if (subgraphPath)}}contractEntitiesMap{{/if}});

packages/codegen/src/templates/import-state-template.handlebars

@@ -6,8 +6,11 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { ImportStateCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 import { EventWatcher } from '../events';
 import { State } from '../entity/State';
@@ -16,9 +19,21 @@ const log = debug('vulcanize:import-state');
 
 export const main = async (): Promise<any> => {
   const importStateCmd = new ImportStateCmd();
-  await importStateCmd.init(Database, Indexer, EventWatcher);
+  await importStateCmd.init(Database);
 
-  await importStateCmd.exec(State);
+  {{#if (subgraphPath)}}
+  const { graphWatcher, graphDb } = await getGraphDbAndWatcher(
+    importStateCmd.config!.server,
+    importStateCmd.clients!.ethClient,
+    importStateCmd.ethProvider!,
+    importStateCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await importStateCmd.initIndexer(Indexer, EventWatcher{{#if (subgraphPath)}}, graphWatcher{{/if}});
+  await importStateCmd.exec(State{{#if (subgraphPath)}}, graphDb{{/if}});
 };
 
 main().catch(err => {

packages/codegen/src/templates/index-block-template.handlebars

@@ -6,16 +6,31 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { IndexBlockCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:index-block');
 
 const main = async (): Promise<void> => {
   const indexBlockCmd = new IndexBlockCmd();
-  await indexBlockCmd.init(Database, Indexer);
+  await indexBlockCmd.init(Database);
 
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    indexBlockCmd.config!.server,
+    indexBlockCmd.clients!.ethClient,
+    indexBlockCmd.ethProvider!,
+    indexBlockCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await indexBlockCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
   await indexBlockCmd.exec();
 };
 

packages/codegen/src/templates/indexer-template.handlebars

@@ -27,6 +27,9 @@ import {
   updateStateForMappingType,
   {{#if (subgraphPath)}}
   BlockHeight,
+  updateSubgraphState,
+  dumpSubgraphState,
+  GraphWatcherInterface,
   {{/if}}
   StateKind,
   StateStatus,
@@ -36,7 +39,7 @@ import {
   Clients
 } from '@cerc-io/util';
 {{#if (subgraphPath)}}
-import { GraphWatcher, updateSubgraphState, dumpSubgraphState } from '@cerc-io/graph-node';
+import { GraphWatcher } from '@cerc-io/graph-node';
 {{/if}}
 
 {{#each contracts as | contract |}}
@@ -89,7 +92,7 @@ export class Indexer implements IndexerInterface {
   _subgraphStateMap: Map<string, any>
 
   {{/if}}
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue{{#if (subgraphPath)}}, graphWatcher?: GraphWatcher{{/if}}) {
+  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue{{#if (subgraphPath)}}, graphWatcher?: GraphWatcherInterface{{/if}}) {
     assert(db);
     assert(clients.ethClient);
 
@@ -100,7 +103,7 @@ export class Indexer implements IndexerInterface {
     this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
     {{#if (subgraphPath)}}
     assert(graphWatcher);
-    this._graphWatcher = graphWatcher;
+    this._graphWatcher = graphWatcher as GraphWatcher;
     {{/if}}
 
     this._abiMap = new Map();

packages/codegen/src/templates/inspect-cid-template.handlebars

@@ -6,16 +6,31 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { InspectCIDCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:inspect-cid');
 
 const main = async (): Promise<void> => {
   const inspectCIDCmd = new InspectCIDCmd();
-  await inspectCIDCmd.init(Database, Indexer);
+  await inspectCIDCmd.init(Database);
 
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    inspectCIDCmd.config!.server,
+    inspectCIDCmd.clients!.ethClient,
+    inspectCIDCmd.ethProvider!,
+    inspectCIDCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await inspectCIDCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
   await inspectCIDCmd.exec();
 };
 

packages/codegen/src/templates/job-runner-template.handlebars

@@ -6,6 +6,9 @@ import debug from 'debug';
 
 import { JobRunnerCmd } from '@cerc-io/cli';
 import { JobRunner } from '@cerc-io/util';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
 import { Indexer } from './indexer';
 import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database';
@@ -14,7 +17,20 @@ const log = debug('vulcanize:job-runner');
 
 export const main = async (): Promise<any> => {
   const jobRunnerCmd = new JobRunnerCmd();
-  await jobRunnerCmd.init(Database, Indexer{{#if (subgraphPath)}}, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}});
+  await jobRunnerCmd.init(Database);
+
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    jobRunnerCmd.config!.server,
+    jobRunnerCmd.clients!.ethClient,
+    jobRunnerCmd.ethProvider!,
+    jobRunnerCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await jobRunnerCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();

packages/codegen/src/templates/reset-watcher-template.handlebars

@@ -3,8 +3,11 @@
 //
 
 import { ResetWatcherCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../../database';
 import { Indexer } from '../../indexer';
 
 export const command = 'watcher';
@@ -19,7 +22,19 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const resetWatcherCmd = new ResetWatcherCmd();
-  await resetWatcherCmd.init(argv, Database, Indexer);
+  await resetWatcherCmd.init(argv, Database);
 
+  {{#if (subgraphPath)}}
+  const { graphWatcher, graphDb } = await getGraphDbAndWatcher(
+    resetWatcherCmd.config!.server,
+    resetWatcherCmd.clients!.ethClient,
+    resetWatcherCmd.ethProvider!,
+    resetWatcherCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await resetWatcherCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
   await resetWatcherCmd.exec();
 };

packages/codegen/src/templates/server-template.handlebars

@@ -8,6 +8,9 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { ServerCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
 import { createResolvers } from './resolvers';
 import { Indexer } from './indexer';
@@ -18,10 +21,22 @@ const log = debug('vulcanize:server');
 
 export const main = async (): Promise<any> => {
   const serverCmd = new ServerCmd();
-  await serverCmd.init(Database, Indexer, EventWatcher{{#if (subgraphPath)}}, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}});
+  await serverCmd.init(Database);
+
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    serverCmd.config!.server,
+    serverCmd.clients!.ethClient,
+    serverCmd.ethProvider!,
+    serverCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await serverCmd.initIndexer(Indexer, EventWatcher{{#if (subgraphPath)}}, graphWatcher{{/if}});
 
   const typeDefs = fs.readFileSync(path.join(__dirname, 'schema.gql')).toString();
-
   return serverCmd.exec(createResolvers, typeDefs);
 };
 

packages/codegen/src/templates/subscriber-template.handlebars

@@ -4,9 +4,10 @@
 
 import { EventSubscriber, EntitySubscriberInterface, InsertEvent, UpdateEvent } from 'typeorm';
 
+import { afterEntityInsertOrUpdate } from '@cerc-io/util';
+
 import { FrothyEntity } from './FrothyEntity';
 import { ENTITY_TO_LATEST_ENTITY_MAP, SUBGRAPH_ENTITIES } from '../database';
-import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node';
 
 @EventSubscriber()
 export class EntitySubscriber implements EntitySubscriberInterface {

packages/codegen/src/templates/watch-contract-template.handlebars

@@ -6,16 +6,31 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { WatchContractCmd } from '@cerc-io/cli';
+{{#if (subgraphPath)}}
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
+{{/if}}
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:watch-contract');
 
 const main = async (): Promise<void> => {
   const watchContractCmd = new WatchContractCmd();
-  await watchContractCmd.init(Database, Indexer);
+  await watchContractCmd.init(Database);
 
+  {{#if (subgraphPath)}}
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    watchContractCmd.config!.server,
+    watchContractCmd.clients!.ethClient,
+    watchContractCmd.ethProvider!,
+    watchContractCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  {{/if}}
+  await watchContractCmd.initIndexer(Indexer{{#if (subgraphPath)}}, graphWatcher{{/if}});
   await watchContractCmd.exec();
 };