Refactor graph-node database and move to util (#259)

* Move graph-database from graph-node to util

* Refactor and remove graph-node dependency from cli package

* Modify dependencies using depcheck

* Implement CLI refactoring changes in other watchers

* Review changes to remove eden comment and fix local import in util

* Import GraphDatabase from util instead of graph-node

* Move graph-node non assemblyscript code to util package

* Implement CLI refactoring changes in codegen

* Fix graph-node tests after refactoring

* Move fromStateEntityValues to graph state utils

packages/graph-test-watcher/src/cli/checkpoint-cmds/create.ts

@@ -3,8 +3,9 @@
 //
 
 import { CreateCheckpointCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../../database';
 import { Indexer } from '../../indexer';
 
 export const command = 'create';
@@ -26,7 +27,17 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const createCheckpointCmd = new CreateCheckpointCmd();
-  await createCheckpointCmd.init(argv, Database, Indexer);
+  await createCheckpointCmd.init(argv, Database);
 
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    createCheckpointCmd.config!.server,
+    createCheckpointCmd.clients!.ethClient,
+    createCheckpointCmd.ethProvider!,
+    createCheckpointCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await createCheckpointCmd.initIndexer(Indexer, graphWatcher);
   await createCheckpointCmd.exec();
 };

packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts

@@ -3,8 +3,9 @@
 //
 
 import { VerifyCheckpointCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../../database';
 import { Indexer } from '../../indexer';
 
 export const command = 'verify';
@@ -22,7 +23,17 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const verifyCheckpointCmd = new VerifyCheckpointCmd();
-  await verifyCheckpointCmd.init(argv, Database, Indexer);
+  await verifyCheckpointCmd.init(argv, Database);
 
-  await verifyCheckpointCmd.exec();
+  const { graphWatcher, graphDb } = await getGraphDbAndWatcher(
+    verifyCheckpointCmd.config!.server,
+    verifyCheckpointCmd.clients!.ethClient,
+    verifyCheckpointCmd.ethProvider!,
+    verifyCheckpointCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await verifyCheckpointCmd.initIndexer(Indexer, graphWatcher);
+  await verifyCheckpointCmd.exec(graphDb);
 };

packages/graph-test-watcher/src/cli/export-state.ts

@@ -6,16 +6,27 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { ExportStateCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:export-state');
 
 const main = async (): Promise<void> => {
   const exportStateCmd = new ExportStateCmd();
-  await exportStateCmd.init(Database, Indexer);
+  await exportStateCmd.init(Database);
 
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    exportStateCmd.config!.server,
+    exportStateCmd.clients!.ethClient,
+    exportStateCmd.ethProvider!,
+    exportStateCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await exportStateCmd.initIndexer(Indexer, graphWatcher);
   await exportStateCmd.exec();
 };
 

packages/graph-test-watcher/src/cli/import-state.ts

@@ -6,8 +6,9 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { ImportStateCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../database';
 import { Indexer } from '../indexer';
 import { EventWatcher } from '../events';
 import { State } from '../entity/State';
@@ -16,9 +17,19 @@ const log = debug('vulcanize:import-state');
 
 export const main = async (): Promise<any> => {
   const importStateCmd = new ImportStateCmd();
-  await importStateCmd.init(Database, Indexer, EventWatcher);
+  await importStateCmd.init(Database);
 
-  await importStateCmd.exec(State);
+  const { graphWatcher, graphDb } = await getGraphDbAndWatcher(
+    importStateCmd.config!.server,
+    importStateCmd.clients!.ethClient,
+    importStateCmd.ethProvider!,
+    importStateCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await importStateCmd.initIndexer(Indexer, EventWatcher, graphWatcher);
+  await importStateCmd.exec(State, graphDb);
 };
 
 main().catch(err => {

packages/graph-test-watcher/src/cli/index-block.ts

@@ -6,16 +6,27 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { IndexBlockCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:index-block');
 
 const main = async (): Promise<void> => {
   const indexBlockCmd = new IndexBlockCmd();
-  await indexBlockCmd.init(Database, Indexer);
+  await indexBlockCmd.init(Database);
 
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    indexBlockCmd.config!.server,
+    indexBlockCmd.clients!.ethClient,
+    indexBlockCmd.ethProvider!,
+    indexBlockCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await indexBlockCmd.initIndexer(Indexer, graphWatcher);
   await indexBlockCmd.exec();
 };
 

packages/graph-test-watcher/src/cli/inspect-cid.ts

@@ -6,16 +6,27 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { InspectCIDCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:inspect-cid');
 
 const main = async (): Promise<void> => {
   const inspectCIDCmd = new InspectCIDCmd();
-  await inspectCIDCmd.init(Database, Indexer);
+  await inspectCIDCmd.init(Database);
 
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    inspectCIDCmd.config!.server,
+    inspectCIDCmd.clients!.ethClient,
+    inspectCIDCmd.ethProvider!,
+    inspectCIDCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await inspectCIDCmd.initIndexer(Indexer, graphWatcher);
   await inspectCIDCmd.exec();
 };
 

packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts

@@ -3,8 +3,9 @@
 //
 
 import { ResetWatcherCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../../database';
 import { Indexer } from '../../indexer';
 
 export const command = 'watcher';
@@ -19,7 +20,17 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const resetWatcherCmd = new ResetWatcherCmd();
-  await resetWatcherCmd.init(argv, Database, Indexer);
+  await resetWatcherCmd.init(argv, Database);
 
+  const { graphWatcher, graphDb } = await getGraphDbAndWatcher(
+    resetWatcherCmd.config!.server,
+    resetWatcherCmd.clients!.ethClient,
+    resetWatcherCmd.ethProvider!,
+    resetWatcherCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await resetWatcherCmd.initIndexer(Indexer, graphWatcher);
   await resetWatcherCmd.exec();
 };

packages/graph-test-watcher/src/cli/watch-contract.ts

@@ -6,16 +6,27 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { WatchContractCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
-import { Database } from '../database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:watch-contract');
 
 const main = async (): Promise<void> => {
   const watchContractCmd = new WatchContractCmd();
-  await watchContractCmd.init(Database, Indexer);
+  await watchContractCmd.init(Database);
 
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    watchContractCmd.config!.server,
+    watchContractCmd.clients!.ethClient,
+    watchContractCmd.ethProvider!,
+    watchContractCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await watchContractCmd.initIndexer(Indexer, graphWatcher);
   await watchContractCmd.exec();
 };
 

packages/graph-test-watcher/src/entity/Subscriber.ts

@@ -4,7 +4,7 @@
 
 import { EventSubscriber, EntitySubscriberInterface, InsertEvent, UpdateEvent } from 'typeorm';
 
-import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node';
+import { afterEntityInsertOrUpdate } from '@cerc-io/util';
 
 import { FrothyEntity } from './FrothyEntity';
 import { ENTITY_TO_LATEST_ENTITY_MAP, SUBGRAPH_ENTITIES } from '../database';

packages/graph-test-watcher/src/fill.ts

@@ -7,7 +7,8 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { FillCmd } from '@cerc-io/cli';
-import { getContractEntitiesMap } from '@cerc-io/graph-node';
+import { getContractEntitiesMap } from '@cerc-io/util';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
 import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from './database';
 import { Indexer } from './indexer';
@@ -17,13 +18,22 @@ const log = debug('vulcanize:fill');
 
 export const main = async (): Promise<any> => {
   const fillCmd = new FillCmd();
-  await fillCmd.init(Database, Indexer, EventWatcher, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
+  await fillCmd.init(Database);
 
-  const indexer = fillCmd.indexer as Indexer;
-  assert(indexer);
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    fillCmd.config!.server,
+    fillCmd.clients!.ethClient,
+    fillCmd.ethProvider!,
+    fillCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await fillCmd.initIndexer(Indexer, EventWatcher, graphWatcher);
 
   // Get contractEntitiesMap required for fill-state
-  const contractEntitiesMap = getContractEntitiesMap(indexer.graphWatcher.dataSources);
+  // NOTE: Assuming each entity type is only mapped to a single contract
+  const contractEntitiesMap = getContractEntitiesMap(graphWatcher.dataSources);
 
   await fillCmd.exec(contractEntitiesMap);
 };

packages/graph-test-watcher/src/indexer.ts

@@ -28,9 +28,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  GraphWatcherInterface,
+  updateSubgraphState,
+  dumpSubgraphState
 } from '@cerc-io/util';
-import { GraphWatcher, updateSubgraphState, dumpSubgraphState } from '@cerc-io/graph-node';
+import { GraphWatcher } from '@cerc-io/graph-node';
 
 import { Database, ENTITIES, SUBGRAPH_ENTITIES } from './database';
 import { Contract } from './entity/Contract';
@@ -68,7 +71,7 @@ export class Indexer implements IndexerInterface {
 
   _subgraphStateMap: Map<string, any>
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher?: GraphWatcher) {
+  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher?: GraphWatcherInterface) {
     assert(db);
     assert(clients.ethClient);
 
@@ -79,7 +82,7 @@ export class Indexer implements IndexerInterface {
     this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     assert(graphWatcher);
-    this._graphWatcher = graphWatcher;
+    this._graphWatcher = graphWatcher as GraphWatcher;
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();

packages/graph-test-watcher/src/job-runner.ts

@@ -6,15 +6,27 @@ import debug from 'debug';
 
 import { JobRunnerCmd } from '@cerc-io/cli';
 import { JobRunner } from '@cerc-io/util';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
 import { Indexer } from './indexer';
-import { Database } from './database';
+import { Database, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP } from './database';
 
 const log = debug('vulcanize:job-runner');
 
 export const main = async (): Promise<any> => {
   const jobRunnerCmd = new JobRunnerCmd();
-  await jobRunnerCmd.init(Database, Indexer);
+  await jobRunnerCmd.init(Database);
+
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    jobRunnerCmd.config!.server,
+    jobRunnerCmd.clients!.ethClient,
+    jobRunnerCmd.ethProvider!,
+    jobRunnerCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await jobRunnerCmd.initIndexer(Indexer, graphWatcher);
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();

packages/graph-test-watcher/src/server.ts

@@ -8,6 +8,7 @@ import 'reflect-metadata';
 import debug from 'debug';
 
 import { ServerCmd } from '@cerc-io/cli';
+import { getGraphDbAndWatcher } from '@cerc-io/graph-node';
 
 import { createResolvers } from './resolvers';
 import { Indexer } from './indexer';
@@ -18,10 +19,20 @@ const log = debug('vulcanize:server');
 
 export const main = async (): Promise<any> => {
   const serverCmd = new ServerCmd();
-  await serverCmd.init(Database, Indexer, EventWatcher, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
+  await serverCmd.init(Database);
+
+  const { graphWatcher } = await getGraphDbAndWatcher(
+    serverCmd.config!.server,
+    serverCmd.clients!.ethClient,
+    serverCmd.ethProvider!,
+    serverCmd.database!.baseDatabase,
+    ENTITY_QUERY_TYPE_MAP,
+    ENTITY_TO_LATEST_ENTITY_MAP
+  );
+
+  await serverCmd.initIndexer(Indexer, EventWatcher, graphWatcher);
 
   const typeDefs = fs.readFileSync(path.join(__dirname, 'schema.gql')).toString();
-
   return serverCmd.exec(createResolvers, typeDefs);
 };