Refactor code to initialize clients (#290)

* Refactor CLI code

* Refactor code to initialize clients

* Don't return config from function to initialize clients

packages/codegen/src/templates/checkpoint-template.handlebars

@@ -2,15 +2,11 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import assert from 'assert';
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import { getDefaultProvider } from 'ethers';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig } from '@vulcanize/util';
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients } from '@vulcanize/util';
 
 import { Database } from '../database';
 import { Indexer } from '../indexer';
@@ -42,36 +38,12 @@ const main = async (): Promise<void> => {
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getDefaultProvider(rpcProviderEndpoint);
-
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
   const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash);
 
   log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`);

packages/codegen/src/templates/export-state-template.handlebars

@@ -6,13 +6,10 @@ import assert from 'assert';
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import { getDefaultProvider } from 'ethers';
 import fs from 'fs';
 import path from 'path';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig } from '@vulcanize/util';
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients } from '@vulcanize/util';
 import * as codec from '@ipld/dag-cbor';
 
 import { Database } from '../database';
@@ -40,36 +37,12 @@ const main = async (): Promise<void> => {
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getDefaultProvider(rpcProviderEndpoint);
-
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
 
   const exportData: any = {
     snapshotBlock: {},

packages/codegen/src/templates/fill-template.handlebars

@@ -9,9 +9,7 @@ import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
 import { PubSub } from 'apollo-server-express';
 
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
-import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, getCustomProvider } from '@vulcanize/util';
+import { Config, getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, initClients } from '@vulcanize/util';
 
 import { Database } from './database';
 import { Indexer } from './indexer';
@@ -42,40 +40,19 @@ export const main = async (): Promise<any> => {
     }
   }).argv;
 
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  assert(upstream, 'Missing upstream config');
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint, blockDelayInMilliSecs }, cache: cacheConfig } = upstream;
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getCustomProvider(rpcProviderEndpoint);
-
   // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
   // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
   const pubsub = new PubSub();
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
 
   const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
   assert(dbConnectionString, 'Missing job queue db connection string');
@@ -83,11 +60,9 @@ export const main = async (): Promise<any> => {
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
   await jobQueue.start();
 
-  const eventWatcher = new EventWatcher(upstream, ethClient, postgraphileClient, indexer, pubsub, jobQueue);
+  const eventWatcher = new EventWatcher(config.upstream, ethClient, postgraphileClient, indexer, pubsub, jobQueue);
 
-  assert(jobQueueConfig, 'Missing job queue config');
-
-  await fillBlocks(jobQueue, indexer, postgraphileClient, eventWatcher, blockDelayInMilliSecs, argv);
+  await fillBlocks(jobQueue, indexer, postgraphileClient, eventWatcher, argv);
 };
 
 main().catch(err => {

packages/codegen/src/templates/import-state-template.handlebars

@@ -8,13 +8,10 @@ import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
 import { PubSub } from 'apollo-server-express';
-import { getDefaultProvider } from 'ethers';
 import fs from 'fs';
 import path from 'path';
 
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
-import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH } from '@vulcanize/util';
+import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, Config, initClients } from '@vulcanize/util';
 import * as codec from '@ipld/dag-cbor';
 
 import { Database } from '../database';
@@ -43,40 +40,19 @@ export const main = async (): Promise<any> => {
     }
   }).argv;
 
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  assert(upstream, 'Missing upstream config');
-  const { ethServer: { gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getDefaultProvider(rpcProviderEndpoint);
-
   // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
   // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
   const pubsub = new PubSub();
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
+
+  const jobQueueConfig = config.jobQueue;
+  assert(jobQueueConfig, 'Missing job queue config');
 
   const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
   assert(dbConnectionString, 'Missing job queue db connection string');
@@ -86,8 +62,6 @@ export const main = async (): Promise<any> => {
 
   const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue);
 
-  assert(jobQueueConfig, 'Missing job queue config');
-
   // Import data.
   const importFilePath = path.resolve(argv.importFile);
   const encodedImportData = fs.readFileSync(importFilePath);

packages/codegen/src/templates/inspect-cid-template.handlebars

@@ -6,12 +6,9 @@ import assert from 'assert';
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import { getDefaultProvider } from 'ethers';
 import util from 'util';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig } from '@vulcanize/util';
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients } from '@vulcanize/util';
 
 import { Database } from '../database';
 import { Indexer } from '../indexer';
@@ -39,36 +36,12 @@ const main = async (): Promise<void> => {
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getDefaultProvider(rpcProviderEndpoint);
-
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
 
   const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid);
   assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.');

packages/codegen/src/templates/job-runner-template.handlebars

@@ -7,11 +7,9 @@ import 'reflect-metadata';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
-
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
 import {
   getConfig,
+  Config,
   JobQueue,
   JobRunner as BaseJobRunner,
   QUEUE_BLOCK_PROCESSING,
@@ -21,7 +19,7 @@ import {
   QUEUE_IPFS,
   JobQueueConfig,
   DEFAULT_CONFIG_PATH,
-  getCustomProvider
+  initClients
 } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
@@ -123,37 +121,15 @@ export const main = async (): Promise<any> => {
     })
     .argv;
 
-  const config = await getConfig(argv.f);
+  const config: Config = await getConfig(argv.f);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getCustomProvider(rpcProviderEndpoint);
-  const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
 
+  const jobQueueConfig = config.jobQueue;
   assert(jobQueueConfig, 'Missing job queue config');
 
   const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
@@ -162,7 +138,7 @@ export const main = async (): Promise<any> => {
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
   await jobQueue.start();
 
-  const jobRunner = new JobRunner(jobQueueConfig, serverConfig, indexer, jobQueue);
+  const jobRunner = new JobRunner(jobQueueConfig, config.server, indexer, jobQueue);
   await jobRunner.start();
 };
 

packages/codegen/src/templates/reset-state-template.handlebars

@@ -6,7 +6,7 @@ import debug from 'debug';
 import { MoreThan } from 'typeorm';
 import assert from 'assert';
 
-import { getConfig, getResetConfig, resetJobs } from '@vulcanize/util';
+import { getConfig, initClients, resetJobs } from '@vulcanize/util';
 
 import { Database } from '../../database';
 import { Indexer } from '../../indexer';
@@ -31,13 +31,13 @@ export const builder = {
 export const handler = async (argv: any): Promise<void> => {
   const config = await getConfig(argv.configFile);
   await resetJobs(config);
-  const { dbConfig, serverConfig, ethClient, postgraphileClient, ethProvider } = await getResetConfig(config);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
   // Initialize database.
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
 
   const syncStatus = await indexer.getSyncStatus();
   assert(syncStatus, 'Missing syncStatus');

packages/codegen/src/templates/server-template.handlebars

@@ -14,9 +14,7 @@ import debug from 'debug';
 import 'graphql-import-node';
 import { createServer } from 'http';
 
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
-import { DEFAULT_CONFIG_PATH, getConfig, JobQueue, KIND_ACTIVE, getCustomProvider } from '@vulcanize/util';
+import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
 
 import { createResolvers } from './resolvers';
 import { Indexer } from './indexer';
@@ -36,51 +34,28 @@ export const main = async (): Promise<any> => {
     })
     .argv;
 
-  const config = await getConfig(argv.f);
+  const config: Config = await getConfig(argv.f);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: serverConfig } = config;
+  const { host, port, kind: watcherKind } = config.server;
 
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const { host, port, kind: watcherKind } = serverConfig;
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getCustomProvider(rpcProviderEndpoint);
-
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
-
   // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
   // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
   const pubsub = new PubSub();
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider);
 
+  const jobQueueConfig = config.jobQueue;
   assert(jobQueueConfig, 'Missing job queue config');
+
   const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
   assert(dbConnectionString, 'Missing job queue db connection string');
 
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
 
-  const eventWatcher = new EventWatcher(upstream, ethClient, postgraphileClient, indexer, pubsub, jobQueue);
+  const eventWatcher = new EventWatcher(config.upstream, ethClient, postgraphileClient, indexer, pubsub, jobQueue);
 
   if (watcherKind === KIND_ACTIVE) {
     await jobQueue.start();

packages/codegen/src/templates/watch-contract-template.handlebars

@@ -2,15 +2,11 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import assert from 'assert';
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
-import { getDefaultProvider } from 'ethers';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig } from '@vulcanize/util';
-import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients } from '@vulcanize/util';
 
 import { Database } from '../database';
 import { Indexer } from '../indexer';
@@ -54,36 +50,12 @@ const main = async (): Promise<void> => {
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
+  const { ethClient, postgraphileClient, ethProvider } = await initClients(config);
 
-  const { upstream, database: dbConfig, server: serverConfig } = config;
-
-  assert(upstream, 'Missing upstream config');
-  assert(dbConfig, 'Missing database config');
-  assert(serverConfig, 'Missing server config');
-
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
-  const { ethServer: { gqlApiEndpoint, gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream;
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    gqlSubscriptionEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const postgraphileClient = new EthClient({
-    gqlEndpoint: gqlPostgraphileEndpoint,
-    cache
-  });
-
-  const ethProvider = getDefaultProvider(rpcProviderEndpoint);
-
-  const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
+  const indexer = new Indexer(config.server, db, ethClient, postgraphileClient, ethProvider);
   await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock);
 
   await db.close();