From 63a2c5804eed92bfda8d7f4c12cf48896e1049ad Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Fri, 25 Nov 2022 00:01:20 -0600 Subject: [PATCH] Accommodate CLI refactoring changes to codegen (#261) * Use watch-contract CLI from cli package in codegen * Use reset CLIs from cli package in codegen * Use create checkpoint CLI from cli package in codegen * Use inspect-cid CLI from cli package in codegen * Use import state CLI from cli package in codegen * Use export-state and verify-checkpoint CLI from cli package in codegen * Use server CLI from cli package in codegen * Use job-runner CLI from cli package in codegen * Use index-block CLI from cli package in codegen * Use fill CLI from cli package in codegen * Minor codegen fixes * Update watcher instructions to run job-runner before server --- packages/codegen/src/checkpoint.ts | 6 +- .../src/data/entities/StateSyncStatus.yaml | 3 + packages/codegen/src/export-state.ts | 4 +- packages/codegen/src/fill.ts | 11 +- packages/codegen/src/generate-code.ts | 35 ++-- packages/codegen/src/import-state.ts | 4 +- packages/codegen/src/inspect-cid.ts | 4 +- packages/codegen/src/reset.ts | 107 +++--------- .../checkpoint-create-template.handlebars | 47 +----- .../checkpoint-verify-template.handlebars | 50 +----- .../src/templates/events-template.handlebars | 7 +- .../export-state-template.handlebars | 152 +----------------- .../templates/fill-state-template.handlebars | 104 ------------ .../src/templates/fill-template.handlebars | 110 ++----------- .../src/templates/hooks-template.handlebars | 2 +- .../import-state-template.handlebars | 125 +------------- .../templates/index-block-template.handlebars | 62 +------ .../src/templates/indexer-template.handlebars | 20 ++- .../templates/inspect-cid-template.handlebars | 65 +------- .../templates/job-runner-template.handlebars | 124 ++------------ .../src/templates/package-template.handlebars | 10 +- .../src/templates/readme-template.handlebars | 20 ++- .../templates/reset-state-template.handlebars | 46 +----- .../reset-watcher-template.handlebars | 52 +----- .../templates/resolvers-template.handlebars | 17 +- .../src/templates/server-template.handlebars | 75 +-------- .../watch-contract-template.handlebars | 79 +-------- packages/codegen/src/visitor.ts | 17 -- packages/codegen/src/watch-contract.ts | 4 +- packages/eden-watcher/README.md | 20 ++- packages/eden-watcher/package.json | 1 - .../src/cli/checkpoint-cmds/verify.ts | 6 +- packages/eden-watcher/src/server.ts | 1 - packages/erc20-watcher/README.md | 22 +-- packages/erc20-watcher/package.json | 1 - packages/erc20-watcher/src/server.ts | 1 - packages/erc721-watcher/README.md | 20 ++- packages/erc721-watcher/package.json | 1 - packages/erc721-watcher/src/server.ts | 1 - packages/graph-node/src/database.ts | 12 -- packages/graph-node/src/watcher.ts | 9 +- packages/graph-test-watcher/README.md | 18 ++- packages/graph-test-watcher/package.json | 1 - .../src/cli/checkpoint-cmds/verify.ts | 6 +- packages/graph-test-watcher/src/server.ts | 1 - packages/mobymask-watcher/README.md | 18 ++- packages/mobymask-watcher/package.json | 1 - packages/mobymask-watcher/src/server.ts | 1 - yarn.lock | 5 - 49 files changed, 249 insertions(+), 1259 deletions(-) delete mode 100644 packages/codegen/src/templates/fill-state-template.handlebars diff --git a/packages/codegen/src/checkpoint.ts b/packages/codegen/src/checkpoint.ts index 41c21d1e..2fce4ef1 100644 --- a/packages/codegen/src/checkpoint.ts +++ b/packages/codegen/src/checkpoint.ts @@ -15,15 +15,15 @@ const VERIFY_TEMPLATE_FILE = './templates/checkpoint-verify-template.handlebars' * Writes the checkpoint file generated from a template to a stream. * @param outStream A writable output stream to write the checkpoint file to. */ -export function exportCheckpoint (checkpointOutStream: Writable, checkpointCreateOutStream: Writable, checkpointVerifyOutStream: Writable | undefined, subgraphPath: string): void { +export function exportCheckpoint (checkpointOutStream: Writable, checkpointCreateOutStream: Writable, checkpointVerifyOutStream: Writable | undefined): void { const checkpointTemplateString = fs.readFileSync(path.resolve(__dirname, CHECKPOINT_TEMPLATE_FILE)).toString(); const checkpointTemplate = Handlebars.compile(checkpointTemplateString); - const checkpoint = checkpointTemplate({ subgraphPath }); + const checkpoint = checkpointTemplate({}); checkpointOutStream.write(checkpoint); const createCheckpointTemplateString = fs.readFileSync(path.resolve(__dirname, CREATE_TEMPLATE_FILE)).toString(); const createCheckpointTemplate = Handlebars.compile(createCheckpointTemplateString); - const createCheckpoint = createCheckpointTemplate({ subgraphPath }); + const createCheckpoint = createCheckpointTemplate({}); checkpointCreateOutStream.write(createCheckpoint); if (checkpointVerifyOutStream) { diff --git a/packages/codegen/src/data/entities/StateSyncStatus.yaml b/packages/codegen/src/data/entities/StateSyncStatus.yaml index 0a62d5c1..f402e2ba 100644 --- a/packages/codegen/src/data/entities/StateSyncStatus.yaml +++ b/packages/codegen/src/data/entities/StateSyncStatus.yaml @@ -12,6 +12,9 @@ columns: pgType: integer tsType: number columnType: Column + columnOptions: + - option: nullable + value: true imports: - toImport: - Entity diff --git a/packages/codegen/src/export-state.ts b/packages/codegen/src/export-state.ts index 74f9f3b7..5d95f505 100644 --- a/packages/codegen/src/export-state.ts +++ b/packages/codegen/src/export-state.ts @@ -13,9 +13,9 @@ const TEMPLATE_FILE = './templates/export-state-template.handlebars'; * Writes the export-state file generated from a template to a stream. * @param outStream A writable output stream to write the export-state file to. */ -export function exportState (outStream: Writable, subgraphPath: string): void { +export function exportState (outStream: Writable): void { const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); const template = Handlebars.compile(templateString); - const exportState = template({ subgraphPath }); + const exportState = template({}); outStream.write(exportState); } diff --git a/packages/codegen/src/fill.ts b/packages/codegen/src/fill.ts index 2dc254e2..5741a175 100644 --- a/packages/codegen/src/fill.ts +++ b/packages/codegen/src/fill.ts @@ -8,23 +8,14 @@ import Handlebars from 'handlebars'; import { Writable } from 'stream'; const FILL_TEMPLATE_FILE = './templates/fill-template.handlebars'; -const FILL_STATE_TEMPLATE_FILE = './templates/fill-state-template.handlebars'; /** * Writes the fill file generated from a template to a stream. * @param fillOutStream A writable output stream to write the fill file to. - * @param fillStateOutStream A writable output stream to write the fill state file to. */ -export function exportFill (fillOutStream: Writable, fillStateOutStream: Writable | undefined, subgraphPath: string): void { +export function exportFill (fillOutStream: Writable, subgraphPath: string): void { const templateString = fs.readFileSync(path.resolve(__dirname, FILL_TEMPLATE_FILE)).toString(); const template = Handlebars.compile(templateString); const fill = template({ subgraphPath }); fillOutStream.write(fill); - - if (fillStateOutStream) { - const templateString = fs.readFileSync(path.resolve(__dirname, FILL_STATE_TEMPLATE_FILE)).toString(); - const template = Handlebars.compile(templateString); - const fillState = template({}); - fillStateOutStream.write(fillState); - } } diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 8d326316..1eddf688 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -38,6 +38,7 @@ import { exportInspectCID } from './inspect-cid'; import { getSubgraphConfig } from './utils/subgraph'; import { exportIndexBlock } from './index-block'; import { exportSubscriber } from './subscriber'; +import { exportReset } from './reset'; const main = async (): Promise => { const argv = await yargs(hideBin(process.argv)) @@ -238,7 +239,14 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) { outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/cli/watch-contract.ts')) : process.stdout; - exportWatchContract(outStream, config.subgraphPath); + exportWatchContract(outStream); + + const resetOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset.ts')); + const resetJQOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/job-queue.ts')); + const resetWatcherOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/watcher.ts')); + const resetStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/state.ts')); + + exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream); let checkpointOutStream, checkpointCreateOutStream, checkpointVerifyOutStream; @@ -256,7 +264,7 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) { } } - exportCheckpoint(checkpointOutStream, checkpointCreateOutStream, checkpointVerifyOutStream, config.subgraphPath); + exportCheckpoint(checkpointOutStream, checkpointCreateOutStream, checkpointVerifyOutStream); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/hooks.ts')) @@ -266,15 +274,7 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) { const fillOutStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/fill.ts')) : process.stdout; - - let fillStateOutStream; - if (config.subgraphPath) { - fillStateOutStream = outputDir - ? fs.createWriteStream(path.join(outputDir, 'src/fill-state.ts')) - : process.stdout; - } - - exportFill(fillOutStream, fillStateOutStream, config.subgraphPath); + exportFill(fillOutStream, config.subgraphPath); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/types.ts')) @@ -298,27 +298,20 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) { : process.stdout; visitor.exportClient(outStream, schemaContent, path.join(outputDir, 'src/gql')); - const resetOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset.ts')); - const resetJQOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/job-queue.ts')); - const resetWatcherOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/watcher.ts')); - const resetStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/state.ts')); - - visitor.exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream, config.subgraphPath); - outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/cli/export-state.ts')) : process.stdout; - exportState(outStream, config.subgraphPath); + exportState(outStream); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/cli/import-state.ts')) : process.stdout; - importState(outStream, config.subgraphPath); + importState(outStream); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/cli/inspect-cid.ts')) : process.stdout; - exportInspectCID(outStream, config.subgraphPath); + exportInspectCID(outStream); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/cli/index-block.ts')) diff --git a/packages/codegen/src/import-state.ts b/packages/codegen/src/import-state.ts index 8d6facbb..00019e29 100644 --- a/packages/codegen/src/import-state.ts +++ b/packages/codegen/src/import-state.ts @@ -13,9 +13,9 @@ const TEMPLATE_FILE = './templates/import-state-template.handlebars'; * Writes the import-state file generated from a template to a stream. * @param outStream A writable output stream to write the import-state file to. */ -export function importState (outStream: Writable, subgraphPath: string): void { +export function importState (outStream: Writable): void { const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); const template = Handlebars.compile(templateString); - const importState = template({ subgraphPath }); + const importState = template({}); outStream.write(importState); } diff --git a/packages/codegen/src/inspect-cid.ts b/packages/codegen/src/inspect-cid.ts index 82c5e3fb..63083cce 100644 --- a/packages/codegen/src/inspect-cid.ts +++ b/packages/codegen/src/inspect-cid.ts @@ -13,9 +13,9 @@ const TEMPLATE_FILE = './templates/inspect-cid-template.handlebars'; * Writes the inspect-cid file generated from a template to a stream. * @param outStream A writable output stream to write the inspect-cid file to. */ -export function exportInspectCID (outStream: Writable, subgraphPath: string): void { +export function exportInspectCID (outStream: Writable): void { const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); const template = Handlebars.compile(templateString); - const inspectCid = template({ subgraphPath }); + const inspectCid = template({}); outStream.write(inspectCid); } diff --git a/packages/codegen/src/reset.ts b/packages/codegen/src/reset.ts index 2ed65455..a70d6455 100644 --- a/packages/codegen/src/reset.ts +++ b/packages/codegen/src/reset.ts @@ -12,90 +12,31 @@ const RESET_JQ_TEMPLATE_FILE = './templates/reset-job-queue-template.handlebars' const RESET_WATCHER_TEMPLATE_FILE = './templates/reset-watcher-template.handlebars'; const RESET_STATE_TEMPLATE_FILE = './templates/reset-state-template.handlebars'; -export class Reset { - _queries: Array; - _resetTemplateString: string; - _resetJQTemplateString: string; - _resetWatcherTemplateString: string; - _resetStateTemplateString: string; +/** + * Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams. + * @param resetOutStream A writable output stream to write the reset file to. + * @param resetJQOutStream A writable output stream to write the reset job-queue file to. + * @param resetWatcherOutStream A writable output stream to write the reset watcher file to. + * @param resetStateOutStream A writable output stream to write the reset state file to. + */ +export function exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable): void { + const resetTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_TEMPLATE_FILE)).toString(); + const resetTemplate = Handlebars.compile(resetTemplateString); + const resetString = resetTemplate({}); + resetOutStream.write(resetString); - constructor () { - this._queries = []; - this._resetTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_TEMPLATE_FILE)).toString(); - this._resetJQTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_JQ_TEMPLATE_FILE)).toString(); - this._resetWatcherTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_WATCHER_TEMPLATE_FILE)).toString(); - this._resetStateTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_STATE_TEMPLATE_FILE)).toString(); - } + const resetJQTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_JQ_TEMPLATE_FILE)).toString(); + const resetJQTemplate = Handlebars.compile(resetJQTemplateString); + const resetJQString = resetJQTemplate({}); + resetJQOutStream.write(resetJQString); - /** - * Stores the query to be passed to the template. - * @param name Name of the query. - */ - addQuery (name: string): void { - // Check if the query is already added. - if (this._queries.some(query => query.name === name)) { - return; - } + const resetWatcherTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_WATCHER_TEMPLATE_FILE)).toString(); + const resetWatcherTemplate = Handlebars.compile(resetWatcherTemplateString); + const resetWatcher = resetWatcherTemplate({}); + resetWatcherOutStream.write(resetWatcher); - const queryObject = { - name, - entityName: '' - }; - - // eth_call mode: Capitalize first letter of entity name (balanceOf -> BalanceOf). - // storage mode: Capiltalize second letter of entity name (_balances -> _Balances). - queryObject.entityName = (name.charAt(0) === '_') - ? `_${name.charAt(1).toUpperCase()}${name.slice(2)}` - : `${name.charAt(0).toUpperCase()}${name.slice(1)}`; - - this._queries.push(queryObject); - } - - addSubgraphEntities (subgraphSchemaDocument: any): void { - const subgraphTypeDefs = subgraphSchemaDocument.definitions; - - subgraphTypeDefs.forEach((def: any) => { - if (def.kind !== 'ObjectTypeDefinition') { - return; - } - - this._queries.push({ - entityName: def.name.value - }); - }); - } - - /** - * Writes the reset.ts, job-queue.ts, state.ts files generated from templates to respective streams. - * @param outStream A writable output stream to write the database file to. - */ - - /** - * Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams. - * @param resetOutStream A writable output stream to write the reset file to. - * @param resetJQOutStream A writable output stream to write the reset job-queue file to. - * @param resetWatcherOutStream A writable output stream to write the reset watcher file to. - * @param resetStateOutStream A writable output stream to write the reset state file to. - */ - exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable, subgraphPath: string): void { - const resetTemplate = Handlebars.compile(this._resetTemplateString); - const resetString = resetTemplate({}); - resetOutStream.write(resetString); - - const resetJQTemplate = Handlebars.compile(this._resetJQTemplateString); - const resetJQString = resetJQTemplate({}); - resetJQOutStream.write(resetJQString); - - const resetWatcherTemplate = Handlebars.compile(this._resetWatcherTemplateString); - const obj = { - queries: this._queries, - subgraphPath - }; - const resetWatcher = resetWatcherTemplate(obj); - resetWatcherOutStream.write(resetWatcher); - - const resetStateTemplate = Handlebars.compile(this._resetStateTemplateString); - const resetState = resetStateTemplate({}); - resetStateOutStream.write(resetState); - } + const resetStateTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_STATE_TEMPLATE_FILE)).toString(); + const resetStateTemplate = Handlebars.compile(resetStateTemplateString); + const resetState = resetStateTemplate({}); + resetStateOutStream.write(resetState); } diff --git a/packages/codegen/src/templates/checkpoint-create-template.handlebars b/packages/codegen/src/templates/checkpoint-create-template.handlebars index 5f69004f..63f844ce 100644 --- a/packages/codegen/src/templates/checkpoint-create-template.handlebars +++ b/packages/codegen/src/templates/checkpoint-create-template.handlebars @@ -2,19 +2,11 @@ // Copyright 2022 Vulcanize, Inc. // -import debug from 'debug'; -import assert from 'assert'; - -import { getConfig, initClients, JobQueue, Config } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { CreateCheckpointCmd } from '@cerc-io/cli'; import { Database } from '../../database'; import { Indexer } from '../../indexer'; -const log = debug('vulcanize:checkpoint-create'); - export const command = 'create'; export const desc = 'Create checkpoint'; @@ -33,39 +25,8 @@ export const builder = { }; export const handler = async (argv: any): Promise => { - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); + const createCheckpointCmd = new CreateCheckpointCmd(); + await createCheckpointCmd.init(argv, Database, Indexer); - const db = new Database(config.database); - await db.init(); - - {{#if (subgraphPath)}} - const graphDb = new GraphDatabase(config.server, db.baseDatabase); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - {{/if}} - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - const blockHash = await indexer.processCLICheckpoint(argv.address, argv.blockHash); - - log(`Created a checkpoint for contract ${argv.address} at block-hash ${blockHash}`); - - await db.close(); + await createCheckpointCmd.exec(); }; diff --git a/packages/codegen/src/templates/checkpoint-verify-template.handlebars b/packages/codegen/src/templates/checkpoint-verify-template.handlebars index e2829dbf..9513d719 100644 --- a/packages/codegen/src/templates/checkpoint-verify-template.handlebars +++ b/packages/codegen/src/templates/checkpoint-verify-template.handlebars @@ -2,65 +2,27 @@ // Copyright 2022 Vulcanize, Inc. // -import path from 'path'; -import debug from 'debug'; -import assert from 'assert'; +import { VerifyCheckpointCmd } from '@cerc-io/cli'; -import { getConfig, initClients, JobQueue, Config, verifyCheckpointData } from '@cerc-io/util'; -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; - -import { Database, ENTITY_TO_LATEST_ENTITY_MAP, ENTITY_QUERY_TYPE_MAP } from '../../database'; +import { Database } from '../../database'; import { Indexer } from '../../indexer'; -const log = debug('vulcanize:checkpoint-verify'); - export const command = 'verify'; export const desc = 'Verify checkpoint'; export const builder = { cid: { - alias: 'c', type: 'string', + alias: 'c', demandOption: true, describe: 'Checkpoint CID to be verified' } }; export const handler = async (argv: any): Promise => { - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); + const verifyCheckpointCmd = new VerifyCheckpointCmd(); + await verifyCheckpointCmd.init(argv, Database, Indexer); - const db = new Database(config.database); - await db.init(); - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher); - await indexer.init(); - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - const state = await indexer.getStateByCID(argv.cid); - assert(state, 'State for the provided CID doesn\'t exist.'); - const data = indexer.getStateData(state); - - log(`Verifying checkpoint data for contract ${state.contractAddress}`); - await verifyCheckpointData(graphDb, state.block, data); - log('Checkpoint data verified'); - - await db.close(); + await verifyCheckpointCmd.exec(); }; diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index f7bc0374..4cb6e10d 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -11,7 +11,8 @@ import { EventWatcher as BaseEventWatcher, EventWatcherInterface, QUEUE_BLOCK_PROCESSING, - QUEUE_EVENT_PROCESSING + QUEUE_EVENT_PROCESSING, + IndexerInterface } from '@cerc-io/util'; import { Indexer } from './indexer'; @@ -24,12 +25,12 @@ export class EventWatcher implements EventWatcherInterface { _pubsub: PubSub _jobQueue: JobQueue - constructor (ethClient: EthClient, indexer: Indexer, pubsub: PubSub, jobQueue: JobQueue) { + constructor (ethClient: EthClient, indexer: IndexerInterface, pubsub: PubSub, jobQueue: JobQueue) { assert(ethClient); assert(indexer); this._ethClient = ethClient; - this._indexer = indexer; + this._indexer = indexer as Indexer; this._pubsub = pubsub; this._jobQueue = jobQueue; this._baseEventWatcher = new BaseEventWatcher(this._ethClient, this._indexer, this._pubsub, this._jobQueue); diff --git a/packages/codegen/src/templates/export-state-template.handlebars b/packages/codegen/src/templates/export-state-template.handlebars index edc8d3e5..2411b960 100644 --- a/packages/codegen/src/templates/export-state-template.handlebars +++ b/packages/codegen/src/templates/export-state-template.handlebars @@ -2,163 +2,21 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import fs from 'fs'; -import path from 'path'; -import { - Config, - DEFAULT_CONFIG_PATH, - getConfig, - initClients, - JobQueue, - {{#if (subgraphPath)}} - {{/if}} - StateKind -} from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} -import * as codec from '@ipld/dag-cbor'; +import { ExportStateCmd } from '@cerc-io/cli'; -import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database'; +import { Database } from '../database'; import { Indexer } from '../indexer'; const log = debug('vulcanize:export-state'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - exportFile: { - alias: 'o', - type: 'string', - describe: 'Export file path' - }, - blockNumber: { - type: 'number', - describe: 'Block number to create snapshot at' - } - }).argv; + const exportStateCmd = new ExportStateCmd(); + await exportStateCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - const exportData: any = { - snapshotBlock: {}, - contracts: [], - stateCheckpoints: [] - }; - - const contracts = await db.getContracts(); - - // Get latest block with hooks processed. - let block = await indexer.getLatestStateIndexedBlock(); - assert(block); - - if (argv.blockNumber) { - if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); - } - - const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); - - if (!blocksAtSnapshotHeight.length) { - throw new Error(`No blocks at snapshot height ${argv.blockNumber}`); - } - - block = blocksAtSnapshotHeight[0]; - } - - log(`Creating export snapshot at block height ${block.blockNumber}`); - - // Export snapshot block. - exportData.snapshotBlock = { - blockNumber: block.blockNumber, - blockHash: block.blockHash - }; - - // Export contracts and checkpoints. - for (const contract of contracts) { - if (contract.startingBlock > block.blockNumber) { - continue; - } - - exportData.contracts.push({ - address: contract.address, - kind: contract.kind, - checkpoint: contract.checkpoint, - startingBlock: block.blockNumber - }); - - // Create and export checkpoint if checkpointing is on for the contract. - if (contract.checkpoint) { - await indexer.createCheckpoint(contract.address, block.blockHash); - - const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(state); - - const data = indexer.getStateData(state); - - exportData.stateCheckpoints.push({ - contractAddress: state.contractAddress, - cid: state.cid, - kind: state.kind, - data - }); - } - } - - if (argv.exportFile) { - const encodedExportData = codec.encode(exportData); - - const filePath = path.resolve(argv.exportFile); - const fileDir = path.dirname(filePath); - - if (!fs.existsSync(fileDir)) fs.mkdirSync(fileDir, { recursive: true }); - - fs.writeFileSync(filePath, encodedExportData); - } else { - log(exportData); - } + await exportStateCmd.exec(); }; main().catch(err => { diff --git a/packages/codegen/src/templates/fill-state-template.handlebars b/packages/codegen/src/templates/fill-state-template.handlebars deleted file mode 100644 index f3dbf949..00000000 --- a/packages/codegen/src/templates/fill-state-template.handlebars +++ /dev/null @@ -1,104 +0,0 @@ -// -// Copyright 2022 Vulcanize, Inc. -// - -import 'reflect-metadata'; -import debug from 'debug'; -import { Between } from 'typeorm'; - -import { Database as GraphDatabase, prepareEntityState } from '@cerc-io/graph-node'; - -import { Indexer } from './indexer'; - -const log = debug('vulcanize:fill-state'); - -export const fillState = async ( - indexer: Indexer, - graphDb: GraphDatabase, - dataSources: any[], - argv: { - startBlock: number, - endBlock: number - } -): Promise => { - const { startBlock, endBlock } = argv; - if (startBlock > endBlock) { - log('endBlock should be greater than or equal to startBlock'); - process.exit(1); - } - - // NOTE: Assuming all blocks in the given range are in the pruned region - log(`Filling state for subgraph entities in range: [${startBlock}, ${endBlock}]`); - - // Check that there are no existing diffs in this range - const existingStates = await indexer.getStates({ block: { blockNumber: Between(startBlock, endBlock) } }); - if (existingStates.length > 0) { - log('found existing state(s) in the given range'); - process.exit(1); - } - - // Map: contractAddress -> entities updated - const contractEntitiesMap: Map = new Map(); - - // Populate contractEntitiesMap using data sources from subgraph - // NOTE: Assuming each entity type is only mapped to a single contract - // This is true for eden subgraph; may not be the case for other subgraphs - dataSources.forEach((dataSource: any) => { - const { source: { address: contractAddress }, mapping: { entities } } = dataSource; - contractEntitiesMap.set(contractAddress, entities as string[]); - }); - - // Fill state for blocks in the given range - for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) { - // Get the canonical block hash at current height - const blocks = await indexer.getBlocksAtHeight(blockNumber, false); - - if (blocks.length === 0) { - log(`block not found at height ${blockNumber}`); - process.exit(1); - } else if (blocks.length > 1) { - log(`found more than one non-pruned block at height ${blockNumber}`); - process.exit(1); - } - - const blockHash = blocks[0].blockHash; - - // Create initial state for contracts - await indexer.createInit(blockHash, blockNumber); - - // Fill state for each contract in contractEntitiesMap - const contractStatePromises = Array.from(contractEntitiesMap.entries()) - .map(async ([contractAddress, entities]): Promise => { - // Get all the updated entities at this block - const updatedEntitiesListPromises = entities.map(async (entity): Promise => { - return graphDb.getEntitiesForBlock(blockHash, entity); - }); - const updatedEntitiesList = await Promise.all(updatedEntitiesListPromises); - - // Populate state with all the updated entities of each entity type - updatedEntitiesList.forEach((updatedEntities, index) => { - const entityName = entities[index]; - - updatedEntities.forEach((updatedEntity) => { - // Prepare diff data for the entity update - const diffData = prepareEntityState(updatedEntity, entityName, indexer.getRelationsMap()); - - // Update the in-memory subgraph state - indexer.updateSubgraphState(contractAddress, diffData); - }); - }); - }); - - await Promise.all(contractStatePromises); - - // Persist subgraph state to the DB - await indexer.dumpSubgraphState(blockHash, true); - await indexer.updateStateSyncStatusIndexedBlock(blockNumber); - - // Create checkpoints - await indexer.processCheckpoint(blockHash); - await indexer.updateStateSyncStatusCheckpointBlock(blockNumber); - } - - log(`Filled state for subgraph entities in range: [${startBlock}, ${endBlock}]`); -}; diff --git a/packages/codegen/src/templates/fill-template.handlebars b/packages/codegen/src/templates/fill-template.handlebars index 9804a6da..1c7ffe0e 100644 --- a/packages/codegen/src/templates/fill-template.handlebars +++ b/packages/codegen/src/templates/fill-template.handlebars @@ -2,113 +2,37 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import 'reflect-metadata'; -import yargs from 'yargs'; -import { hideBin } from 'yargs/helpers'; -import debug from 'debug'; -import { PubSub } from 'graphql-subscriptions'; - -import { Config, getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, initClients } from '@cerc-io/util'; {{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; +import assert from 'assert'; +{{/if}} +import 'reflect-metadata'; +import debug from 'debug'; + +import { FillCmd } from '@cerc-io/cli'; +{{#if (subgraphPath)}} +import { getContractEntitiesMap } from '@cerc-io/graph-node'; {{/if}} import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; -{{#if (subgraphPath)}} -import { fillState } from './fill-state'; -{{/if}} -const log = debug('vulcanize:server'); +const log = debug('vulcanize:fill'); export const main = async (): Promise => { - const argv = await yargs(hideBin(process.argv)).parserConfiguration({ - 'parse-numbers': false - }).env( - 'FILL' - ).options({ - configFile: { - alias: 'f', - type: 'string', - demandOption: true, - describe: 'configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - startBlock: { - type: 'number', - demandOption: true, - describe: 'Block number to start processing at' - }, - {{#if (subgraphPath)}} - state: { - type: 'boolean', - default: false, - describe: 'Fill state for subgraph entities' - }, - {{/if}} - endBlock: { - type: 'number', - demandOption: true, - describe: 'Block number to stop processing at' - }, - prefetch: { - type: 'boolean', - default: false, - describe: 'Block and events prefetch mode' - }, - batchBlocks: { - type: 'number', - default: 10, - describe: 'Number of blocks prefetched in batch' - } - }).argv; + const fillCmd = new FillCmd(); + await fillCmd.init(Database, Indexer, EventWatcher{{#if (subgraphPath)}}, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}}); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); {{#if (subgraphPath)}} + const indexer = fillCmd.indexer as Indexer; + assert(indexer); - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); + // Get contractEntitiesMap required for fill-state + // NOTE: Assuming each entity type is only mapped to a single contract + const contractEntitiesMap = getContractEntitiesMap(indexer.graphWatcher.dataSources); - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - if (argv.state) { - assert(config.server.enableState, 'State creation disabled'); - await fillState(indexer, graphDb, graphWatcher.dataSources, argv); - - return; - } - {{/if}} - - // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. - // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries - const pubsub = new PubSub(); - - const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue); - - await fillBlocks(jobQueue, indexer, eventWatcher, jobQueueConfig.blockDelayInMilliSecs, argv); + await fillCmd.exec({{#if (subgraphPath)}}contractEntitiesMap{{/if}}); }; main().catch(err => { diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 7e43a634..77d99109 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -37,7 +37,7 @@ export async function createInitialState (indexer: Indexer, contractAddress: str /** * Hook function to create state diff. - * @param indexer Indexer instance that contains methods to fetch the contract varaiable values. + * @param indexer Indexer instance that contains methods to fetch the contract variable values. * @param blockHash Block hash of the concerned block. */ export async function createStateDiff (indexer: Indexer, blockHash: string): Promise { diff --git a/packages/codegen/src/templates/import-state-template.handlebars b/packages/codegen/src/templates/import-state-template.handlebars index 632bb34b..871f1b00 100644 --- a/packages/codegen/src/templates/import-state-template.handlebars +++ b/packages/codegen/src/templates/import-state-template.handlebars @@ -2,22 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; import 'reflect-metadata'; -import yargs from 'yargs'; -import { hideBin } from 'yargs/helpers'; import debug from 'debug'; -import { PubSub } from 'graphql-subscriptions'; -import fs from 'fs'; -import path from 'path'; -import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, Config, initClients, StateKind } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} -import * as codec from '@ipld/dag-cbor'; +import { ImportStateCmd } from '@cerc-io/cli'; -import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database'; +import { Database } from '../database'; import { Indexer } from '../indexer'; import { EventWatcher } from '../events'; import { State } from '../entity/State'; @@ -25,115 +15,10 @@ import { State } from '../entity/State'; const log = debug('vulcanize:import-state'); export const main = async (): Promise => { - const argv = await yargs(hideBin(process.argv)).parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - demandOption: true, - describe: 'configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - importFile: { - alias: 'i', - type: 'string', - demandOption: true, - describe: 'Import file path (JSON)' - } - }).argv; + const importStateCmd = new ImportStateCmd(); + await importStateCmd.init(Database, Indexer, EventWatcher); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. - // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries - const pubsub = new PubSub(); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue); - - // Import data. - const importFilePath = path.resolve(argv.importFile); - const encodedImportData = fs.readFileSync(importFilePath); - const importData = codec.decode(Buffer.from(encodedImportData)) as any; - - // Fill the snapshot block. - await fillBlocks( - jobQueue, - indexer, - eventWatcher, - jobQueueConfig.blockDelayInMilliSecs, - { - prefetch: true, - startBlock: importData.snapshotBlock.blockNumber, - endBlock: importData.snapshotBlock.blockNumber - } - ); - - // Fill the Contracts. - for (const contract of importData.contracts) { - await indexer.watchContract(contract.address, contract.kind, contract.checkpoint, contract.startingBlock); - } - - // Get the snapshot block. - const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash); - assert(block); - - // Fill the States. - for (const checkpoint of importData.stateCheckpoints) { - let state = new State(); - - state = Object.assign(state, checkpoint); - state.block = block; - - state.data = Buffer.from(codec.encode(state.data)); - - state = await indexer.saveOrUpdateState(state); - {{#if (subgraphPath)}} - await graphWatcher.updateEntitiesFromState(state); - {{/if}} - } - - // Mark snapshot block as completely processed. - block.isComplete = true; - await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); - await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber); - await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber); - await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber); - await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber); - - // The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block. - await indexer.removeStates(block.blockNumber, StateKind.Init); - await indexer.removeStates(block.blockNumber, StateKind.DiffStaged); - - log(`Import completed for snapshot block at height ${block.blockNumber}`); + await importStateCmd.exec(State); }; main().catch(err => { diff --git a/packages/codegen/src/templates/index-block-template.handlebars b/packages/codegen/src/templates/index-block-template.handlebars index 31aa6e1a..6b59df5e 100644 --- a/packages/codegen/src/templates/index-block-template.handlebars +++ b/packages/codegen/src/templates/index-block-template.handlebars @@ -2,73 +2,21 @@ // Copyright 2022 Vulcanize, Inc. // -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import assert from 'assert'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, indexBlock } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { IndexBlockCmd } from '@cerc-io/cli'; -import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database'; +import { Database } from '../database'; import { Indexer } from '../indexer'; const log = debug('vulcanize:index-block'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - block: { - type: 'number', - require: true, - demandOption: true, - describe: 'Block number to index' - } - }).argv; + const indexBlockCmd = new IndexBlockCmd(); + await indexBlockCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - await indexBlock(indexer, jobQueueConfig.eventsInBatch, argv); - - await db.close(); + await indexBlockCmd.exec(); }; main().catch(err => { diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 13981992..ad25a194 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -31,7 +31,9 @@ import { StateKind, StateStatus, ResultEvent, - getResultEvent + getResultEvent, + DatabaseInterface, + Clients } from '@cerc-io/util'; {{#if (subgraphPath)}} import { GraphWatcher, updateSubgraphState, dumpSubgraphState } from '@cerc-io/graph-node'; @@ -87,16 +89,17 @@ export class Indexer implements IndexerInterface { _subgraphStateMap: Map {{/if}} - constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue{{#if (subgraphPath)}}, graphWatcher: GraphWatcher{{/if}}) { + constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue{{#if (subgraphPath)}}, graphWatcher?: GraphWatcher{{/if}}) { assert(db); - assert(ethClient); + assert(clients.ethClient); - this._db = db; - this._ethClient = ethClient; + this._db = db as Database; + this._ethClient = clients.ethClient; this._ethProvider = ethProvider; this._serverConfig = serverConfig; this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); {{#if (subgraphPath)}} + assert(graphWatcher); this._graphWatcher = graphWatcher; {{/if}} @@ -105,12 +108,7 @@ export class Indexer implements IndexerInterface { this._contractMap = new Map(); {{#each contracts as | contract |}} - const { - abi: {{contract.contractName}}ABI, - {{#if contract.contractStorageLayout}} - storageLayout: {{contract.contractName}}StorageLayout - {{/if}} - } = {{contract.contractName}}Artifacts; + const { abi: {{contract.contractName}}ABI{{#if contract.contractStorageLayout}}, storageLayout: {{contract.contractName}}StorageLayout{{/if}} } = {{contract.contractName}}Artifacts; {{/each}} {{#each contracts as | contract |}} diff --git a/packages/codegen/src/templates/inspect-cid-template.handlebars b/packages/codegen/src/templates/inspect-cid-template.handlebars index abe3c7f2..3b1f44d7 100644 --- a/packages/codegen/src/templates/inspect-cid-template.handlebars +++ b/packages/codegen/src/templates/inspect-cid-template.handlebars @@ -2,16 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import util from 'util'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { InspectCIDCmd } from '@cerc-io/cli'; import { Database } from '../database'; import { Indexer } from '../indexer'; @@ -19,61 +13,10 @@ import { Indexer } from '../indexer'; const log = debug('vulcanize:inspect-cid'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - cid: { - alias: 'c', - type: 'string', - demandOption: true, - describe: 'CID to be inspected' - } - }).argv; + const inspectCIDCmd = new InspectCIDCmd(); + await inspectCIDCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - const state = await indexer.getStateByCID(argv.cid); - assert(state, 'State for the provided CID doesn\'t exist.'); - - const stateData = await indexer.getStateData(state); - - log(util.inspect(stateData, false, null)); + await inspectCIDCmd.exec(); }; main().catch(err => { diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index 4c82dfc0..fa6685f1 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -2,130 +2,26 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import 'reflect-metadata'; -import yargs from 'yargs'; -import { hideBin } from 'yargs/helpers'; import debug from 'debug'; -import { - getConfig, - Config, - JobQueue, - JobRunner as BaseJobRunner, - QUEUE_BLOCK_PROCESSING, - QUEUE_EVENT_PROCESSING, - QUEUE_BLOCK_CHECKPOINT, - QUEUE_HOOKS, - JOB_KIND_PRUNE, - JobQueueConfig, - DEFAULT_CONFIG_PATH, - initClients, - startMetricsServer -} from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { JobRunnerCmd } from '@cerc-io/cli'; +import { JobRunner } from '@cerc-io/util'; import { Indexer } from './indexer'; import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database'; const log = debug('vulcanize:job-runner'); -export class JobRunner { - _indexer: Indexer - _jobQueue: JobQueue - _baseJobRunner: BaseJobRunner - _jobQueueConfig: JobQueueConfig - - constructor (jobQueueConfig: JobQueueConfig, indexer: Indexer, jobQueue: JobQueue) { - this._jobQueueConfig = jobQueueConfig; - this._indexer = indexer; - this._jobQueue = jobQueue; - this._baseJobRunner = new BaseJobRunner(this._jobQueueConfig, this._indexer, this._jobQueue); - } - - async start (): Promise { - await this._jobQueue.deleteAllJobs(); - await this._baseJobRunner.resetToPrevIndexedBlock(); - await this.subscribeBlockProcessingQueue(); - await this.subscribeEventProcessingQueue(); - await this.subscribeBlockCheckpointQueue(); - await this.subscribeHooksQueue(); - } - - async subscribeBlockProcessingQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { - await this._baseJobRunner.processBlock(job); - }); - } - - async subscribeEventProcessingQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_EVENT_PROCESSING, async (job) => { - await this._baseJobRunner.processEvent(job); - }); - } - - async subscribeHooksQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { - await this._baseJobRunner.processHooks(job); - }); - } - - async subscribeBlockCheckpointQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { - await this._baseJobRunner.processCheckpoint(job); - }); - } -} - export const main = async (): Promise => { - const argv = await yargs(hideBin(process.argv)) - .option('f', { - alias: 'config-file', - demandOption: true, - describe: 'configuration file path (toml)', - type: 'string', - default: DEFAULT_CONFIG_PATH - }) - .argv; + const jobRunnerCmd = new JobRunnerCmd(); + await jobRunnerCmd.init(Database, Indexer{{#if (subgraphPath)}}, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}}); - const config: Config = await getConfig(argv.f); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - await indexer.addContracts(); - {{/if}} - - const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue); - await jobRunner.start(); - - startMetricsServer(config, indexer); + await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise => { + await jobRunner.subscribeBlockProcessingQueue(); + await jobRunner.subscribeEventProcessingQueue(); + await jobRunner.subscribeBlockCheckpointQueue(); + await jobRunner.subscribeHooksQueue(); + }); }; main().then(() => { diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index 9a3a29be..aeb9b201 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -42,18 +42,18 @@ "@apollo/client": "^3.3.19", "@ethersproject/providers": "^5.4.4", "@ipld/dag-cbor": "^6.0.12", - "@cerc-io/ipld-eth-client": "^0.2.13", - "@cerc-io/solidity-mapper": "^0.2.13", - "@cerc-io/util": "^0.2.13", + "@cerc-io/cli": "^0.2.15", + "@cerc-io/ipld-eth-client": "^0.2.15", + "@cerc-io/solidity-mapper": "^0.2.15", + "@cerc-io/util": "^0.2.15", {{#if (subgraphPath)}} - "@cerc-io/graph-node": "^0.2.13", + "@cerc-io/graph-node": "^0.2.15", {{/if}} "apollo-type-bigint": "^0.1.3", "debug": "^4.3.1", "ethers": "^5.4.4", "express": "^4.18.2", "graphql": "^15.5.0", - "graphql-import-node": "^0.0.4", "graphql-subscriptions": "^2.0.0", "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", diff --git a/packages/codegen/src/templates/readme-template.handlebars b/packages/codegen/src/templates/readme-template.handlebars index 344c8d6e..6dea60ab 100644 --- a/packages/codegen/src/templates/readme-template.handlebars +++ b/packages/codegen/src/templates/readme-template.handlebars @@ -73,13 +73,15 @@ To enable GQL requests caching: ## Run -* Run the watcher: +* If the watcher is a `lazy` watcher: - ```bash - yarn server - ``` + * Run the server: -GQL console: http://localhost:{{port}}/graphql + ```bash + yarn server + ``` + + GQL console: http://localhost:{{port}}/graphql * If the watcher is an `active` watcher: @@ -89,6 +91,14 @@ GQL console: http://localhost:{{port}}/graphql yarn job-runner ``` + * Run the server: + + ```bash + yarn server + ``` + + GQL console: http://localhost:{{port}}/graphql + * To watch a contract: ```bash diff --git a/packages/codegen/src/templates/reset-state-template.handlebars b/packages/codegen/src/templates/reset-state-template.handlebars index 500c62a8..33211d6e 100644 --- a/packages/codegen/src/templates/reset-state-template.handlebars +++ b/packages/codegen/src/templates/reset-state-template.handlebars @@ -2,14 +2,10 @@ // Copyright 2022 Vulcanize, Inc. // -import debug from 'debug'; - -import { getConfig, Config } from '@cerc-io/util'; +import { ResetStateCmd } from '@cerc-io/cli'; import { Database } from '../../database'; -const log = debug('vulcanize:reset-state'); - export const command = 'state'; export const desc = 'Reset State to a given block number'; @@ -21,42 +17,8 @@ export const builder = { }; export const handler = async (argv: any): Promise => { - const { blockNumber } = argv; - const config: Config = await getConfig(argv.configFile); + const resetStateCmd = new ResetStateCmd(); + await resetStateCmd.init(argv, Database); - // Initialize database - const db = new Database(config.database); - await db.init(); - - // Create a DB transaction - const dbTx = await db.createTransactionRunner(); - - console.time('time:reset-state'); - try { - // Delete all State entries in the given range - await db.removeStatesAfterBlock(dbTx, blockNumber); - - // Reset the stateSyncStatus. - const stateSyncStatus = await db.getStateSyncStatus(); - - if (stateSyncStatus) { - if (stateSyncStatus.latestIndexedBlockNumber > blockNumber) { - await db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, true); - } - - if (stateSyncStatus.latestCheckpointBlockNumber > blockNumber) { - await db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, true); - } - } - - dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - console.timeEnd('time:reset-state'); - - log(`Reset State successfully to block ${blockNumber}`); + await resetStateCmd.exec(); }; diff --git a/packages/codegen/src/templates/reset-watcher-template.handlebars b/packages/codegen/src/templates/reset-watcher-template.handlebars index d65b3e4c..295adc45 100644 --- a/packages/codegen/src/templates/reset-watcher-template.handlebars +++ b/packages/codegen/src/templates/reset-watcher-template.handlebars @@ -2,24 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import debug from 'debug'; -import { MoreThan } from 'typeorm'; -import assert from 'assert'; - -import { getConfig, initClients, resetJobs, JobQueue, Config } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { ResetWatcherCmd } from '@cerc-io/cli'; import { Database } from '../../database'; import { Indexer } from '../../indexer'; -import { BlockProgress } from '../../entity/BlockProgress'; - -{{#each queries as | query |}} -import { {{query.entityName}} } from '../../entity/{{query.entityName}}'; -{{/each}} - -const log = debug('vulcanize:reset-watcher'); export const command = 'watcher'; @@ -32,38 +18,8 @@ export const builder = { }; export const handler = async (argv: any): Promise => { - const config: Config = await getConfig(argv.configFile); - await resetJobs(config); - const { ethClient, ethProvider } = await initClients(config); + const resetWatcherCmd = new ResetWatcherCmd(); + await resetWatcherCmd.init(argv, Database, Indexer); - // Initialize database. - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - await indexer.resetWatcherToBlock(argv.blockNumber); - log('Reset watcher successfully'); + await resetWatcherCmd.exec(); }; diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index ef097dbc..9e3881fe 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -8,7 +8,17 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState, setGQLCacheHints } from '@cerc-io/util'; +import { + ValueResult, + BlockHeight, + gqlTotalQueryCount, + gqlQueryCount, + jsonBigIntStringReplacer, + getResultState, + setGQLCacheHints, + IndexerInterface, + EventWatcherInterface +} from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -21,8 +31,9 @@ import { {{query.entityName}} } from './entity/{{query.entityName}}'; const log = debug('vulcanize:resolver'); -export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatcher): Promise => { - assert(indexer); +export const createResolvers = async (indexerArg: IndexerInterface, eventWatcherArg: EventWatcherInterface): Promise => { + const indexer = indexerArg as Indexer; + const eventWatcher = eventWatcherArg as EventWatcher; const gqlCacheConfig = indexer.serverConfig.gqlCache; diff --git a/packages/codegen/src/templates/server-template.handlebars b/packages/codegen/src/templates/server-template.handlebars index 807b4df6..07f010b3 100644 --- a/packages/codegen/src/templates/server-template.handlebars +++ b/packages/codegen/src/templates/server-template.handlebars @@ -4,19 +4,10 @@ import fs from 'fs'; import path from 'path'; -import assert from 'assert'; import 'reflect-metadata'; -import express, { Application } from 'express'; -import { PubSub } from 'graphql-subscriptions'; -import yargs from 'yargs'; -import { hideBin } from 'yargs/helpers'; import debug from 'debug'; -import 'graphql-import-node'; -import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer, createAndStartServer } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { ServerCmd } from '@cerc-io/cli'; import { createResolvers } from './resolvers'; import { Indexer } from './indexer'; @@ -26,70 +17,12 @@ import { EventWatcher } from './events'; const log = debug('vulcanize:server'); export const main = async (): Promise => { - const argv = await yargs(hideBin(process.argv)) - .option('f', { - alias: 'config-file', - demandOption: true, - describe: 'configuration file path (toml)', - type: 'string', - default: DEFAULT_CONFIG_PATH - }) - .argv; + const serverCmd = new ServerCmd(); + await serverCmd.init(Database, Indexer, EventWatcher{{#if (subgraphPath)}}, {}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}}); - const config: Config = await getConfig(argv.f); - const { ethClient, ethProvider } = await initClients(config); - - const { kind: watcherKind } = config.server; - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. - // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries - const pubsub = new PubSub(); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue); - - if (watcherKind === KIND_ACTIVE) { - await jobQueue.start(); - // Delete jobs to prevent creating jobs after completion of processing previous block. - await jobQueue.deleteAllJobs(); - await eventWatcher.start(); - } - - const resolvers = await createResolvers(indexer, eventWatcher); const typeDefs = fs.readFileSync(path.join(__dirname, 'schema.gql')).toString(); - // Create an Express app - const app: Application = express(); - const server = createAndStartServer(app, typeDefs, resolvers, config.server); - - startGQLMetricsServer(config); - - return { app, server }; + return serverCmd.exec(createResolvers, typeDefs); }; main().then(() => { diff --git a/packages/codegen/src/templates/watch-contract-template.handlebars b/packages/codegen/src/templates/watch-contract-template.handlebars index 41e25940..bf1e7834 100644 --- a/packages/codegen/src/templates/watch-contract-template.handlebars +++ b/packages/codegen/src/templates/watch-contract-template.handlebars @@ -2,15 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import assert from 'assert'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { WatchContractCmd } from '@cerc-io/cli'; import { Database } from '../database'; import { Indexer } from '../indexer'; @@ -18,76 +13,10 @@ import { Indexer } from '../indexer'; const log = debug('vulcanize:watch-contract'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - address: { - type: 'string', - require: true, - demandOption: true, - describe: 'Address of the deployed contract' - }, - kind: { - type: 'string', - require: true, - demandOption: true, - describe: 'Kind of contract' - }, - checkpoint: { - type: 'boolean', - require: true, - demandOption: true, - describe: 'Turn checkpointing on' - }, - startingBlock: { - type: 'number', - default: 1, - describe: 'Starting block' - } - }).argv; + const watchContractCmd = new WatchContractCmd(); + await watchContractCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.server, db.baseDatabase); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock); - - await db.close(); - await jobQueue.stop(); + await watchContractCmd.exec(); }; main().catch(err => { diff --git a/packages/codegen/src/visitor.ts b/packages/codegen/src/visitor.ts index 6049dd4f..07d2a783 100644 --- a/packages/codegen/src/visitor.ts +++ b/packages/codegen/src/visitor.ts @@ -12,7 +12,6 @@ import { Indexer } from './indexer'; import { Resolvers } from './resolvers'; import { Schema } from './schema'; import { Client } from './client'; -import { Reset } from './reset'; import { Param } from './utils/types'; import { MODE_ETH_CALL, MODE_STORAGE } from './utils/constants'; import { parseSubgraphSchema } from './utils/subgraph'; @@ -25,7 +24,6 @@ export class Visitor { _entity: Entity; _database: Database; _client: Client; - _reset: Reset; _types: Types; _contract?: { name: string, kind: string }; @@ -37,7 +35,6 @@ export class Visitor { this._entity = new Entity(); this._database = new Database(); this._client = new Client(); - this._reset = new Reset(); this._types = new Types(); } @@ -75,7 +72,6 @@ export class Visitor { this._entity.addQuery(name, params, returnType); this._database.addQuery(name, params, returnType); this._client.addQuery(name, params, returnType); - this._reset.addQuery(name); assert(this._contract); this._indexer.addQuery(this._contract.name, MODE_ETH_CALL, name, params, returnType); @@ -123,7 +119,6 @@ export class Visitor { this._entity.addQuery(name, params, returnType); this._database.addQuery(name, params, returnType); this._client.addQuery(name, params, returnType); - this._reset.addQuery(name); assert(this._contract); this._indexer.addQuery(this._contract.name, MODE_STORAGE, name, params, returnType, stateVariableType); @@ -153,7 +148,6 @@ export class Visitor { this._types.addSubgraphTypes(subgraphSchemaDocument); this._entity.addSubgraphEntities(subgraphSchemaDocument); this._resolvers.addSubgraphResolvers(subgraphSchemaDocument); - this._reset.addSubgraphEntities(subgraphSchemaDocument); this._indexer.addSubgraphEntities(subgraphSchemaDocument); this._database.addSubgraphEntities(subgraphSchemaDocument); } @@ -210,17 +204,6 @@ export class Visitor { this._client.exportClient(outStream, schemaContent, gqlDir); } - /** - * Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams. - * @param resetOutStream A writable output stream to write the reset file to. - * @param resetJQOutStream A writable output stream to write the reset job-queue file to. - * @param resetWatcherOutStream A writable output stream to write the reset watcher file to. - * @param resetStateOutStream A writable output stream to write the reset state file to. - */ - exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable, subgraphPath: string): void { - this._reset.exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream, subgraphPath); - } - /** * Writes the types file generated from a template to a stream. * @param outStream A writable output stream to write the database file to. diff --git a/packages/codegen/src/watch-contract.ts b/packages/codegen/src/watch-contract.ts index 207b748c..b2ff6248 100644 --- a/packages/codegen/src/watch-contract.ts +++ b/packages/codegen/src/watch-contract.ts @@ -13,9 +13,9 @@ const TEMPLATE_FILE = './templates/watch-contract-template.handlebars'; * Writes the watch-contract file generated from a template to a stream. * @param outStream A writable output stream to write the watch-contract file to. */ -export function exportWatchContract (outStream: Writable, subgraphPath: string): void { +export function exportWatchContract (outStream: Writable): void { const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); const template = Handlebars.compile(templateString); - const events = template({ subgraphPath }); + const events = template({}); outStream.write(events); } diff --git a/packages/eden-watcher/README.md b/packages/eden-watcher/README.md index bf674c7e..0800c0cf 100644 --- a/packages/eden-watcher/README.md +++ b/packages/eden-watcher/README.md @@ -61,13 +61,15 @@ ## Run -* Run the watcher: +* If the watcher is a `lazy` watcher: - ```bash - yarn server - ``` + * Run the server: -GQL console: http://localhost:3012/graphql + ```bash + yarn server + ``` + + GQL console: http://localhost:3012/graphql * If the watcher is an `active` watcher: @@ -77,6 +79,14 @@ GQL console: http://localhost:3012/graphql yarn job-runner ``` + * Run the server: + + ```bash + yarn server + ``` + + GQL console: http://localhost:3012/graphql + * To watch a contract: ```bash diff --git a/packages/eden-watcher/package.json b/packages/eden-watcher/package.json index c2afd190..611f2744 100644 --- a/packages/eden-watcher/package.json +++ b/packages/eden-watcher/package.json @@ -51,7 +51,6 @@ "ethers": "^5.4.4", "express": "^4.18.2", "graphql": "^15.5.0", - "graphql-import-node": "^0.0.4", "graphql-subscriptions": "^2.0.0", "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", diff --git a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts index 51b127ca..9513d719 100644 --- a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts +++ b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts @@ -21,8 +21,8 @@ export const builder = { }; export const handler = async (argv: any): Promise => { - const createCheckpointCmd = new VerifyCheckpointCmd(); - await createCheckpointCmd.init(argv, Database, Indexer); + const verifyCheckpointCmd = new VerifyCheckpointCmd(); + await verifyCheckpointCmd.init(argv, Database, Indexer); - await createCheckpointCmd.exec(); + await verifyCheckpointCmd.exec(); }; diff --git a/packages/eden-watcher/src/server.ts b/packages/eden-watcher/src/server.ts index 45810884..3d432ae3 100644 --- a/packages/eden-watcher/src/server.ts +++ b/packages/eden-watcher/src/server.ts @@ -6,7 +6,6 @@ import fs from 'fs'; import path from 'path'; import 'reflect-metadata'; import debug from 'debug'; -import 'graphql-import-node'; import { ServerCmd } from '@cerc-io/cli'; diff --git a/packages/erc20-watcher/README.md b/packages/erc20-watcher/README.md index a78f6b66..81815056 100644 --- a/packages/erc20-watcher/README.md +++ b/packages/erc20-watcher/README.md @@ -70,16 +70,6 @@ yarn build Run the watcher: -```bash -$ yarn server - -# For development. -$ yarn server:dev - -# For specifying config file. -$ yarn server -f environments/local.toml -``` - Start the job runner: ```bash @@ -92,6 +82,18 @@ $ yarn job-runner:dev $ yarn job-runner -f environments/local.toml ``` +Start the server: + +```bash +$ yarn server + +# For development. +$ yarn server:dev + +# For specifying config file. +$ yarn server -f environments/local.toml +``` + GQL console: http://localhost:3001/graphql Deploy an ERC20 token: diff --git a/packages/erc20-watcher/package.json b/packages/erc20-watcher/package.json index b863358b..55488128 100644 --- a/packages/erc20-watcher/package.json +++ b/packages/erc20-watcher/package.json @@ -52,7 +52,6 @@ "ethers": "^5.4.4", "express": "^4.18.2", "graphql": "^15.5.0", - "graphql-import-node": "^0.0.4", "graphql-request": "^3.4.0", "graphql-subscriptions": "^2.0.0", "json-bigint": "^1.0.0", diff --git a/packages/erc20-watcher/src/server.ts b/packages/erc20-watcher/src/server.ts index fe9f6d11..cb9d3151 100644 --- a/packages/erc20-watcher/src/server.ts +++ b/packages/erc20-watcher/src/server.ts @@ -3,7 +3,6 @@ // import debug from 'debug'; -import 'graphql-import-node'; import { ServerCmd } from '@cerc-io/cli'; diff --git a/packages/erc721-watcher/README.md b/packages/erc721-watcher/README.md index 682e72c3..e616d858 100644 --- a/packages/erc721-watcher/README.md +++ b/packages/erc721-watcher/README.md @@ -71,11 +71,15 @@ Follow the steps below or follow the [Demo](./demo.md) * Run the watcher: - ```bash - yarn server - ``` +* If the watcher is a `lazy` watcher: -GQL console: http://localhost:3006/graphql + * Run the server: + + ```bash + yarn server + ``` + + GQL console: http://localhost:3006/graphql * If the watcher is an `active` watcher: @@ -85,6 +89,14 @@ GQL console: http://localhost:3006/graphql yarn job-runner ``` + * Run the server: + + ```bash + yarn server + ``` + + GQL console: http://localhost:3012/graphql + * To watch a contract: ```bash diff --git a/packages/erc721-watcher/package.json b/packages/erc721-watcher/package.json index 25a0a54c..fea96f07 100644 --- a/packages/erc721-watcher/package.json +++ b/packages/erc721-watcher/package.json @@ -58,7 +58,6 @@ "ethers": "^5.4.4", "express": "^4.18.2", "graphql": "^15.5.0", - "graphql-import-node": "^0.0.4", "graphql-subscriptions": "^2.0.0", "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", diff --git a/packages/erc721-watcher/src/server.ts b/packages/erc721-watcher/src/server.ts index 5c769659..20915d12 100644 --- a/packages/erc721-watcher/src/server.ts +++ b/packages/erc721-watcher/src/server.ts @@ -6,7 +6,6 @@ import fs from 'fs'; import path from 'path'; import 'reflect-metadata'; import debug from 'debug'; -import 'graphql-import-node'; import { ServerCmd } from '@cerc-io/cli'; diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index 7ba42f1c..364c6b2d 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -194,18 +194,6 @@ export class Database { } } - async getEntitiesForBlock (blockHash: string, tableName: string): Promise { - const repo = this._conn.getRepository(tableName); - - const entities = await repo.find({ - where: { - blockHash - } - }); - - return entities; - } - async getEntityIdsAtBlockNumber (blockNumber: number, tableName: string): Promise { const repo = this._conn.getRepository(tableName); diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index 958311ce..938a4ddc 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -12,10 +12,9 @@ import { SelectionNode } from 'graphql'; import { ResultObject } from '@vulcanize/assemblyscript/lib/loader'; import { EthClient } from '@cerc-io/ipld-eth-client'; -import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, StateInterface, IndexerInterface, BlockProgressInterface } from '@cerc-io/util'; +import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IndexerInterface, BlockProgressInterface } from '@cerc-io/util'; import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils'; -import { updateEntitiesFromState } from './state-utils'; import { Context, GraphData, instantiate } from './loader'; import { Database, DEFAULT_LIMIT } from './database'; @@ -349,12 +348,6 @@ export class GraphWatcher { } } - // TODO Remove after updating codegen CLIs - async updateEntitiesFromState (state: StateInterface) { - assert(this._indexer); - await updateEntitiesFromState(this._database, this._indexer, state); - } - updateEntityCacheFrothyBlocks (blockProgress: BlockProgressInterface): void { assert(this._indexer); this._database.updateEntityCacheFrothyBlocks(blockProgress, this._indexer.serverConfig.clearEntitiesCacheInterval); diff --git a/packages/graph-test-watcher/README.md b/packages/graph-test-watcher/README.md index 648569f1..aa5a0426 100644 --- a/packages/graph-test-watcher/README.md +++ b/packages/graph-test-watcher/README.md @@ -63,13 +63,13 @@ ## Run -* Run the watcher: + * Run the server: - ```bash - yarn server - ``` + ```bash + yarn server + ``` -GQL console: http://localhost:3008/graphql + GQL console: http://localhost:3008/graphql * If the watcher is an `active` watcher: @@ -79,6 +79,14 @@ GQL console: http://localhost:3008/graphql yarn job-runner ``` + * Run the server: + + ```bash + yarn server + ``` + + GQL console: http://localhost:3008/graphql + * To watch a contract: ```bash diff --git a/packages/graph-test-watcher/package.json b/packages/graph-test-watcher/package.json index 884cd803..fdcfb201 100644 --- a/packages/graph-test-watcher/package.json +++ b/packages/graph-test-watcher/package.json @@ -51,7 +51,6 @@ "ethers": "^5.4.4", "express": "^4.18.2", "graphql": "^15.5.0", - "graphql-import-node": "^0.0.4", "graphql-subscriptions": "^2.0.0", "json-bigint": "^1.0.0", "lodash": "^4.17.21", diff --git a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts index 51b127ca..9513d719 100644 --- a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts +++ b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts @@ -21,8 +21,8 @@ export const builder = { }; export const handler = async (argv: any): Promise => { - const createCheckpointCmd = new VerifyCheckpointCmd(); - await createCheckpointCmd.init(argv, Database, Indexer); + const verifyCheckpointCmd = new VerifyCheckpointCmd(); + await verifyCheckpointCmd.init(argv, Database, Indexer); - await createCheckpointCmd.exec(); + await verifyCheckpointCmd.exec(); }; diff --git a/packages/graph-test-watcher/src/server.ts b/packages/graph-test-watcher/src/server.ts index 45810884..3d432ae3 100644 --- a/packages/graph-test-watcher/src/server.ts +++ b/packages/graph-test-watcher/src/server.ts @@ -6,7 +6,6 @@ import fs from 'fs'; import path from 'path'; import 'reflect-metadata'; import debug from 'debug'; -import 'graphql-import-node'; import { ServerCmd } from '@cerc-io/cli'; diff --git a/packages/mobymask-watcher/README.md b/packages/mobymask-watcher/README.md index 2b63d434..bbc66634 100644 --- a/packages/mobymask-watcher/README.md +++ b/packages/mobymask-watcher/README.md @@ -63,13 +63,13 @@ Follow the steps below or follow the [Demo](./demo.md) -* Run the watcher: + * Run the server: - ```bash - yarn server - ``` + ```bash + yarn server + ``` -GQL console: http://localhost:3010/graphql + GQL console: http://localhost:3010/graphql * If the watcher is an `active` watcher: @@ -79,6 +79,14 @@ GQL console: http://localhost:3010/graphql yarn job-runner ``` + * Run the server: + + ```bash + yarn server + ``` + + GQL console: http://localhost:3010/graphql + * To watch a contract: ```bash diff --git a/packages/mobymask-watcher/package.json b/packages/mobymask-watcher/package.json index af7591f8..32c19780 100644 --- a/packages/mobymask-watcher/package.json +++ b/packages/mobymask-watcher/package.json @@ -49,7 +49,6 @@ "ethers": "^5.4.4", "express": "^4.18.2", "graphql": "^15.5.0", - "graphql-import-node": "^0.0.4", "graphql-subscriptions": "^2.0.0", "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", diff --git a/packages/mobymask-watcher/src/server.ts b/packages/mobymask-watcher/src/server.ts index 5c769659..20915d12 100644 --- a/packages/mobymask-watcher/src/server.ts +++ b/packages/mobymask-watcher/src/server.ts @@ -6,7 +6,6 @@ import fs from 'fs'; import path from 'path'; import 'reflect-metadata'; import debug from 'debug'; -import 'graphql-import-node'; import { ServerCmd } from '@cerc-io/cli'; diff --git a/yarn.lock b/yarn.lock index 3d39c4b5..bfc592cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7965,11 +7965,6 @@ graphql-compose@^9.0.3: graphql-type-json "0.3.2" object-path "0.11.7" -graphql-import-node@^0.0.4: - version "0.0.4" - resolved "https://registry.yarnpkg.com/graphql-import-node/-/graphql-import-node-0.0.4.tgz#0522f058978c7e1b99d1e6be1b851ee17007b111" - integrity sha512-okpdABQIgIM0qdx9Mfgdu6fTsFEazZmHZdEU34cijkUj9D1db1SyPRGHPxbXmbacamhEF41ckxpCAgHiGliryQ== - graphql-request@^3.4.0: version "3.4.0" resolved "https://registry.yarnpkg.com/graphql-request/-/graphql-request-3.4.0.tgz#3a400cd5511eb3c064b1873afb059196bbea9c2b"