diff --git a/packages/codegen/src/fill.ts b/packages/codegen/src/fill.ts new file mode 100644 index 00000000..6d72107b --- /dev/null +++ b/packages/codegen/src/fill.ts @@ -0,0 +1,21 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import fs from 'fs'; +import path from 'path'; +import Handlebars from 'handlebars'; +import { Writable } from 'stream'; + +const TEMPLATE_FILE = './templates/fill-template.handlebars'; + +/** + * Writes the fill file generated from a template to a stream. + * @param outStream A writable output stream to write the fill file to. + */ +export function exportFill (outStream: Writable): void { + const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); + const template = Handlebars.compile(templateString); + const fill = template({}); + outStream.write(fill); +} diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 2c73774a..b7e57d3f 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -25,6 +25,8 @@ import { exportJobRunner } from './job-runner'; import { exportWatchContract } from './watch-contract'; import { exportLint } from './lint'; import { registerHandlebarHelpers } from './utils/handlebar-helpers'; +import { exportHooks } from './hooks'; +import { exportFill } from './fill'; const main = async (): Promise => { const argv = await yargs(hideBin(process.argv)) @@ -206,6 +208,22 @@ function generateWatcher (data: string, visitor: Visitor, argv: any) { : process.stdout; exportWatchContract(outStream); + let hooksOutStream; + let exampleOutStream; + if (outputDir) { + hooksOutStream = fs.createWriteStream(path.join(outputDir, 'src/hooks.ts')); + exampleOutStream = fs.createWriteStream(path.join(outputDir, 'src/hooks.example.ts')); + } else { + hooksOutStream = process.stdout; + exampleOutStream = process.stdout; + } + exportHooks(hooksOutStream, exampleOutStream); + + outStream = outputDir + ? fs.createWriteStream(path.join(outputDir, 'src/fill.ts')) + : process.stdout; + exportFill(outStream); + let rcOutStream; let ignoreOutStream; if (outputDir) { diff --git a/packages/codegen/src/hooks.ts b/packages/codegen/src/hooks.ts new file mode 100644 index 00000000..6c2783be --- /dev/null +++ b/packages/codegen/src/hooks.ts @@ -0,0 +1,30 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import fs from 'fs'; +import path from 'path'; +import Handlebars from 'handlebars'; +import { Writable } from 'stream'; + +const HOOKS_TEMPLATE_FILE = './templates/hooks-template.handlebars'; +const EXAMPLE_TEMPLATE_FILE = './templates/hooks-example-template.handlebars'; + +/** + * Writes the hooks and hooks.example files generated from templates to a stream. + * @param outStream A writable output stream to write the hooks file to. + * @param exampleOutStream A writable output stream to write the hooks.example file to. + */ +export function exportHooks (hooksOutStream: Writable, exampleOutStream: Writable): void { + const hooksTemplateString = fs.readFileSync(path.resolve(__dirname, HOOKS_TEMPLATE_FILE)).toString(); + const exampleTemplateString = fs.readFileSync(path.resolve(__dirname, EXAMPLE_TEMPLATE_FILE)).toString(); + + const hooksTemplate = Handlebars.compile(hooksTemplateString); + const exampleTemplate = Handlebars.compile(exampleTemplateString); + + const hooks = hooksTemplate({}); + const example = exampleTemplate({}); + + hooksOutStream.write(hooks); + exampleOutStream.write(example); +} diff --git a/packages/codegen/src/templates/events-template.handlebars b/packages/codegen/src/templates/events-template.handlebars index bd7eea99..c6b5d751 100644 --- a/packages/codegen/src/templates/events-template.handlebars +++ b/packages/codegen/src/templates/events-template.handlebars @@ -95,18 +95,13 @@ export class EventWatcher { async publishEventToSubscribers (dbEvent: Event, timeElapsedInSeconds: number): Promise { if (dbEvent && dbEvent.eventName !== UNKNOWN_EVENT_NAME) { - const { block: { blockHash }, contract: contractAddress } = dbEvent; const resultEvent = this._indexer.getResultEvent(dbEvent); log(`pushing event to GQL subscribers (${timeElapsedInSeconds}s elapsed): ${resultEvent.event.__typename}`); // Publishing the event here will result in pushing the payload to GQL subscribers for `onEvent`. await this._pubsub.publish(EVENT, { - onEvent: { - blockHash, - contractAddress, - event: resultEvent - } + onEvent: resultEvent }); } } diff --git a/packages/codegen/src/templates/fill-template.handlebars b/packages/codegen/src/templates/fill-template.handlebars new file mode 100644 index 00000000..ea43e1ca --- /dev/null +++ b/packages/codegen/src/templates/fill-template.handlebars @@ -0,0 +1,92 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import assert from 'assert'; +import 'reflect-metadata'; +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import debug from 'debug'; +import { PubSub } from 'apollo-server-express'; +import { getDefaultProvider } from 'ethers'; + +import { getCache } from '@vulcanize/cache'; +import { EthClient } from '@vulcanize/ipld-eth-client'; +import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH } from '@vulcanize/util'; + +import { Database } from './database'; +import { Indexer } from './indexer'; +import { EventWatcher } from './events'; + +const log = debug('vulcanize:server'); + +export const main = async (): Promise => { + const argv = await yargs(hideBin(process.argv)).parserConfiguration({ + 'parse-numbers': false + }).options({ + configFile: { + alias: 'f', + type: 'string', + demandOption: true, + describe: 'configuration file path (toml)', + default: DEFAULT_CONFIG_PATH + }, + startBlock: { + type: 'number', + demandOption: true, + describe: 'Block number to start processing at' + }, + endBlock: { + type: 'number', + demandOption: true, + describe: 'Block number to stop processing at' + } + }).argv; + + const config = await getConfig(argv.configFile); + + assert(config.server, 'Missing server config'); + + const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config; + + assert(dbConfig, 'Missing database config'); + + const db = new Database(dbConfig); + await db.init(); + + assert(upstream, 'Missing upstream config'); + const { ethServer: { gqlPostgraphileEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstream; + assert(gqlPostgraphileEndpoint, 'Missing upstream ethServer.gqlPostgraphileEndpoint'); + + const cache = await getCache(cacheConfig); + const ethClient = new EthClient({ + gqlEndpoint: gqlPostgraphileEndpoint, + gqlSubscriptionEndpoint: gqlPostgraphileEndpoint, + cache + }); + + const ethProvider = getDefaultProvider(rpcProviderEndpoint); + + // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. + // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries + const pubsub = new PubSub(); + const indexer = new Indexer(db, ethClient, ethProvider); + + const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; + assert(dbConnectionString, 'Missing job queue db connection string'); + + const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); + await jobQueue.start(); + + const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue); + + assert(jobQueueConfig, 'Missing job queue config'); + + await fillBlocks(jobQueue, indexer, ethClient, eventWatcher, argv); +}; + +main().then(() => { + process.exit(); +}).catch(err => { + log(err); +}); diff --git a/packages/codegen/src/templates/hooks-example-template.handlebars b/packages/codegen/src/templates/hooks-example-template.handlebars new file mode 100644 index 00000000..04b5ebf2 --- /dev/null +++ b/packages/codegen/src/templates/hooks-example-template.handlebars @@ -0,0 +1,51 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import assert from 'assert'; + +import { Indexer, ResultEvent } from './indexer'; + +/** + * Event hook function. + * @param indexer Indexer instance that contains methods to fetch and update the contract values in the database. + * @param eventData ResultEvent object containing necessary information. + */ +export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Promise { + assert(indexer); + assert(eventData); + + // The following code is for ERC20 contract implementation. + + // Perform indexing based on the type of event. + switch (eventData.event.__typename) { + // In case of ERC20 'Transfer' event. + case 'TransferEvent': { + // On a transfer, balances for both parties change. + // Therefore, trigger indexing for both sender and receiver. + + // Get event fields from eventData. + // const { from, to } = eventData.event; + + // Update balance entry for sender in the database. + // await indexer.balanceOf(eventData.block.hash, eventData.contract, from); + + // Update balance entry for receiver in the database. + // await indexer.balanceOf(eventData.block.hash, eventData.contract, to); + + break; + } + // In case of ERC20 'Approval' event. + case 'ApprovalEvent': { + // On an approval, allowance for (owner, spender) combination changes. + + // Get event fields from eventData. + // const { owner, spender } = eventData.event; + + // Update allowance entry for (owner, spender) combination in the database. + // await indexer.allowance(eventData.block.hash, eventData.contract, owner, spender); + + break; + } + } +} diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars new file mode 100644 index 00000000..27e856ae --- /dev/null +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -0,0 +1,19 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import assert from 'assert'; + +import { Indexer, ResultEvent } from './indexer'; + +/** + * Event hook function. + * @param indexer Indexer instance that contains methods to fetch and update the contract values in the database. + * @param eventData ResultEvent object containing necessary information. + */ +export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Promise { + assert(indexer); + assert(eventData); + + // Perform indexing based on the type of event. +} diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index ee709b78..86564a89 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -20,6 +20,7 @@ import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; import artifacts from './artifacts/{{inputFileName}}.json'; +import { handleEvent } from './hooks'; const log = debug('vulcanize:indexer'); @@ -27,9 +28,16 @@ const log = debug('vulcanize:indexer'); const {{capitalize event.name}}_EVENT = '{{event.name}}'; {{/each}} -interface ResultEvent { - block: any; - tx: any; +export type ResultEvent = { + block: { + hash: string; + number: number; + timestamp: number; + parentHash: string; + }; + tx: { + hash: string; + }; contract: string; @@ -153,8 +161,10 @@ export class Indexer { {{/each}} async triggerIndexingOnEvent (event: Event): Promise { - // TODO: Implement custom hooks. - assert(event); + const resultEvent = this.getResultEvent(event); + + // Call custom hook function for indexing on event. + await handleEvent(this, resultEvent); } async processEvent (event: Event): Promise { diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index 0e50699e..1358b7de 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -9,7 +9,8 @@ "build": "tsc", "server": "DEBUG=vulcanize:* ts-node src/server.ts", "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts", - "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts" + "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts", + "fill": "DEBUG=vulcanize:* ts-node src/fill.ts" }, "repository": { "type": "git", diff --git a/packages/codegen/src/templates/readme-template.handlebars b/packages/codegen/src/templates/readme-template.handlebars index 661f23df..6cec7402 100644 --- a/packages/codegen/src/templates/readme-template.handlebars +++ b/packages/codegen/src/templates/readme-template.handlebars @@ -35,9 +35,17 @@ {{folderName}}-job-queue=# exit ``` -* Update `environments/local.toml` with database connection settings. +* Update the [config](./environments/local.toml) with database connection settings. -* Update the `upstream` config in `environments/local.toml` and provide the `ipld-eth-server` GQL API and the `indexer-db` postgraphile endpoints. +* Update the `upstream` config in the [config file](./environments/local.toml) and provide the `ipld-eth-server` GQL API and the `indexer-db` postgraphile endpoints. + +## Customize + +* Indexing on an event: + + * Edit the custom hook function `handleEvent` (triggered on an event) in [hooks.ts](./src/hooks.ts) to perform corresponding indexing using the `Indexer` object. + + * Refer to [hooks.example.ts](./src/hooks.example.ts) for an example hook function for events in an ERC20 contract. ## Run @@ -60,5 +68,11 @@ GQL console: http://localhost:3008/graphql * To watch a contract: ```bash - yarn watch:contract --address CONTRACT_ADDRESS --kind {{contractName}} --starting-block BLOCK_NUMBER + yarn watch:contract --address --kind {{contractName}} --starting-block [block-number] + ``` + + * To fill a block range: + + ```bash + yarn fill --startBlock --endBlock ``` diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index 81dcdcf8..81ecee1b 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -9,10 +9,11 @@ import debug from 'debug'; import { ValueResult } from '@vulcanize/util'; import { Indexer } from './indexer'; +import { EventWatcher } from './events'; const log = debug('vulcanize:resolver'); -export const createResolvers = async (indexer: Indexer): Promise => { +export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatcher): Promise => { assert(indexer); return { @@ -26,6 +27,12 @@ export const createResolvers = async (indexer: Indexer): Promise => { } }, + Subscription: { + onEvent: { + subscribe: () => eventWatcher.getEventIterator() + } + }, + Query: { {{#each queries}} {{this.name}}: (_: any, { blockHash, contractAddress diff --git a/packages/codegen/src/templates/server-template.handlebars b/packages/codegen/src/templates/server-template.handlebars index 2b6f4214..c4fb0a24 100644 --- a/packages/codegen/src/templates/server-template.handlebars +++ b/packages/codegen/src/templates/server-template.handlebars @@ -66,23 +66,24 @@ export const main = async (): Promise => { const indexer = new Indexer(db, ethClient, ethProvider); + // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. + // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries + const pubsub = new PubSub(); + + assert(jobQueueConfig, 'Missing job queue config'); + const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; + assert(dbConnectionString, 'Missing job queue db connection string'); + + const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); + + const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue); + if (watcherKind === KIND_ACTIVE) { - // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. - // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries - const pubsub = new PubSub(); - - assert(jobQueueConfig, 'Missing job queue config'); - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); await jobQueue.start(); - - const eventWatcher = new EventWatcher(ethClient, indexer, pubsub, jobQueue); await eventWatcher.start(); } - const resolvers = await createResolvers(indexer); + const resolvers = await createResolvers(indexer, eventWatcher); const app: Application = express(); const typeDefs = fs.readFileSync(path.join(__dirname, 'schema.gql')).toString();