From ac74da6ea65a5a2a1e73deaac3da13f42e8f95d9 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:07:56 +0530 Subject: [PATCH] Add a CLI to backfill watcher event data (#538) * Add a CLI to backfill watcher event data * Add required codegen template * Increment package version --- lerna.json | 2 +- packages/cache/package.json | 2 +- packages/cli/package.json | 12 +- packages/cli/src/backfill-events-data.ts | 127 ++++++++++++++++++ packages/cli/src/index.ts | 1 + packages/codegen/package.json | 4 +- packages/codegen/src/backfill-events-data.ts | 21 +++ packages/codegen/src/generate-code.ts | 6 + .../backfill-events-data-template.handlebars | 26 ++++ .../src/templates/package-template.handlebars | 10 +- packages/graph-node/package.json | 10 +- packages/ipld-eth-client/package.json | 6 +- packages/peer/package.json | 2 +- packages/rpc-eth-client/package.json | 8 +- packages/solidity-mapper/package.json | 2 +- packages/test/package.json | 2 +- packages/tracing-client/package.json | 2 +- packages/util/package.json | 8 +- 18 files changed, 216 insertions(+), 35 deletions(-) create mode 100644 packages/cli/src/backfill-events-data.ts create mode 100644 packages/codegen/src/backfill-events-data.ts create mode 100644 packages/codegen/src/templates/backfill-events-data-template.handlebars diff --git a/lerna.json b/lerna.json index 505cd75a..1085a982 100644 --- a/lerna.json +++ b/lerna.json @@ -2,7 +2,7 @@ "packages": [ "packages/*" ], - "version": "0.2.108", + "version": "0.2.109", "npmClient": "yarn", "useWorkspaces": true, "command": { diff --git a/packages/cache/package.json b/packages/cache/package.json index afec33e5..f07ca902 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cache", - "version": "0.2.108", + "version": "0.2.109", "description": "Generic object cache", "main": "dist/index.js", "scripts": { diff --git a/packages/cli/package.json b/packages/cli/package.json index 5596a701..6c94e760 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cli", - "version": "0.2.108", + "version": "0.2.109", "main": "dist/index.js", "license": "AGPL-3.0", "scripts": { @@ -15,13 +15,13 @@ }, "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.108", - "@cerc-io/ipld-eth-client": "^0.2.108", + "@cerc-io/cache": "^0.2.109", + "@cerc-io/ipld-eth-client": "^0.2.109", "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.108", - "@cerc-io/rpc-eth-client": "^0.2.108", - "@cerc-io/util": "^0.2.108", + "@cerc-io/peer": "^0.2.109", + "@cerc-io/rpc-eth-client": "^0.2.109", + "@cerc-io/util": "^0.2.109", "@ethersproject/providers": "^5.4.4", "@graphql-tools/utils": "^9.1.1", "@ipld/dag-cbor": "^8.0.0", diff --git a/packages/cli/src/backfill-events-data.ts b/packages/cli/src/backfill-events-data.ts new file mode 100644 index 00000000..d7467144 --- /dev/null +++ b/packages/cli/src/backfill-events-data.ts @@ -0,0 +1,127 @@ +// +// Copyright 2024 Vulcanize, Inc. +// + +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import assert from 'assert'; +import { ConnectionOptions, Repository } from 'typeorm'; +import debug from 'debug'; + +import { DEFAULT_CONFIG_PATH, JSONbigNative, DatabaseInterface, Config, EventInterface } from '@cerc-io/util'; + +import { BaseCmd } from './base'; + +const log = debug('vulcanize:backfill-events-data'); + +interface Arguments { + configFile: string; + batchSize: number; +} + +export class BackfillEventsDataCmd { + _argv?: Arguments; + _baseCmd: BaseCmd; + + constructor () { + this._baseCmd = new BaseCmd(); + } + + get config (): Config { + return this._baseCmd.config; + } + + get database (): DatabaseInterface { + return this._baseCmd.database; + } + + async initConfig (): Promise { + this._argv = this._getArgv(); + assert(this._argv); + + return this._baseCmd.initConfig(this._argv.configFile); + } + + async init ( + Database: new ( + config: ConnectionOptions + ) => DatabaseInterface + ): Promise { + await this.initConfig(); + + this._baseCmd._database = new Database(this.config.database); + await this.database.init(); + } + + async exec (eventEntity: new () => EventInterface): Promise { + assert(this._argv); + + const eventRepository: Repository = this.database._conn.getRepository(eventEntity); + + // Get the total count of events + const totalEvents = await eventRepository.count(); + + const batchSize = Number(this._argv.batchSize); + let page = 0; + let processedCount = 0; + let eventsWithNullData: EventInterface[]; + + while (processedCount < totalEvents) { + // Fetch events in batches with pagination + eventsWithNullData = await eventRepository.find({ + order: { id: 'ASC' }, + skip: page * batchSize, + take: batchSize + }); + + for (const event of eventsWithNullData) { + // Parse extra info and check if data field is present + const parsedExtraInfo = JSON.parse(event.extraInfo); + + // Derive data and topics + if (parsedExtraInfo.data) { + event.data = parsedExtraInfo.data; + [event.topic0, event.topic1, event.topic2, event.topic3] = parsedExtraInfo.topics; + + // Update extraInfo + delete parsedExtraInfo.data; + delete parsedExtraInfo.topics; + + event.extraInfo = JSONbigNative.stringify(parsedExtraInfo); + } + } + + // Save updated events + await eventRepository.save(eventsWithNullData); + + // Update the processed count and progress + processedCount += eventsWithNullData.length; + const progress = ((processedCount / totalEvents) * 100).toFixed(2); + log(`Processed ${processedCount}/${totalEvents} events (${progress}% complete)`); + + // Move to the next batch + eventsWithNullData = []; + page++; + } + + log('Done.'); + await this.database.close(); + } + + _getArgv (): any { + return yargs(hideBin(process.argv)) + .option('configFile', { + alias: 'f', + describe: 'configuration file path (toml)', + type: 'string', + default: DEFAULT_CONFIG_PATH + }) + .option('b', { + alias: 'batch-size', + describe: 'batch size to process events in', + type: 'number', + default: 1000 + }) + .argv; + } +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 5b381481..3e4a0f5a 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -17,3 +17,4 @@ export * from './fill'; export * from './create-state-gql'; export * from './peer'; export * from './utils'; +export * from './backfill-events-data'; diff --git a/packages/codegen/package.json b/packages/codegen/package.json index a0ad5db7..48183eb0 100644 --- a/packages/codegen/package.json +++ b/packages/codegen/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/codegen", - "version": "0.2.108", + "version": "0.2.109", "description": "Code generator", "private": true, "main": "index.js", @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/util": "^0.2.108", + "@cerc-io/util": "^0.2.109", "@graphql-tools/load-files": "^6.5.2", "@npmcli/package-json": "^5.0.0", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", diff --git a/packages/codegen/src/backfill-events-data.ts b/packages/codegen/src/backfill-events-data.ts new file mode 100644 index 00000000..7da32aa3 --- /dev/null +++ b/packages/codegen/src/backfill-events-data.ts @@ -0,0 +1,21 @@ +// +// Copyright 2024 Vulcanize, Inc. +// + +import fs from 'fs'; +import path from 'path'; +import Handlebars from 'handlebars'; +import { Writable } from 'stream'; + +const TEMPLATE_FILE = './templates/backfill-events-data-template.handlebars'; + +/** + * Writes the backfill-events-data file generated from a template to a stream. + * @param outStream A writable output stream to write the backfill-events-data file to. + */ +export function exportBackfillEventsData (outStream: Writable): void { + const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString(); + const template = Handlebars.compile(templateString); + const content = template({}); + outStream.write(content); +} diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 1bf675b0..5b0e29d8 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -40,6 +40,7 @@ import { exportIndexBlock } from './index-block'; import { exportSubscriber } from './subscriber'; import { exportReset } from './reset'; import { filterInheritedContractNodes, writeFileToStream } from './utils/helpers'; +import { exportBackfillEventsData } from './backfill-events-data'; const main = async (): Promise => { const argv = await yargs(hideBin(process.argv)) @@ -389,6 +390,11 @@ function generateWatcher (visitor: Visitor, contracts: any[], configFile: string : process.stdout; exportIndexBlock(outStream); + outStream = outputDir + ? fs.createWriteStream(path.join(outputDir, 'src/cli/backfill-events-data.ts')) + : process.stdout; + exportBackfillEventsData(outStream); + if (config.subgraphPath) { outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/entity/Subscriber.ts')) diff --git a/packages/codegen/src/templates/backfill-events-data-template.handlebars b/packages/codegen/src/templates/backfill-events-data-template.handlebars new file mode 100644 index 00000000..1f3d9963 --- /dev/null +++ b/packages/codegen/src/templates/backfill-events-data-template.handlebars @@ -0,0 +1,26 @@ +// +// Copyright 2024 Vulcanize, Inc. +// + +import 'reflect-metadata'; +import debug from 'debug'; + +import { BackfillEventsDataCmd } from '@cerc-io/cli'; + +import { Database } from '../database'; +import { Event } from '../entity/Event'; + +const log = debug('vulcanize:backfill-events-data'); + +const main = async (): Promise => { + const backFillCmd = new BackfillEventsDataCmd(); + await backFillCmd.init(Database); + + await backFillCmd.exec(Event); +}; + +main().catch(err => { + log(err); +}).finally(() => { + process.exit(0); +}); diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index b0fefde3..f1e4c46c 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -41,12 +41,12 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.108", - "@cerc-io/ipld-eth-client": "^0.2.108", - "@cerc-io/solidity-mapper": "^0.2.108", - "@cerc-io/util": "^0.2.108", + "@cerc-io/cli": "^0.2.109", + "@cerc-io/ipld-eth-client": "^0.2.109", + "@cerc-io/solidity-mapper": "^0.2.109", + "@cerc-io/util": "^0.2.109", {{#if (subgraphPath)}} - "@cerc-io/graph-node": "^0.2.108", + "@cerc-io/graph-node": "^0.2.109", {{/if}} "@ethersproject/providers": "^5.4.4", "debug": "^4.3.1", diff --git a/packages/graph-node/package.json b/packages/graph-node/package.json index b9580456..0b9f0f1e 100644 --- a/packages/graph-node/package.json +++ b/packages/graph-node/package.json @@ -1,10 +1,10 @@ { "name": "@cerc-io/graph-node", - "version": "0.2.108", + "version": "0.2.109", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { - "@cerc-io/solidity-mapper": "^0.2.108", + "@cerc-io/solidity-mapper": "^0.2.109", "@ethersproject/providers": "^5.4.4", "@graphprotocol/graph-ts": "^0.22.0", "@nomiclabs/hardhat-ethers": "^2.0.2", @@ -51,9 +51,9 @@ "dependencies": { "@apollo/client": "^3.3.19", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", - "@cerc-io/cache": "^0.2.108", - "@cerc-io/ipld-eth-client": "^0.2.108", - "@cerc-io/util": "^0.2.108", + "@cerc-io/cache": "^0.2.109", + "@cerc-io/ipld-eth-client": "^0.2.109", + "@cerc-io/util": "^0.2.109", "@types/json-diff": "^0.5.2", "@types/yargs": "^17.0.0", "bn.js": "^4.11.9", diff --git a/packages/ipld-eth-client/package.json b/packages/ipld-eth-client/package.json index 2d84ab4d..ee58417e 100644 --- a/packages/ipld-eth-client/package.json +++ b/packages/ipld-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/ipld-eth-client", - "version": "0.2.108", + "version": "0.2.109", "description": "IPLD ETH Client", "main": "dist/index.js", "scripts": { @@ -20,8 +20,8 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.108", - "@cerc-io/util": "^0.2.108", + "@cerc-io/cache": "^0.2.109", + "@cerc-io/util": "^0.2.109", "cross-fetch": "^3.1.4", "debug": "^4.3.1", "ethers": "^5.4.4", diff --git a/packages/peer/package.json b/packages/peer/package.json index c2278ef7..f80ce19c 100644 --- a/packages/peer/package.json +++ b/packages/peer/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/peer", - "version": "0.2.108", + "version": "0.2.109", "description": "libp2p module", "main": "dist/index.js", "exports": "./dist/index.js", diff --git a/packages/rpc-eth-client/package.json b/packages/rpc-eth-client/package.json index 2c915b52..1450b860 100644 --- a/packages/rpc-eth-client/package.json +++ b/packages/rpc-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/rpc-eth-client", - "version": "0.2.108", + "version": "0.2.109", "description": "RPC ETH Client", "main": "dist/index.js", "scripts": { @@ -19,9 +19,9 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/cache": "^0.2.108", - "@cerc-io/ipld-eth-client": "^0.2.108", - "@cerc-io/util": "^0.2.108", + "@cerc-io/cache": "^0.2.109", + "@cerc-io/ipld-eth-client": "^0.2.109", + "@cerc-io/util": "^0.2.109", "chai": "^4.3.4", "ethers": "^5.4.4", "left-pad": "^1.3.0", diff --git a/packages/solidity-mapper/package.json b/packages/solidity-mapper/package.json index ae180933..323d5759 100644 --- a/packages/solidity-mapper/package.json +++ b/packages/solidity-mapper/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/solidity-mapper", - "version": "0.2.108", + "version": "0.2.109", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { diff --git a/packages/test/package.json b/packages/test/package.json index 787f9530..d49fff18 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/test", - "version": "0.2.108", + "version": "0.2.109", "main": "dist/index.js", "license": "AGPL-3.0", "private": true, diff --git a/packages/tracing-client/package.json b/packages/tracing-client/package.json index ad71d7a2..b5cceb93 100644 --- a/packages/tracing-client/package.json +++ b/packages/tracing-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/tracing-client", - "version": "0.2.108", + "version": "0.2.109", "description": "ETH VM tracing client", "main": "dist/index.js", "scripts": { diff --git a/packages/util/package.json b/packages/util/package.json index 3e47437f..ea3bf78a 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,13 +1,13 @@ { "name": "@cerc-io/util", - "version": "0.2.108", + "version": "0.2.109", "main": "dist/index.js", "license": "AGPL-3.0", "dependencies": { "@apollo/utils.keyvaluecache": "^1.0.1", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.108", - "@cerc-io/solidity-mapper": "^0.2.108", + "@cerc-io/peer": "^0.2.109", + "@cerc-io/solidity-mapper": "^0.2.109", "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1", "@ethersproject/properties": "^5.7.0", "@ethersproject/providers": "^5.4.4", @@ -55,7 +55,7 @@ "yargs": "^17.0.1" }, "devDependencies": { - "@cerc-io/cache": "^0.2.108", + "@cerc-io/cache": "^0.2.109", "@nomiclabs/hardhat-waffle": "^2.0.1", "@types/bunyan": "^1.8.8", "@types/express": "^4.17.14",