Add a CLI to backfill watcher event data (#538)

* Add a CLI to backfill watcher event data

* Add required codegen template

* Increment package version

lerna.json

@@ -2,7 +2,7 @@
   "packages": [
     "packages/*"
   ],
-  "version": "0.2.108",
+  "version": "0.2.109",
   "npmClient": "yarn",
   "useWorkspaces": true,
   "command": {

packages/cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/cache",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "description": "Generic object cache",
   "main": "dist/index.js",
   "scripts": {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/cli",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "scripts": {
@@ -15,13 +15,13 @@
   },
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@cerc-io/cache": "^0.2.108",
+    "@cerc-io/cache": "^0.2.109",
-    "@cerc-io/ipld-eth-client": "^0.2.108",
+    "@cerc-io/ipld-eth-client": "^0.2.109",
     "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
     "@cerc-io/nitro-node": "^0.1.15",
-    "@cerc-io/peer": "^0.2.108",
+    "@cerc-io/peer": "^0.2.109",
-    "@cerc-io/rpc-eth-client": "^0.2.108",
+    "@cerc-io/rpc-eth-client": "^0.2.109",
-    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/util": "^0.2.109",
     "@ethersproject/providers": "^5.4.4",
     "@graphql-tools/utils": "^9.1.1",
     "@ipld/dag-cbor": "^8.0.0",

packages/cli/src/backfill-events-data.ts

@@ -0,0 +1,127 @@
+//
+// Copyright 2024 Vulcanize, Inc.
+//
+
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import assert from 'assert';
+import { ConnectionOptions, Repository } from 'typeorm';
+import debug from 'debug';
+
+import { DEFAULT_CONFIG_PATH, JSONbigNative, DatabaseInterface, Config, EventInterface } from '@cerc-io/util';
+
+import { BaseCmd } from './base';
+
+const log = debug('vulcanize:backfill-events-data');
+
+interface Arguments {
+  configFile: string;
+  batchSize: number;
+}
+
+export class BackfillEventsDataCmd {
+  _argv?: Arguments;
+  _baseCmd: BaseCmd;
+
+  constructor () {
+    this._baseCmd = new BaseCmd();
+  }
+
+  get config (): Config {
+    return this._baseCmd.config;
+  }
+
+  get database (): DatabaseInterface {
+    return this._baseCmd.database;
+  }
+
+  async initConfig<ConfigType> (): Promise<ConfigType> {
+    this._argv = this._getArgv();
+    assert(this._argv);
+
+    return this._baseCmd.initConfig(this._argv.configFile);
+  }
+
+  async init (
+    Database: new (
+      config: ConnectionOptions
+    ) => DatabaseInterface
+  ): Promise<void> {
+    await this.initConfig();
+
+    this._baseCmd._database = new Database(this.config.database);
+    await this.database.init();
+  }
+
+  async exec (eventEntity: new () => EventInterface): Promise<void> {
+    assert(this._argv);
+
+    const eventRepository: Repository<EventInterface> = this.database._conn.getRepository(eventEntity);
+
+    // Get the total count of events
+    const totalEvents = await eventRepository.count();
+
+    const batchSize = Number(this._argv.batchSize);
+    let page = 0;
+    let processedCount = 0;
+    let eventsWithNullData: EventInterface[];
+
+    while (processedCount < totalEvents) {
+      // Fetch events in batches with pagination
+      eventsWithNullData = await eventRepository.find({
+        order: { id: 'ASC' },
+        skip: page * batchSize,
+        take: batchSize
+      });
+
+      for (const event of eventsWithNullData) {
+        // Parse extra info and check if data field is present
+        const parsedExtraInfo = JSON.parse(event.extraInfo);
+
+        // Derive data and topics
+        if (parsedExtraInfo.data) {
+          event.data = parsedExtraInfo.data;
+          [event.topic0, event.topic1, event.topic2, event.topic3] = parsedExtraInfo.topics;
+
+          // Update extraInfo
+          delete parsedExtraInfo.data;
+          delete parsedExtraInfo.topics;
+
+          event.extraInfo = JSONbigNative.stringify(parsedExtraInfo);
+        }
+      }
+
+      // Save updated events
+      await eventRepository.save(eventsWithNullData);
+
+      // Update the processed count and progress
+      processedCount += eventsWithNullData.length;
+      const progress = ((processedCount / totalEvents) * 100).toFixed(2);
+      log(`Processed ${processedCount}/${totalEvents} events (${progress}% complete)`);
+
+      // Move to the next batch
+      eventsWithNullData = [];
+      page++;
+    }
+
+    log('Done.');
+    await this.database.close();
+  }
+
+  _getArgv (): any {
+    return yargs(hideBin(process.argv))
+      .option('configFile', {
+        alias: 'f',
+        describe: 'configuration file path (toml)',
+        type: 'string',
+        default: DEFAULT_CONFIG_PATH
+      })
+      .option('b', {
+        alias: 'batch-size',
+        describe: 'batch size to process events in',
+        type: 'number',
+        default: 1000
+      })
+      .argv;
+  }
+}

packages/cli/src/index.ts

@@ -17,3 +17,4 @@ export * from './fill';
 export * from './create-state-gql';
 export * from './peer';
 export * from './utils';
+export * from './backfill-events-data';

packages/codegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/codegen",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "description": "Code generator",
   "private": true,
   "main": "index.js",
@@ -20,7 +20,7 @@
   },
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
-    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/util": "^0.2.109",
     "@graphql-tools/load-files": "^6.5.2",
     "@npmcli/package-json": "^5.0.0",
     "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

packages/codegen/src/backfill-events-data.ts

@@ -0,0 +1,21 @@
+//
+// Copyright 2024 Vulcanize, Inc.
+//
+
+import fs from 'fs';
+import path from 'path';
+import Handlebars from 'handlebars';
+import { Writable } from 'stream';
+
+const TEMPLATE_FILE = './templates/backfill-events-data-template.handlebars';
+
+/**
+ * Writes the backfill-events-data file generated from a template to a stream.
+ * @param outStream A writable output stream to write the backfill-events-data file to.
+ */
+export function exportBackfillEventsData (outStream: Writable): void {
+  const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString();
+  const template = Handlebars.compile(templateString);
+  const content = template({});
+  outStream.write(content);
+}

packages/codegen/src/generate-code.ts

@@ -40,6 +40,7 @@ import { exportIndexBlock } from './index-block';
 import { exportSubscriber } from './subscriber';
 import { exportReset } from './reset';
 import { filterInheritedContractNodes, writeFileToStream } from './utils/helpers';
+import { exportBackfillEventsData } from './backfill-events-data';
 
 const main = async (): Promise<void> => {
   const argv = await yargs(hideBin(process.argv))
@@ -389,6 +390,11 @@ function generateWatcher (visitor: Visitor, contracts: any[], configFile: string
     : process.stdout;
   exportIndexBlock(outStream);
 
+  outStream = outputDir
+    ? fs.createWriteStream(path.join(outputDir, 'src/cli/backfill-events-data.ts'))
+    : process.stdout;
+  exportBackfillEventsData(outStream);
+
   if (config.subgraphPath) {
     outStream = outputDir
       ? fs.createWriteStream(path.join(outputDir, 'src/entity/Subscriber.ts'))

packages/codegen/src/templates/backfill-events-data-template.handlebars

@@ -0,0 +1,26 @@
+//
+// Copyright 2024 Vulcanize, Inc.
+//
+
+import 'reflect-metadata';
+import debug from 'debug';
+
+import { BackfillEventsDataCmd } from '@cerc-io/cli';
+
+import { Database } from '../database';
+import { Event } from '../entity/Event';
+
+const log = debug('vulcanize:backfill-events-data');
+
+const main = async (): Promise<void> => {
+  const backFillCmd = new BackfillEventsDataCmd();
+  await backFillCmd.init(Database);
+
+  await backFillCmd.exec(Event);
+};
+
+main().catch(err => {
+  log(err);
+}).finally(() => {
+  process.exit(0);
+});

packages/codegen/src/templates/package-template.handlebars

@@ -41,12 +41,12 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@cerc-io/cli": "^0.2.108",
+    "@cerc-io/cli": "^0.2.109",
-    "@cerc-io/ipld-eth-client": "^0.2.108",
+    "@cerc-io/ipld-eth-client": "^0.2.109",
-    "@cerc-io/solidity-mapper": "^0.2.108",
+    "@cerc-io/solidity-mapper": "^0.2.109",
-    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/util": "^0.2.109",
     {{#if (subgraphPath)}}
-    "@cerc-io/graph-node": "^0.2.108",
+    "@cerc-io/graph-node": "^0.2.109",
     {{/if}}
     "@ethersproject/providers": "^5.4.4",
     "debug": "^4.3.1",

packages/graph-node/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@cerc-io/graph-node",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "devDependencies": {
-    "@cerc-io/solidity-mapper": "^0.2.108",
+    "@cerc-io/solidity-mapper": "^0.2.109",
     "@ethersproject/providers": "^5.4.4",
     "@graphprotocol/graph-ts": "^0.22.0",
     "@nomiclabs/hardhat-ethers": "^2.0.2",
@@ -51,9 +51,9 @@
   "dependencies": {
     "@apollo/client": "^3.3.19",
     "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
-    "@cerc-io/cache": "^0.2.108",
+    "@cerc-io/cache": "^0.2.109",
-    "@cerc-io/ipld-eth-client": "^0.2.108",
+    "@cerc-io/ipld-eth-client": "^0.2.109",
-    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/util": "^0.2.109",
     "@types/json-diff": "^0.5.2",
     "@types/yargs": "^17.0.0",
     "bn.js": "^4.11.9",

packages/ipld-eth-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/ipld-eth-client",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "description": "IPLD ETH Client",
   "main": "dist/index.js",
   "scripts": {
@@ -20,8 +20,8 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@cerc-io/cache": "^0.2.108",
+    "@cerc-io/cache": "^0.2.109",
-    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/util": "^0.2.109",
     "cross-fetch": "^3.1.4",
     "debug": "^4.3.1",
     "ethers": "^5.4.4",

packages/peer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/peer",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "description": "libp2p module",
   "main": "dist/index.js",
   "exports": "./dist/index.js",

packages/rpc-eth-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/rpc-eth-client",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "description": "RPC ETH Client",
   "main": "dist/index.js",
   "scripts": {
@@ -19,9 +19,9 @@
   },
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
-    "@cerc-io/cache": "^0.2.108",
+    "@cerc-io/cache": "^0.2.109",
-    "@cerc-io/ipld-eth-client": "^0.2.108",
+    "@cerc-io/ipld-eth-client": "^0.2.109",
-    "@cerc-io/util": "^0.2.108",
+    "@cerc-io/util": "^0.2.109",
     "chai": "^4.3.4",
     "ethers": "^5.4.4",
     "left-pad": "^1.3.0",

packages/solidity-mapper/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/solidity-mapper",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "devDependencies": {

packages/test/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/test",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "private": true,

packages/tracing-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/tracing-client",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "description": "ETH VM tracing client",
   "main": "dist/index.js",
   "scripts": {

packages/util/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@cerc-io/util",
-  "version": "0.2.108",
+  "version": "0.2.109",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "dependencies": {
     "@apollo/utils.keyvaluecache": "^1.0.1",
     "@cerc-io/nitro-node": "^0.1.15",
-    "@cerc-io/peer": "^0.2.108",
+    "@cerc-io/peer": "^0.2.109",
-    "@cerc-io/solidity-mapper": "^0.2.108",
+    "@cerc-io/solidity-mapper": "^0.2.109",
     "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
     "@ethersproject/properties": "^5.7.0",
     "@ethersproject/providers": "^5.4.4",
@@ -55,7 +55,7 @@
     "yargs": "^17.0.1"
   },
   "devDependencies": {
-    "@cerc-io/cache": "^0.2.108",
+    "@cerc-io/cache": "^0.2.109",
     "@nomiclabs/hardhat-waffle": "^2.0.1",
     "@types/bunyan": "^1.8.8",
     "@types/express": "^4.17.14",