mirror of
https://github.com/cerc-io/watcher-ts
synced 2025-01-04 18:46:47 +00:00
Add a CLI to backfill watcher event data (#538)
* Add a CLI to backfill watcher event data * Add required codegen template * Increment package version
This commit is contained in:
prathamesh0
GitHub
parent
5d7b7fe5b4
commit
ac74da6ea6
@ -2,7 +2,7 @@
|
||||
"packages": [
|
||||
"packages/*"
|
||||
],
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"npmClient": "yarn",
|
||||
"useWorkspaces": true,
|
||||
"command": {
|
||||
|
||||
2
packages/cache/package.json
vendored
2
packages/cache/package.json
vendored
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/cache",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"description": "Generic object cache",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/cli",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"main": "dist/index.js",
|
||||
"license": "AGPL-3.0",
|
||||
"scripts": {
|
||||
@ -15,13 +15,13 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@apollo/client": "^3.7.1",
|
||||
"@cerc-io/cache": "^0.2.108",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.108",
|
||||
"@cerc-io/cache": "^0.2.109",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.109",
|
||||
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
|
||||
"@cerc-io/nitro-node": "^0.1.15",
|
||||
"@cerc-io/peer": "^0.2.108",
|
||||
"@cerc-io/rpc-eth-client": "^0.2.108",
|
||||
"@cerc-io/util": "^0.2.108",
|
||||
"@cerc-io/peer": "^0.2.109",
|
||||
"@cerc-io/rpc-eth-client": "^0.2.109",
|
||||
"@cerc-io/util": "^0.2.109",
|
||||
"@ethersproject/providers": "^5.4.4",
|
||||
"@graphql-tools/utils": "^9.1.1",
|
||||
"@ipld/dag-cbor": "^8.0.0",
|
||||
|
||||
127
packages/cli/src/backfill-events-data.ts
Normal file
127
packages/cli/src/backfill-events-data.ts
Normal file
@ -0,0 +1,127 @@
|
||||
//
|
||||
// Copyright 2024 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import yargs from 'yargs';
|
||||
import { hideBin } from 'yargs/helpers';
|
||||
import assert from 'assert';
|
||||
import { ConnectionOptions, Repository } from 'typeorm';
|
||||
import debug from 'debug';
|
||||
|
||||
import { DEFAULT_CONFIG_PATH, JSONbigNative, DatabaseInterface, Config, EventInterface } from '@cerc-io/util';
|
||||
|
||||
import { BaseCmd } from './base';
|
||||
|
||||
const log = debug('vulcanize:backfill-events-data');
|
||||
|
||||
interface Arguments {
|
||||
configFile: string;
|
||||
batchSize: number;
|
||||
}
|
||||
|
||||
export class BackfillEventsDataCmd {
|
||||
_argv?: Arguments;
|
||||
_baseCmd: BaseCmd;
|
||||
|
||||
constructor () {
|
||||
this._baseCmd = new BaseCmd();
|
||||
}
|
||||
|
||||
get config (): Config {
|
||||
return this._baseCmd.config;
|
||||
}
|
||||
|
||||
get database (): DatabaseInterface {
|
||||
return this._baseCmd.database;
|
||||
}
|
||||
|
||||
async initConfig<ConfigType> (): Promise<ConfigType> {
|
||||
this._argv = this._getArgv();
|
||||
assert(this._argv);
|
||||
|
||||
return this._baseCmd.initConfig(this._argv.configFile);
|
||||
}
|
||||
|
||||
async init (
|
||||
Database: new (
|
||||
config: ConnectionOptions
|
||||
) => DatabaseInterface
|
||||
): Promise<void> {
|
||||
await this.initConfig();
|
||||
|
||||
this._baseCmd._database = new Database(this.config.database);
|
||||
await this.database.init();
|
||||
}
|
||||
|
||||
async exec (eventEntity: new () => EventInterface): Promise<void> {
|
||||
assert(this._argv);
|
||||
|
||||
const eventRepository: Repository<EventInterface> = this.database._conn.getRepository(eventEntity);
|
||||
|
||||
// Get the total count of events
|
||||
const totalEvents = await eventRepository.count();
|
||||
|
||||
const batchSize = Number(this._argv.batchSize);
|
||||
let page = 0;
|
||||
let processedCount = 0;
|
||||
let eventsWithNullData: EventInterface[];
|
||||
|
||||
while (processedCount < totalEvents) {
|
||||
// Fetch events in batches with pagination
|
||||
eventsWithNullData = await eventRepository.find({
|
||||
order: { id: 'ASC' },
|
||||
skip: page * batchSize,
|
||||
take: batchSize
|
||||
});
|
||||
|
||||
for (const event of eventsWithNullData) {
|
||||
// Parse extra info and check if data field is present
|
||||
const parsedExtraInfo = JSON.parse(event.extraInfo);
|
||||
|
||||
// Derive data and topics
|
||||
if (parsedExtraInfo.data) {
|
||||
event.data = parsedExtraInfo.data;
|
||||
[event.topic0, event.topic1, event.topic2, event.topic3] = parsedExtraInfo.topics;
|
||||
|
||||
// Update extraInfo
|
||||
delete parsedExtraInfo.data;
|
||||
delete parsedExtraInfo.topics;
|
||||
|
||||
event.extraInfo = JSONbigNative.stringify(parsedExtraInfo);
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated events
|
||||
await eventRepository.save(eventsWithNullData);
|
||||
|
||||
// Update the processed count and progress
|
||||
processedCount += eventsWithNullData.length;
|
||||
const progress = ((processedCount / totalEvents) * 100).toFixed(2);
|
||||
log(`Processed ${processedCount}/${totalEvents} events (${progress}% complete)`);
|
||||
|
||||
// Move to the next batch
|
||||
eventsWithNullData = [];
|
||||
page++;
|
||||
}
|
||||
|
||||
log('Done.');
|
||||
await this.database.close();
|
||||
}
|
||||
|
||||
_getArgv (): any {
|
||||
return yargs(hideBin(process.argv))
|
||||
.option('configFile', {
|
||||
alias: 'f',
|
||||
describe: 'configuration file path (toml)',
|
||||
type: 'string',
|
||||
default: DEFAULT_CONFIG_PATH
|
||||
})
|
||||
.option('b', {
|
||||
alias: 'batch-size',
|
||||
describe: 'batch size to process events in',
|
||||
type: 'number',
|
||||
default: 1000
|
||||
})
|
||||
.argv;
|
||||
}
|
||||
}
|
||||
@ -17,3 +17,4 @@ export * from './fill';
|
||||
export * from './create-state-gql';
|
||||
export * from './peer';
|
||||
export * from './utils';
|
||||
export * from './backfill-events-data';
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/codegen",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"description": "Code generator",
|
||||
"private": true,
|
||||
"main": "index.js",
|
||||
@ -20,7 +20,7 @@
|
||||
},
|
||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||
"dependencies": {
|
||||
"@cerc-io/util": "^0.2.108",
|
||||
"@cerc-io/util": "^0.2.109",
|
||||
"@graphql-tools/load-files": "^6.5.2",
|
||||
"@npmcli/package-json": "^5.0.0",
|
||||
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
|
||||
|
||||
21
packages/codegen/src/backfill-events-data.ts
Normal file
21
packages/codegen/src/backfill-events-data.ts
Normal file
@ -0,0 +1,21 @@
|
||||
//
|
||||
// Copyright 2024 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import Handlebars from 'handlebars';
|
||||
import { Writable } from 'stream';
|
||||
|
||||
const TEMPLATE_FILE = './templates/backfill-events-data-template.handlebars';
|
||||
|
||||
/**
|
||||
* Writes the backfill-events-data file generated from a template to a stream.
|
||||
* @param outStream A writable output stream to write the backfill-events-data file to.
|
||||
*/
|
||||
export function exportBackfillEventsData (outStream: Writable): void {
|
||||
const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString();
|
||||
const template = Handlebars.compile(templateString);
|
||||
const content = template({});
|
||||
outStream.write(content);
|
||||
}
|
||||
@ -40,6 +40,7 @@ import { exportIndexBlock } from './index-block';
|
||||
import { exportSubscriber } from './subscriber';
|
||||
import { exportReset } from './reset';
|
||||
import { filterInheritedContractNodes, writeFileToStream } from './utils/helpers';
|
||||
import { exportBackfillEventsData } from './backfill-events-data';
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const argv = await yargs(hideBin(process.argv))
|
||||
@ -389,6 +390,11 @@ function generateWatcher (visitor: Visitor, contracts: any[], configFile: string
|
||||
: process.stdout;
|
||||
exportIndexBlock(outStream);
|
||||
|
||||
outStream = outputDir
|
||||
? fs.createWriteStream(path.join(outputDir, 'src/cli/backfill-events-data.ts'))
|
||||
: process.stdout;
|
||||
exportBackfillEventsData(outStream);
|
||||
|
||||
if (config.subgraphPath) {
|
||||
outStream = outputDir
|
||||
? fs.createWriteStream(path.join(outputDir, 'src/entity/Subscriber.ts'))
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
//
|
||||
// Copyright 2024 Vulcanize, Inc.
|
||||
//
|
||||
|
||||
import 'reflect-metadata';
|
||||
import debug from 'debug';
|
||||
|
||||
import { BackfillEventsDataCmd } from '@cerc-io/cli';
|
||||
|
||||
import { Database } from '../database';
|
||||
import { Event } from '../entity/Event';
|
||||
|
||||
const log = debug('vulcanize:backfill-events-data');
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const backFillCmd = new BackfillEventsDataCmd();
|
||||
await backFillCmd.init(Database);
|
||||
|
||||
await backFillCmd.exec(Event);
|
||||
};
|
||||
|
||||
main().catch(err => {
|
||||
log(err);
|
||||
}).finally(() => {
|
||||
process.exit(0);
|
||||
});
|
||||
@ -41,12 +41,12 @@
|
||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||
"dependencies": {
|
||||
"@apollo/client": "^3.3.19",
|
||||
"@cerc-io/cli": "^0.2.108",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.108",
|
||||
"@cerc-io/solidity-mapper": "^0.2.108",
|
||||
"@cerc-io/util": "^0.2.108",
|
||||
"@cerc-io/cli": "^0.2.109",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.109",
|
||||
"@cerc-io/solidity-mapper": "^0.2.109",
|
||||
"@cerc-io/util": "^0.2.109",
|
||||
{{#if (subgraphPath)}}
|
||||
"@cerc-io/graph-node": "^0.2.108",
|
||||
"@cerc-io/graph-node": "^0.2.109",
|
||||
{{/if}}
|
||||
"@ethersproject/providers": "^5.4.4",
|
||||
"debug": "^4.3.1",
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
{
|
||||
"name": "@cerc-io/graph-node",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"main": "dist/index.js",
|
||||
"license": "AGPL-3.0",
|
||||
"devDependencies": {
|
||||
"@cerc-io/solidity-mapper": "^0.2.108",
|
||||
"@cerc-io/solidity-mapper": "^0.2.109",
|
||||
"@ethersproject/providers": "^5.4.4",
|
||||
"@graphprotocol/graph-ts": "^0.22.0",
|
||||
"@nomiclabs/hardhat-ethers": "^2.0.2",
|
||||
@ -51,9 +51,9 @@
|
||||
"dependencies": {
|
||||
"@apollo/client": "^3.3.19",
|
||||
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
|
||||
"@cerc-io/cache": "^0.2.108",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.108",
|
||||
"@cerc-io/util": "^0.2.108",
|
||||
"@cerc-io/cache": "^0.2.109",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.109",
|
||||
"@cerc-io/util": "^0.2.109",
|
||||
"@types/json-diff": "^0.5.2",
|
||||
"@types/yargs": "^17.0.0",
|
||||
"bn.js": "^4.11.9",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/ipld-eth-client",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"description": "IPLD ETH Client",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
@ -20,8 +20,8 @@
|
||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||
"dependencies": {
|
||||
"@apollo/client": "^3.7.1",
|
||||
"@cerc-io/cache": "^0.2.108",
|
||||
"@cerc-io/util": "^0.2.108",
|
||||
"@cerc-io/cache": "^0.2.109",
|
||||
"@cerc-io/util": "^0.2.109",
|
||||
"cross-fetch": "^3.1.4",
|
||||
"debug": "^4.3.1",
|
||||
"ethers": "^5.4.4",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/peer",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"description": "libp2p module",
|
||||
"main": "dist/index.js",
|
||||
"exports": "./dist/index.js",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/rpc-eth-client",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"description": "RPC ETH Client",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
@ -19,9 +19,9 @@
|
||||
},
|
||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||
"dependencies": {
|
||||
"@cerc-io/cache": "^0.2.108",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.108",
|
||||
"@cerc-io/util": "^0.2.108",
|
||||
"@cerc-io/cache": "^0.2.109",
|
||||
"@cerc-io/ipld-eth-client": "^0.2.109",
|
||||
"@cerc-io/util": "^0.2.109",
|
||||
"chai": "^4.3.4",
|
||||
"ethers": "^5.4.4",
|
||||
"left-pad": "^1.3.0",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/solidity-mapper",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"main": "dist/index.js",
|
||||
"license": "AGPL-3.0",
|
||||
"devDependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/test",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"main": "dist/index.js",
|
||||
"license": "AGPL-3.0",
|
||||
"private": true,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@cerc-io/tracing-client",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"description": "ETH VM tracing client",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "@cerc-io/util",
|
||||
"version": "0.2.108",
|
||||
"version": "0.2.109",
|
||||
"main": "dist/index.js",
|
||||
"license": "AGPL-3.0",
|
||||
"dependencies": {
|
||||
"@apollo/utils.keyvaluecache": "^1.0.1",
|
||||
"@cerc-io/nitro-node": "^0.1.15",
|
||||
"@cerc-io/peer": "^0.2.108",
|
||||
"@cerc-io/solidity-mapper": "^0.2.108",
|
||||
"@cerc-io/peer": "^0.2.109",
|
||||
"@cerc-io/solidity-mapper": "^0.2.109",
|
||||
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
|
||||
"@ethersproject/properties": "^5.7.0",
|
||||
"@ethersproject/providers": "^5.4.4",
|
||||
@ -55,7 +55,7 @@
|
||||
"yargs": "^17.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@cerc-io/cache": "^0.2.108",
|
||||
"@cerc-io/cache": "^0.2.109",
|
||||
"@nomiclabs/hardhat-waffle": "^2.0.1",
|
||||
"@types/bunyan": "^1.8.8",
|
||||
"@types/express": "^4.17.14",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user