From 122a64c2f94492ad10a8d5e8fdf6cd0b5f13a846 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Tue, 22 Nov 2022 06:57:49 -0600 Subject: [PATCH] Refactor export-state and verify checkpoint CLIs to cli package (#252) * Refactor export-state CLI to cli package * Refactor verify checkpoint CLI to cli package * Update mock indexer object --- packages/cli/src/checkpoint/create.ts | 2 - packages/cli/src/checkpoint/verify.ts | 86 +++++++++ packages/cli/src/export-state.ts | 182 ++++++++++++++++++ packages/cli/src/index.ts | 2 + .../src/cli/checkpoint-cmds/verify.ts | 49 +---- packages/eden-watcher/src/cli/export-state.ts | 135 +------------ packages/erc20-watcher/src/indexer.ts | 16 ++ .../erc721-watcher/src/cli/export-state.ts | 124 +----------- packages/graph-node/test/utils/indexer.ts | 12 ++ .../src/cli/checkpoint-cmds/verify.ts | 49 +---- .../src/cli/export-state.ts | 135 +------------ .../mobymask-watcher/src/cli/export-state.ts | 124 +----------- packages/util/src/types.ts | 7 +- 13 files changed, 333 insertions(+), 590 deletions(-) create mode 100644 packages/cli/src/checkpoint/verify.ts create mode 100644 packages/cli/src/export-state.ts diff --git a/packages/cli/src/checkpoint/create.ts b/packages/cli/src/checkpoint/create.ts index 94b7622a..58dd9e3b 100644 --- a/packages/cli/src/checkpoint/create.ts +++ b/packages/cli/src/checkpoint/create.ts @@ -30,8 +30,6 @@ interface Arguments { export class CreateCheckpointCmd { _argv?: Arguments _baseCmd: BaseCmd - _database?: DatabaseInterface - _indexer?: IndexerInterface constructor () { this._baseCmd = new BaseCmd(); diff --git a/packages/cli/src/checkpoint/verify.ts b/packages/cli/src/checkpoint/verify.ts new file mode 100644 index 00000000..b602fb86 --- /dev/null +++ b/packages/cli/src/checkpoint/verify.ts @@ -0,0 +1,86 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import debug from 'debug'; +import 'reflect-metadata'; +import assert from 'assert'; +import { ConnectionOptions } from 'typeorm'; + +import { JsonRpcProvider } from '@ethersproject/providers'; +import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; +import { + JobQueue, + DatabaseInterface, + IndexerInterface, + ServerConfig, + Clients, + verifyCheckpointData +} from '@cerc-io/util'; + +import { BaseCmd } from '../base'; + +const log = debug('vulcanize:checkpoint-verify'); + +interface Arguments { + configFile: string; + cid: string; +} + +export class VerifyCheckpointCmd { + _argv?: Arguments + _baseCmd: BaseCmd + + constructor () { + this._baseCmd = new BaseCmd(); + } + + async initConfig (configFile: string): Promise { + return this._baseCmd.initConfig(configFile); + } + + async init ( + argv: any, + Database: new ( + config: ConnectionOptions, + serverConfig?: ServerConfig + ) => DatabaseInterface, + Indexer: new ( + serverConfig: ServerConfig, + db: DatabaseInterface, + clients: Clients, + ethProvider: JsonRpcProvider, + jobQueue: JobQueue, + graphWatcher?: GraphWatcher + ) => IndexerInterface, + clients: { [key: string]: any } = {} + ): Promise { + this._argv = argv; + await this.initConfig(argv.configFile); + + await this._baseCmd.init(Database, Indexer, clients); + } + + async exec (): Promise { + assert(this._argv); + + const database = this._baseCmd.database; + const indexer = this._baseCmd.indexer; + + assert(database); + assert(indexer); + + const graphDb: GraphDatabase | undefined = this._baseCmd.graphDb || database.graphDatabase; + assert(graphDb); + + const state = await indexer.getStateByCID(this._argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); + const data = indexer.getStateData(state); + + log(`Verifying checkpoint data for contract ${state.contractAddress}`); + await verifyCheckpointData(graphDb, state.block, data); + log('Checkpoint data verified'); + + await database.close(); + } +} diff --git a/packages/cli/src/export-state.ts b/packages/cli/src/export-state.ts new file mode 100644 index 00000000..7bee68d5 --- /dev/null +++ b/packages/cli/src/export-state.ts @@ -0,0 +1,182 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import yargs from 'yargs'; +import 'reflect-metadata'; +import assert from 'assert'; +import path from 'path'; +import fs from 'fs'; +import debug from 'debug'; +import { ConnectionOptions } from 'typeorm'; + +import { JsonRpcProvider } from '@ethersproject/providers'; +import { GraphWatcher } from '@cerc-io/graph-node'; +import { + DEFAULT_CONFIG_PATH, + JobQueue, + DatabaseInterface, + IndexerInterface, + ServerConfig, + StateKind, + Clients +} from '@cerc-io/util'; +import * as codec from '@ipld/dag-cbor'; + +import { BaseCmd } from './base'; + +const log = debug('vulcanize:export-state'); + +interface Arguments { + configFile: string; + exportFile: string; + blockNumber: number; +} + +export class ExportStateCmd { + _argv?: Arguments + _baseCmd: BaseCmd; + + constructor () { + this._baseCmd = new BaseCmd(); + } + + async initConfig (): Promise { + this._argv = this._getArgv(); + assert(this._argv); + + return this._baseCmd.initConfig(this._argv.configFile); + } + + async init ( + Database: new (config: ConnectionOptions, + serverConfig?: ServerConfig + ) => DatabaseInterface, + Indexer: new ( + serverConfig: ServerConfig, + db: DatabaseInterface, + clients: Clients, + ethProvider: JsonRpcProvider, + jobQueue: JobQueue, + graphWatcher?: GraphWatcher + ) => IndexerInterface, + clients: { [key: string]: any } = {} + ): Promise { + await this.initConfig(); + + await this._baseCmd.init(Database, Indexer, clients); + } + + async exec (): Promise { + assert(this._argv); + + const database = this._baseCmd.database; + const indexer = this._baseCmd.indexer; + + assert(database); + assert(indexer); + + const exportData: any = { + snapshotBlock: {}, + contracts: [], + stateCheckpoints: [] + }; + + const contracts = await database.getContracts(); + let block = await indexer.getLatestStateIndexedBlock(); + assert(block); + + if (this._argv.blockNumber) { + if (this._argv.blockNumber > block.blockNumber) { + throw new Error(`Export snapshot block height ${this._argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); + } + + const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(this._argv.blockNumber, false); + + if (!blocksAtSnapshotHeight.length) { + throw new Error(`No blocks at snapshot height ${this._argv.blockNumber}`); + } + + block = blocksAtSnapshotHeight[0]; + } + + log(`Creating export snapshot at block height ${block.blockNumber}`); + + // Export snapshot block. + exportData.snapshotBlock = { + blockNumber: block.blockNumber, + blockHash: block.blockHash + }; + + // Export contracts and checkpoints. + for (const contract of contracts) { + if (contract.startingBlock > block.blockNumber) { + continue; + } + + exportData.contracts.push({ + address: contract.address, + kind: contract.kind, + checkpoint: contract.checkpoint, + startingBlock: block.blockNumber + }); + + // Create and export checkpoint if checkpointing is on for the contract. + if (contract.checkpoint) { + await indexer.createCheckpoint(contract.address, block.blockHash); + + const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); + assert(state); + + const data = indexer.getStateData(state); + + exportData.stateCheckpoints.push({ + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, + data + }); + } + } + + if (this._argv.exportFile) { + const encodedExportData = codec.encode(exportData); + + const filePath = path.resolve(this._argv.exportFile); + const fileDir = path.dirname(filePath); + + if (!fs.existsSync(fileDir)) fs.mkdirSync(fileDir, { recursive: true }); + + fs.writeFileSync(filePath, encodedExportData); + } else { + log(exportData); + } + + log(`Export completed at height ${block.blockNumber}`); + await database.close(); + } + + _getArgv (): any { + return yargs.parserConfiguration({ + 'parse-numbers': false + }).options({ + configFile: { + alias: 'f', + type: 'string', + require: true, + demandOption: true, + describe: 'Configuration file path (toml)', + default: DEFAULT_CONFIG_PATH + }, + exportFile: { + alias: 'o', + type: 'string', + describe: 'Export file path' + }, + blockNumber: { + type: 'number', + describe: 'Block number to create snapshot at' + } + }).argv; + } +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index f8a94c26..47c4c425 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -6,5 +6,7 @@ export * from './watch-contract'; export * from './reset/watcher'; export * from './reset/state'; export * from './checkpoint/create'; +export * from './checkpoint/verify'; export * from './inspect-cid'; export * from './import-state'; +export * from './export-state'; diff --git a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts index 84419586..51b127ca 100644 --- a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts +++ b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts @@ -2,64 +2,27 @@ // Copyright 2022 Vulcanize, Inc. // -import debug from 'debug'; -import assert from 'assert'; +import { VerifyCheckpointCmd } from '@cerc-io/cli'; -import { getConfig, initClients, JobQueue, Config, verifyCheckpointData } from '@cerc-io/util'; -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; - -import { Database, ENTITY_TO_LATEST_ENTITY_MAP } from '../../database'; +import { Database } from '../../database'; import { Indexer } from '../../indexer'; -const log = debug('vulcanize:checkpoint-verify'); - export const command = 'verify'; export const desc = 'Verify checkpoint'; export const builder = { cid: { - alias: 'c', type: 'string', + alias: 'c', demandOption: true, describe: 'Checkpoint CID to be verified' } }; export const handler = async (argv: any): Promise => { - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); + const createCheckpointCmd = new VerifyCheckpointCmd(); + await createCheckpointCmd.init(argv, Database, Indexer); - const db = new Database(config.database); - await db.init(); - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, { ethClient }, ethProvider, jobQueue, graphWatcher); - await indexer.init(); - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - const state = await indexer.getStateByCID(argv.cid); - assert(state, 'State for the provided CID doesn\'t exist.'); - const data = indexer.getStateData(state); - - log(`Verifying checkpoint data for contract ${state.contractAddress}`); - await verifyCheckpointData(graphDb, state.block, data); - log('Checkpoint data verified'); - - await db.close(); + await createCheckpointCmd.exec(); }; diff --git a/packages/eden-watcher/src/cli/export-state.ts b/packages/eden-watcher/src/cli/export-state.ts index 33f7ff27..2411b960 100644 --- a/packages/eden-watcher/src/cli/export-state.ts +++ b/packages/eden-watcher/src/cli/export-state.ts @@ -2,146 +2,21 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import fs from 'fs'; -import path from 'path'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util'; -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -import * as codec from '@ipld/dag-cbor'; +import { ExportStateCmd } from '@cerc-io/cli'; -import { Database, ENTITY_TO_LATEST_ENTITY_MAP } from '../database'; +import { Database } from '../database'; import { Indexer } from '../indexer'; const log = debug('vulcanize:export-state'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - exportFile: { - alias: 'o', - type: 'string', - describe: 'Export file path' - }, - blockNumber: { - type: 'number', - describe: 'Block number to create snapshot at' - } - }).argv; + const exportStateCmd = new ExportStateCmd(); + await exportStateCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, { ethClient }, ethProvider, jobQueue, graphWatcher); - await indexer.init(); - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - const exportData: any = { - snapshotBlock: {}, - contracts: [], - stateCheckpoints: [] - }; - - const contracts = await db.getContracts(); - let block = await indexer.getLatestStateIndexedBlock(); - assert(block); - - if (argv.blockNumber) { - if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); - } - - const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); - - if (!blocksAtSnapshotHeight.length) { - throw new Error(`No blocks at snapshot height ${argv.blockNumber}`); - } - - block = blocksAtSnapshotHeight[0]; - } - - log(`Creating export snapshot at block height ${block.blockNumber}`); - - // Export snapshot block. - exportData.snapshotBlock = { - blockNumber: block.blockNumber, - blockHash: block.blockHash - }; - - // Export contracts and checkpoints. - for (const contract of contracts) { - if (contract.startingBlock > block.blockNumber) { - continue; - } - - exportData.contracts.push({ - address: contract.address, - kind: contract.kind, - checkpoint: contract.checkpoint, - startingBlock: block.blockNumber - }); - - // Create and export checkpoint if checkpointing is on for the contract. - if (contract.checkpoint) { - await indexer.createCheckpoint(contract.address, block.blockHash); - - const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(state); - - const data = indexer.getStateData(state); - - exportData.stateCheckpoints.push({ - contractAddress: state.contractAddress, - cid: state.cid, - kind: state.kind, - data - }); - } - } - - if (argv.exportFile) { - const encodedExportData = codec.encode(exportData); - - const filePath = path.resolve(argv.exportFile); - const fileDir = path.dirname(filePath); - - if (!fs.existsSync(fileDir)) fs.mkdirSync(fileDir, { recursive: true }); - - fs.writeFileSync(filePath, encodedExportData); - } else { - log(exportData); - } + await exportStateCmd.exec(); }; main().catch(err => { diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 9bf8ba8f..5cc44f0d 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -280,11 +280,22 @@ export class Indexer implements IndexerInterface { return undefined; } + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + // TODO Implement + return undefined; + } + async getStateByCID (cid: string): Promise { // TODO Implement return undefined; } + // Method to be used by export-state CLI. + async createCheckpoint (contractAddress: string, blockHash: string): Promise { + // TODO Implement + return undefined; + } + async saveOrUpdateState (state: State): Promise { return {} as State; } @@ -357,6 +368,11 @@ export class Indexer implements IndexerInterface { return {} as StateSyncStatus; } + async getLatestStateIndexedBlock (): Promise { + // TODO Implement + return {} as BlockProgress; + } + async getLatestCanonicalBlock (): Promise { const syncStatus = await this.getSyncStatus(); assert(syncStatus); diff --git a/packages/erc721-watcher/src/cli/export-state.ts b/packages/erc721-watcher/src/cli/export-state.ts index 39f1803e..2411b960 100644 --- a/packages/erc721-watcher/src/cli/export-state.ts +++ b/packages/erc721-watcher/src/cli/export-state.ts @@ -2,15 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import fs from 'fs'; -import path from 'path'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util'; -import * as codec from '@ipld/dag-cbor'; +import { ExportStateCmd } from '@cerc-io/cli'; import { Database } from '../database'; import { Indexer } from '../indexer'; @@ -18,121 +13,10 @@ import { Indexer } from '../indexer'; const log = debug('vulcanize:export-state'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - exportFile: { - alias: 'o', - type: 'string', - describe: 'Export file path' - }, - blockNumber: { - type: 'number', - describe: 'Block number to create snapshot at' - } - }).argv; + const exportStateCmd = new ExportStateCmd(); + await exportStateCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, { ethClient }, ethProvider, jobQueue); - await indexer.init(); - - const exportData: any = { - snapshotBlock: {}, - contracts: [], - stateCheckpoints: [] - }; - - const contracts = await db.getContracts(); - let block = await indexer.getLatestStateIndexedBlock(); - assert(block); - - if (argv.blockNumber) { - if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); - } - - const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); - - if (!blocksAtSnapshotHeight.length) { - throw new Error(`No blocks at snapshot height ${argv.blockNumber}`); - } - - block = blocksAtSnapshotHeight[0]; - } - - log(`Creating export snapshot at block height ${block.blockNumber}`); - - // Export snapshot block. - exportData.snapshotBlock = { - blockNumber: block.blockNumber, - blockHash: block.blockHash - }; - - // Export contracts and checkpoints. - for (const contract of contracts) { - if (contract.startingBlock > block.blockNumber) { - continue; - } - - exportData.contracts.push({ - address: contract.address, - kind: contract.kind, - checkpoint: contract.checkpoint, - startingBlock: block.blockNumber - }); - - // Create and export checkpoint if checkpointing is on for the contract. - if (contract.checkpoint) { - await indexer.createCheckpoint(contract.address, block.blockHash); - - const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(state); - - const data = indexer.getStateData(state); - - exportData.stateCheckpoints.push({ - contractAddress: state.contractAddress, - cid: state.cid, - kind: state.kind, - data - }); - } - } - - if (argv.exportFile) { - const encodedExportData = codec.encode(exportData); - - const filePath = path.resolve(argv.exportFile); - const fileDir = path.dirname(filePath); - - if (!fs.existsSync(fileDir)) fs.mkdirSync(fileDir, { recursive: true }); - - fs.writeFileSync(filePath, encodedExportData); - } else { - log(exportData); - } + await exportStateCmd.exec(); }; main().catch(err => { diff --git a/packages/graph-node/test/utils/indexer.ts b/packages/graph-node/test/utils/indexer.ts index 42ee273b..4dfabca3 100644 --- a/packages/graph-node/test/utils/indexer.ts +++ b/packages/graph-node/test/utils/indexer.ts @@ -209,10 +209,22 @@ export class Indexer implements IndexerInterface { return undefined; } + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + return undefined; + } + async getStateByCID (cid: string): Promise { return undefined; } + async createCheckpoint (contractAddress: string, blockHash: string): Promise { + return undefined; + } + + async getLatestStateIndexedBlock (): Promise { + return {} as BlockProgressInterface; + } + async saveOrUpdateState (state: StateInterface): Promise { return {} as StateInterface; } diff --git a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts index 84419586..51b127ca 100644 --- a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts +++ b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts @@ -2,64 +2,27 @@ // Copyright 2022 Vulcanize, Inc. // -import debug from 'debug'; -import assert from 'assert'; +import { VerifyCheckpointCmd } from '@cerc-io/cli'; -import { getConfig, initClients, JobQueue, Config, verifyCheckpointData } from '@cerc-io/util'; -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; - -import { Database, ENTITY_TO_LATEST_ENTITY_MAP } from '../../database'; +import { Database } from '../../database'; import { Indexer } from '../../indexer'; -const log = debug('vulcanize:checkpoint-verify'); - export const command = 'verify'; export const desc = 'Verify checkpoint'; export const builder = { cid: { - alias: 'c', type: 'string', + alias: 'c', demandOption: true, describe: 'Checkpoint CID to be verified' } }; export const handler = async (argv: any): Promise => { - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); + const createCheckpointCmd = new VerifyCheckpointCmd(); + await createCheckpointCmd.init(argv, Database, Indexer); - const db = new Database(config.database); - await db.init(); - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, { ethClient }, ethProvider, jobQueue, graphWatcher); - await indexer.init(); - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - const state = await indexer.getStateByCID(argv.cid); - assert(state, 'State for the provided CID doesn\'t exist.'); - const data = indexer.getStateData(state); - - log(`Verifying checkpoint data for contract ${state.contractAddress}`); - await verifyCheckpointData(graphDb, state.block, data); - log('Checkpoint data verified'); - - await db.close(); + await createCheckpointCmd.exec(); }; diff --git a/packages/graph-test-watcher/src/cli/export-state.ts b/packages/graph-test-watcher/src/cli/export-state.ts index 33f7ff27..2411b960 100644 --- a/packages/graph-test-watcher/src/cli/export-state.ts +++ b/packages/graph-test-watcher/src/cli/export-state.ts @@ -2,146 +2,21 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import fs from 'fs'; -import path from 'path'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util'; -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -import * as codec from '@ipld/dag-cbor'; +import { ExportStateCmd } from '@cerc-io/cli'; -import { Database, ENTITY_TO_LATEST_ENTITY_MAP } from '../database'; +import { Database } from '../database'; import { Indexer } from '../indexer'; const log = debug('vulcanize:export-state'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - exportFile: { - alias: 'o', - type: 'string', - describe: 'Export file path' - }, - blockNumber: { - type: 'number', - describe: 'Block number to create snapshot at' - } - }).argv; + const exportStateCmd = new ExportStateCmd(); + await exportStateCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - - const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_TO_LATEST_ENTITY_MAP); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, { ethClient }, ethProvider, jobQueue, graphWatcher); - await indexer.init(); - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - const exportData: any = { - snapshotBlock: {}, - contracts: [], - stateCheckpoints: [] - }; - - const contracts = await db.getContracts(); - let block = await indexer.getLatestStateIndexedBlock(); - assert(block); - - if (argv.blockNumber) { - if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); - } - - const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); - - if (!blocksAtSnapshotHeight.length) { - throw new Error(`No blocks at snapshot height ${argv.blockNumber}`); - } - - block = blocksAtSnapshotHeight[0]; - } - - log(`Creating export snapshot at block height ${block.blockNumber}`); - - // Export snapshot block. - exportData.snapshotBlock = { - blockNumber: block.blockNumber, - blockHash: block.blockHash - }; - - // Export contracts and checkpoints. - for (const contract of contracts) { - if (contract.startingBlock > block.blockNumber) { - continue; - } - - exportData.contracts.push({ - address: contract.address, - kind: contract.kind, - checkpoint: contract.checkpoint, - startingBlock: block.blockNumber - }); - - // Create and export checkpoint if checkpointing is on for the contract. - if (contract.checkpoint) { - await indexer.createCheckpoint(contract.address, block.blockHash); - - const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(state); - - const data = indexer.getStateData(state); - - exportData.stateCheckpoints.push({ - contractAddress: state.contractAddress, - cid: state.cid, - kind: state.kind, - data - }); - } - } - - if (argv.exportFile) { - const encodedExportData = codec.encode(exportData); - - const filePath = path.resolve(argv.exportFile); - const fileDir = path.dirname(filePath); - - if (!fs.existsSync(fileDir)) fs.mkdirSync(fileDir, { recursive: true }); - - fs.writeFileSync(filePath, encodedExportData); - } else { - log(exportData); - } + await exportStateCmd.exec(); }; main().catch(err => { diff --git a/packages/mobymask-watcher/src/cli/export-state.ts b/packages/mobymask-watcher/src/cli/export-state.ts index 39f1803e..2411b960 100644 --- a/packages/mobymask-watcher/src/cli/export-state.ts +++ b/packages/mobymask-watcher/src/cli/export-state.ts @@ -2,15 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; -import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; -import fs from 'fs'; -import path from 'path'; -import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util'; -import * as codec from '@ipld/dag-cbor'; +import { ExportStateCmd } from '@cerc-io/cli'; import { Database } from '../database'; import { Indexer } from '../indexer'; @@ -18,121 +13,10 @@ import { Indexer } from '../indexer'; const log = debug('vulcanize:export-state'); const main = async (): Promise => { - const argv = await yargs.parserConfiguration({ - 'parse-numbers': false - }).options({ - configFile: { - alias: 'f', - type: 'string', - require: true, - demandOption: true, - describe: 'Configuration file path (toml)', - default: DEFAULT_CONFIG_PATH - }, - exportFile: { - alias: 'o', - type: 'string', - describe: 'Export file path' - }, - blockNumber: { - type: 'number', - describe: 'Block number to create snapshot at' - } - }).argv; + const exportStateCmd = new ExportStateCmd(); + await exportStateCmd.init(Database, Indexer); - const config: Config = await getConfig(argv.configFile); - const { ethClient, ethProvider } = await initClients(config); - - const db = new Database(config.database); - await db.init(); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, { ethClient }, ethProvider, jobQueue); - await indexer.init(); - - const exportData: any = { - snapshotBlock: {}, - contracts: [], - stateCheckpoints: [] - }; - - const contracts = await db.getContracts(); - let block = await indexer.getLatestStateIndexedBlock(); - assert(block); - - if (argv.blockNumber) { - if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); - } - - const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); - - if (!blocksAtSnapshotHeight.length) { - throw new Error(`No blocks at snapshot height ${argv.blockNumber}`); - } - - block = blocksAtSnapshotHeight[0]; - } - - log(`Creating export snapshot at block height ${block.blockNumber}`); - - // Export snapshot block. - exportData.snapshotBlock = { - blockNumber: block.blockNumber, - blockHash: block.blockHash - }; - - // Export contracts and checkpoints. - for (const contract of contracts) { - if (contract.startingBlock > block.blockNumber) { - continue; - } - - exportData.contracts.push({ - address: contract.address, - kind: contract.kind, - checkpoint: contract.checkpoint, - startingBlock: block.blockNumber - }); - - // Create and export checkpoint if checkpointing is on for the contract. - if (contract.checkpoint) { - await indexer.createCheckpoint(contract.address, block.blockHash); - - const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(state); - - const data = indexer.getStateData(state); - - exportData.stateCheckpoints.push({ - contractAddress: state.contractAddress, - cid: state.cid, - kind: state.kind, - data - }); - } - } - - if (argv.exportFile) { - const encodedExportData = codec.encode(exportData); - - const filePath = path.resolve(argv.exportFile); - const fileDir = path.dirname(filePath); - - if (!fs.existsSync(fileDir)) fs.mkdirSync(fileDir, { recursive: true }); - - fs.writeFileSync(filePath, encodedExportData); - } else { - log(exportData); - } + await exportStateCmd.exec(); }; main().catch(err => { diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 59f8e337..374ce025 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -92,6 +92,7 @@ export interface IndexerInterface { getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise getBlocksAtHeight (height: number, isPruned: boolean): Promise getLatestCanonicalBlock (): Promise + getLatestStateIndexedBlock (): Promise getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise> getAncestorAtDepth (blockHash: string, depth: number): Promise saveBlockAndFetchEvents (block: DeepPartial): Promise<[BlockProgressInterface, DeepPartial[]]> @@ -120,11 +121,13 @@ export interface IndexerInterface { processCanonicalBlock (blockHash: string, blockNumber: number): Promise processCheckpoint (blockHash: string): Promise processCLICheckpoint (contractAddress: string, blockHash?: string): Promise + createCheckpoint (contractAddress: string, blockHash: string): Promise getStorageValue (storageLayout: StorageLayout, blockHash: string, contractAddress: string, variable: string, ...mappingKeys: MappingKey[]): Promise updateSubgraphState?: (contractAddress: string, data: any) => void updateStateStatusMap (address: string, stateStatus: StateStatus): void getStateData (state: StateInterface): any getStateByCID (cid: string): Promise + getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise saveOrUpdateState (state: StateInterface): Promise removeStates (blockNumber: number, kind: StateKind): Promise resetWatcherToBlock (blockNumber: number): Promise @@ -164,8 +167,8 @@ export interface DatabaseInterface { saveEventEntity (queryRunner: QueryRunner, entity: EventInterface): Promise; removeEntities (queryRunner: QueryRunner, entity: new () => Entity, findConditions?: FindManyOptions | FindConditions): Promise; deleteEntitiesByConditions (queryRunner: QueryRunner, entity: EntityTarget, findConditions: FindConditions): Promise - getContracts?: () => Promise - saveContract?: (queryRunner: QueryRunner, contractAddress: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise + getContracts: () => Promise + saveContract: (queryRunner: QueryRunner, contractAddress: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise getStates (where: FindConditions): Promise getDiffStatesInRange (contractAddress: string, startBlock: number, endBlock: number): Promise