From 5af90bd388ff47a9c882b65c310a134acbed18a5 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Wed, 19 Oct 2022 04:54:14 -0500 Subject: [PATCH] Refactor state creation code (#204) * Remove support for pushing state to IPFS * Move job handlers for state creation to util * Rename state creation related methods and objects * Update mock indexer used in graph-node testing * Fetch and merge diffs in batches while creating a state checkpoint * Fix timing logs while for state checkpoint creation * Refactor method to get state query result to util * Accept contracts for state verification in compare CLI config * Make method to update state status map synchronous --- packages/codegen/README.md | 14 +- .../entities/{IPLDBlock.yaml => State.yaml} | 2 +- .../{IpldStatus.yaml => StateSyncStatus.yaml} | 8 +- packages/codegen/src/entity.ts | 12 +- packages/codegen/src/generate-code.ts | 23 +- packages/codegen/src/reset.ts | 26 +- packages/codegen/src/schema.ts | 16 +- .../checkpoint-verify-template.handlebars | 10 +- .../src/templates/config-template.handlebars | 3 - .../templates/database-template.handlebars | 76 ++-- .../export-state-template.handlebars | 24 +- .../templates/fill-state-template.handlebars | 6 +- .../src/templates/hooks-template.handlebars | 10 +- .../import-state-template.handlebars | 28 +- .../src/templates/indexer-template.handlebars | 117 ++---- .../templates/inspect-cid-template.handlebars | 8 +- .../templates/job-runner-template.handlebars | 165 +-------- .../src/templates/readme-template.handlebars | 26 +- .../reset-ipld-state-template.handlebars | 66 ---- .../templates/reset-state-template.handlebars | 102 +----- .../reset-watcher-template.handlebars | 124 +++++++ .../templates/resolvers-template.handlebars | 10 +- packages/codegen/src/visitor.ts | 7 +- packages/eden-watcher/README.md | 24 +- packages/eden-watcher/environments/local.toml | 3 - .../src/cli/checkpoint-cmds/verify.ts | 10 +- packages/eden-watcher/src/cli/export-state.ts | 24 +- packages/eden-watcher/src/cli/import-state.ts | 28 +- packages/eden-watcher/src/cli/inspect-cid.ts | 8 +- .../src/cli/reset-cmds/ipld-state.ts | 66 ---- .../eden-watcher/src/cli/reset-cmds/state.ts | 102 +----- .../src/cli/reset-cmds/watcher.ts | 124 +++++++ packages/eden-watcher/src/database.ts | 76 ++-- .../src/entity/{IPLDBlock.ts => State.ts} | 2 +- .../{IpldStatus.ts => StateSyncStatus.ts} | 7 +- packages/eden-watcher/src/fill-state.ts | 21 +- packages/eden-watcher/src/hooks.ts | 72 +--- packages/eden-watcher/src/indexer.ts | 119 ++---- packages/eden-watcher/src/job-runner.ts | 166 +-------- packages/eden-watcher/src/resolvers.ts | 10 +- packages/eden-watcher/src/schema.gql | 6 +- .../cli/reset-cmds/{state.ts => watcher.ts} | 8 +- packages/erc20-watcher/src/database.ts | 58 +-- .../erc20-watcher/src/entity/IpldStatus.ts | 20 - .../src/entity/{IPLDBlock.ts => State.ts} | 2 +- .../src/entity/StateSyncStatus.ts} | 7 +- packages/erc20-watcher/src/indexer.ts | 48 ++- packages/erc721-watcher/README.md | 22 +- packages/erc721-watcher/demo.md | 35 +- .../erc721-watcher/environments/local.toml | 3 - .../erc721-watcher/src/cli/export-state.ts | 24 +- .../erc721-watcher/src/cli/import-state.ts | 26 +- .../erc721-watcher/src/cli/inspect-cid.ts | 8 +- .../cli/reset-cmds/{state.ts => watcher.ts} | 24 +- packages/erc721-watcher/src/database.ts | 70 ++-- .../src/entity/State.ts} | 2 +- .../src/entity/StateSyncStatus.ts} | 9 +- packages/erc721-watcher/src/hooks.ts | 10 +- packages/erc721-watcher/src/indexer.ts | 109 ++---- packages/erc721-watcher/src/job-runner.ts | 166 +-------- packages/erc721-watcher/src/resolvers.ts | 10 +- packages/erc721-watcher/src/schema.gql | 6 +- packages/graph-node/README.md | 12 +- .../src/cli/compare/compare-blocks.ts | 50 ++- packages/graph-node/src/cli/compare/utils.ts | 69 ++-- packages/graph-node/src/database.ts | 6 +- packages/graph-node/src/watcher.ts | 14 +- packages/graph-node/test/utils/indexer.ts | 59 ++- packages/graph-test-watcher/README.md | 18 +- .../environments/local.toml | 3 - .../src/cli/checkpoint-cmds/verify.ts | 10 +- .../src/cli/export-state.ts | 24 +- .../src/cli/import-state.ts | 28 +- .../graph-test-watcher/src/cli/inspect-cid.ts | 8 +- .../cli/reset-cmds/{state.ts => watcher.ts} | 25 +- packages/graph-test-watcher/src/database.ts | 70 ++-- .../src/entity/State.ts} | 9 +- .../src/entity/StateSyncStatus.ts} | 9 +- packages/graph-test-watcher/src/hooks.ts | 6 +- packages/graph-test-watcher/src/indexer.ts | 113 ++---- packages/graph-test-watcher/src/job-runner.ts | 166 +-------- packages/graph-test-watcher/src/resolvers.ts | 10 +- packages/graph-test-watcher/src/schema.gql | 6 +- packages/mobymask-watcher/README.md | 18 +- .../mobymask-watcher/environments/local.toml | 3 - .../mobymask-watcher/src/cli/export-state.ts | 24 +- .../mobymask-watcher/src/cli/import-state.ts | 26 +- .../mobymask-watcher/src/cli/inspect-cid.ts | 8 +- .../cli/reset-cmds/{state.ts => watcher.ts} | 24 +- packages/mobymask-watcher/src/database.ts | 70 ++-- .../mobymask-watcher/src/entity/IPLDBlock.ts | 31 -- packages/mobymask-watcher/src/entity/State.ts | 36 ++ .../src/entity/StateSyncStatus.ts | 17 + packages/mobymask-watcher/src/hooks.ts | 10 +- packages/mobymask-watcher/src/indexer.ts | 111 ++---- packages/mobymask-watcher/src/job-runner.ts | 166 +-------- packages/mobymask-watcher/src/resolvers.ts | 10 +- packages/mobymask-watcher/src/schema.gql | 6 +- packages/util/index.ts | 3 +- packages/util/src/common.ts | 89 +++-- packages/util/src/config.ts | 1 - packages/util/src/constants.ts | 2 +- packages/util/src/database.ts | 108 +++--- packages/util/src/indexer.ts | 342 +++++++++--------- packages/util/src/job-runner.ts | 101 +++++- packages/util/src/misc.ts | 4 +- .../src/{ipld-helper.ts => state-helper.ts} | 50 ++- packages/util/src/types.ts | 47 +-- 108 files changed, 1671 insertions(+), 2769 deletions(-) rename packages/codegen/src/data/entities/{IPLDBlock.yaml => State.yaml} (98%) rename packages/codegen/src/data/entities/{IpldStatus.yaml => StateSyncStatus.yaml} (70%) delete mode 100644 packages/codegen/src/templates/reset-ipld-state-template.handlebars create mode 100644 packages/codegen/src/templates/reset-watcher-template.handlebars delete mode 100644 packages/eden-watcher/src/cli/reset-cmds/ipld-state.ts create mode 100644 packages/eden-watcher/src/cli/reset-cmds/watcher.ts rename packages/eden-watcher/src/entity/{IPLDBlock.ts => State.ts} (96%) rename packages/eden-watcher/src/entity/{IpldStatus.ts => StateSyncStatus.ts} (66%) rename packages/erc20-watcher/src/cli/reset-cmds/{state.ts => watcher.ts} (93%) delete mode 100644 packages/erc20-watcher/src/entity/IpldStatus.ts rename packages/erc20-watcher/src/entity/{IPLDBlock.ts => State.ts} (96%) rename packages/{graph-test-watcher/src/entity/IpldStatus.ts => erc20-watcher/src/entity/StateSyncStatus.ts} (66%) rename packages/erc721-watcher/src/cli/reset-cmds/{state.ts => watcher.ts} (82%) rename packages/{graph-test-watcher/src/entity/IPLDBlock.ts => erc721-watcher/src/entity/State.ts} (96%) rename packages/{mobymask-watcher/src/entity/IpldStatus.ts => erc721-watcher/src/entity/StateSyncStatus.ts} (62%) rename packages/graph-test-watcher/src/cli/reset-cmds/{state.ts => watcher.ts} (81%) rename packages/{erc721-watcher/src/entity/IPLDBlock.ts => graph-test-watcher/src/entity/State.ts} (89%) rename packages/{erc721-watcher/src/entity/IpldStatus.ts => graph-test-watcher/src/entity/StateSyncStatus.ts} (62%) rename packages/mobymask-watcher/src/cli/reset-cmds/{state.ts => watcher.ts} (79%) delete mode 100644 packages/mobymask-watcher/src/entity/IPLDBlock.ts create mode 100644 packages/mobymask-watcher/src/entity/State.ts create mode 100644 packages/mobymask-watcher/src/entity/StateSyncStatus.ts rename packages/util/src/{ipld-helper.ts => state-helper.ts} (53%) diff --git a/packages/codegen/README.md b/packages/codegen/README.md index 1464098b..cdaafece 100644 --- a/packages/codegen/README.md +++ b/packages/codegen/README.md @@ -82,18 +82,10 @@ yarn ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - ``` - * In the config file (`environments/local.toml`): * Update the state checkpoint settings. - * Update the IPFS API address in `environments/local.toml`. - * Create the databases configured in `environments/local.toml`. ### Customize @@ -106,11 +98,11 @@ * Generating state: - * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in `src/hooks.ts` to save an initial state `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in `src/hooks.ts` to save an initial `State` using the `Indexer` object. - * Edit the custom hook function `createStateDiff` (triggered on a block) in `src/hooks.ts` to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + * Edit the custom hook function `createStateDiff` (triggered on a block) in `src/hooks.ts` to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated. - * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in `src/hooks.ts` to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in `src/hooks.ts` to save the state in a `checkpoint` `State` using the `Indexer` object. ### Run diff --git a/packages/codegen/src/data/entities/IPLDBlock.yaml b/packages/codegen/src/data/entities/State.yaml similarity index 98% rename from packages/codegen/src/data/entities/IPLDBlock.yaml rename to packages/codegen/src/data/entities/State.yaml index 6cd15c3c..dcf24a93 100644 --- a/packages/codegen/src/data/entities/IPLDBlock.yaml +++ b/packages/codegen/src/data/entities/State.yaml @@ -1,4 +1,4 @@ -className: IPLDBlock +className: State indexOn: - columns: - cid diff --git a/packages/codegen/src/data/entities/IpldStatus.yaml b/packages/codegen/src/data/entities/StateSyncStatus.yaml similarity index 70% rename from packages/codegen/src/data/entities/IpldStatus.yaml rename to packages/codegen/src/data/entities/StateSyncStatus.yaml index f92ba77d..0a62d5c1 100644 --- a/packages/codegen/src/data/entities/IpldStatus.yaml +++ b/packages/codegen/src/data/entities/StateSyncStatus.yaml @@ -1,10 +1,10 @@ -className: IpldStatus +className: StateSyncStatus indexOn: [] columns: - name: id tsType: number columnType: PrimaryGeneratedColumn - - name: latestHooksBlockNumber + - name: latestIndexedBlockNumber pgType: integer tsType: number columnType: Column @@ -12,10 +12,6 @@ columns: pgType: integer tsType: number columnType: Column - - name: latestIPFSBlockNumber - pgType: integer - tsType: number - columnType: Column imports: - toImport: - Entity diff --git a/packages/codegen/src/entity.ts b/packages/codegen/src/entity.ts index f717eada..a94dc954 100644 --- a/packages/codegen/src/entity.ts +++ b/packages/codegen/src/entity.ts @@ -178,8 +178,8 @@ export class Entity { this._addSyncStatusEntity(); this._addContractEntity(); this._addBlockProgressEntity(); - this._addIPLDBlockEntity(); - this._addIpldStatusEntity(); + this._addStateEntity(); + this._addStateSyncStatusEntity(); const template = Handlebars.compile(this._templateString); this._entities.forEach(entityObj => { @@ -278,13 +278,13 @@ export class Entity { this._entities.push(entity); } - _addIPLDBlockEntity (): void { - const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'IPLDBlock.yaml'), 'utf8')); + _addStateEntity (): void { + const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'State.yaml'), 'utf8')); this._entities.push(entity); } - _addIpldStatusEntity (): void { - const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'IpldStatus.yaml'), 'utf8')); + _addStateSyncStatusEntity (): void { + const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'StateSyncStatus.yaml'), 'utf8')); this._entities.push(entity); } diff --git a/packages/codegen/src/generate-code.ts b/packages/codegen/src/generate-code.ts index 7249875e..ddda273f 100644 --- a/packages/codegen/src/generate-code.ts +++ b/packages/codegen/src/generate-code.ts @@ -297,25 +297,12 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) { : process.stdout; visitor.exportClient(outStream, schemaContent, path.join(outputDir, 'src/gql')); - let resetOutStream, resetJQOutStream, resetStateOutStream, resetIPLDStateOutStream; + const resetOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset.ts')); + const resetJQOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/job-queue.ts')); + const resetWatcherOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/watcher.ts')); + const resetStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/state.ts')); - if (outputDir) { - resetOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset.ts')); - resetJQOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/job-queue.ts')); - resetStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/state.ts')); - if (config.subgraphPath) { - resetIPLDStateOutStream = fs.createWriteStream(path.join(outputDir, 'src/cli/reset-cmds/ipld-state.ts')); - } - } else { - resetOutStream = process.stdout; - resetJQOutStream = process.stdout; - resetStateOutStream = process.stdout; - if (config.subgraphPath) { - resetIPLDStateOutStream = process.stdout; - } - } - - visitor.exportReset(resetOutStream, resetJQOutStream, resetStateOutStream, resetIPLDStateOutStream, config.subgraphPath); + visitor.exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream, config.subgraphPath); outStream = outputDir ? fs.createWriteStream(path.join(outputDir, 'src/cli/export-state.ts')) diff --git a/packages/codegen/src/reset.ts b/packages/codegen/src/reset.ts index 39f17a99..2ed65455 100644 --- a/packages/codegen/src/reset.ts +++ b/packages/codegen/src/reset.ts @@ -9,22 +9,22 @@ import { Writable } from 'stream'; const RESET_TEMPLATE_FILE = './templates/reset-template.handlebars'; const RESET_JQ_TEMPLATE_FILE = './templates/reset-job-queue-template.handlebars'; +const RESET_WATCHER_TEMPLATE_FILE = './templates/reset-watcher-template.handlebars'; const RESET_STATE_TEMPLATE_FILE = './templates/reset-state-template.handlebars'; -const RESET_IPLD_STATE_TEMPLATE_FILE = './templates/reset-ipld-state-template.handlebars'; export class Reset { _queries: Array; _resetTemplateString: string; _resetJQTemplateString: string; + _resetWatcherTemplateString: string; _resetStateTemplateString: string; - _resetIPLDStateTemplateString: string; constructor () { this._queries = []; this._resetTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_TEMPLATE_FILE)).toString(); this._resetJQTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_JQ_TEMPLATE_FILE)).toString(); + this._resetWatcherTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_WATCHER_TEMPLATE_FILE)).toString(); this._resetStateTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_STATE_TEMPLATE_FILE)).toString(); - this._resetIPLDStateTemplateString = fs.readFileSync(path.resolve(__dirname, RESET_IPLD_STATE_TEMPLATE_FILE)).toString(); } /** @@ -71,13 +71,13 @@ export class Reset { */ /** - * Writes the reset.ts, job-queue.ts, state.ts files generated from templates to respective streams. + * Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams. * @param resetOutStream A writable output stream to write the reset file to. * @param resetJQOutStream A writable output stream to write the reset job-queue file to. + * @param resetWatcherOutStream A writable output stream to write the reset watcher file to. * @param resetStateOutStream A writable output stream to write the reset state file to. - * @param resetIPLDStateOutStream A writable output stream to write the reset IPLD state file to. */ - exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetStateOutStream: Writable, resetIPLDStateOutStream: Writable | undefined, subgraphPath: string): void { + exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable, subgraphPath: string): void { const resetTemplate = Handlebars.compile(this._resetTemplateString); const resetString = resetTemplate({}); resetOutStream.write(resetString); @@ -86,18 +86,16 @@ export class Reset { const resetJQString = resetJQTemplate({}); resetJQOutStream.write(resetJQString); - const resetStateTemplate = Handlebars.compile(this._resetStateTemplateString); + const resetWatcherTemplate = Handlebars.compile(this._resetWatcherTemplateString); const obj = { queries: this._queries, subgraphPath }; - const resetState = resetStateTemplate(obj); - resetStateOutStream.write(resetState); + const resetWatcher = resetWatcherTemplate(obj); + resetWatcherOutStream.write(resetWatcher); - if (resetIPLDStateOutStream) { - const resetIPLDStateTemplate = Handlebars.compile(this._resetIPLDStateTemplateString); - const resetIPLDStateString = resetIPLDStateTemplate({}); - resetIPLDStateOutStream.write(resetIPLDStateString); - } + const resetStateTemplate = Handlebars.compile(this._resetStateTemplateString); + const resetState = resetStateTemplate({}); + resetStateOutStream.write(resetState); } } diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index fba31c08..103477a0 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -98,9 +98,9 @@ export class Schema { // Add type and query for SyncStatus. this._addSyncStatus(); - // Add IPLDBlock type and queries. - this._addIPLDType(); - this._addIPLDQuery(); + // Add State type and queries. + this._addStateType(); + this._addStateQuery(); // Build the schema. return this._composer.buildSchema(); @@ -354,9 +354,9 @@ export class Schema { }); } - _addIPLDType (): void { + _addStateType (): void { const typeComposer = this._composer.createObjectTC({ - name: 'ResultIPLDBlock', + name: 'ResultState', fields: { block: () => this._composer.getOTC('_Block_').NonNull, contractAddress: 'String!', @@ -368,10 +368,10 @@ export class Schema { this._composer.addSchemaMustHaveType(typeComposer); } - _addIPLDQuery (): void { + _addStateQuery (): void { this._composer.Query.addFields({ getStateByCID: { - type: this._composer.getOTC('ResultIPLDBlock'), + type: this._composer.getOTC('ResultState'), args: { cid: 'String!' } @@ -380,7 +380,7 @@ export class Schema { this._composer.Query.addFields({ getState: { - type: this._composer.getOTC('ResultIPLDBlock'), + type: this._composer.getOTC('ResultState'), args: { blockHash: 'String!', contractAddress: 'String!', diff --git a/packages/codegen/src/templates/checkpoint-verify-template.handlebars b/packages/codegen/src/templates/checkpoint-verify-template.handlebars index 9fd9b815..27960003 100644 --- a/packages/codegen/src/templates/checkpoint-verify-template.handlebars +++ b/packages/codegen/src/templates/checkpoint-verify-template.handlebars @@ -54,12 +54,12 @@ export const handler = async (argv: any): Promise => { graphWatcher.setIndexer(indexer); await graphWatcher.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); - const data = indexer.getIPLDData(ipldBlock); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); + const data = indexer.getStateData(state); - log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`); - await verifyCheckpointData(graphDb, ipldBlock.block, data); + log(`Verifying checkpoint data for contract ${state.contractAddress}`); + await verifyCheckpointData(graphDb, state.block, data); log('Checkpoint data verified'); await db.close(); diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index fd5e610d..621d698a 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -9,9 +9,6 @@ # Checkpoint interval in number of blocks. checkpointInterval = 2000 - # IPFS API address (can be taken from the output on running the IPFS daemon). - ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" - {{#if (subgraphPath)}} subgraphPath = "{{subgraphPath}}" diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 82169fe1..3d2ed7f7 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; {{#each queries as | query |}} import { {{query.entityName}} } from './entity/{{query.entityName}}'; {{/each}} @@ -79,75 +79,69 @@ export class Database implements DatabaseInterface { } {{/each}} - getNewIPLDBlock (): IPLDBlock { - return new IPLDBlock(); + getNewState (): State { + return new State(); } - async getIPLDBlocks (where: FindConditions): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getStates (where: FindConditions): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getIPLDBlocks(repo, where); + return this._baseDatabase.getStates(repo, where); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber); + return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind); + return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind); } - // Fetch all diff IPLDBlocks after the specified block number. - async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + // Fetch all diff States after the specified block number. + async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock); + return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock); } - async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock); + return this._baseDatabase.saveOrUpdateState(repo, state); } - async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind); + await this._baseDatabase.removeStates(repo, blockNumber, kind); } - async removeIPLDBlocksAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocksAfterBlock(repo, blockNumber); + await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber); } - async getIPLDStatus (): Promise { - const repo = this._conn.getRepository(IpldStatus); + async getStateSyncStatus (): Promise { + const repo = this._conn.getRepository(StateSyncStatus); - return this._baseDatabase.getIPLDStatus(repo); + return this._baseDatabase.getStateSyncStatus(repo); } - async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force); } - async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force); - } - - async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); - - return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force); } async getContracts (): Promise { diff --git a/packages/codegen/src/templates/export-state-template.handlebars b/packages/codegen/src/templates/export-state-template.handlebars index bd5cae8b..1c377d45 100644 --- a/packages/codegen/src/templates/export-state-template.handlebars +++ b/packages/codegen/src/templates/export-state-template.handlebars @@ -85,18 +85,18 @@ const main = async (): Promise => { const exportData: any = { snapshotBlock: {}, contracts: [], - ipldCheckpoints: [] + stateCheckpoints: [] }; const contracts = await db.getContracts(); // Get latest block with hooks processed. - let block = await indexer.getLatestHooksProcessedBlock(); + let block = await indexer.getLatestStateIndexedBlock(); assert(block); if (argv.blockNumber) { if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`); + throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); } const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); @@ -133,19 +133,15 @@ const main = async (): Promise => { if (contract.checkpoint) { await indexer.createCheckpoint(contract.address, block.blockHash); - const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(ipldBlock); + const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); + assert(state); - const data = indexer.getIPLDData(ipldBlock); + const data = indexer.getStateData(state); - if (indexer.isIPFSConfigured()) { - await indexer.pushToIPFS(data); - } - - exportData.ipldCheckpoints.push({ - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, + exportData.stateCheckpoints.push({ + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, data }); } diff --git a/packages/codegen/src/templates/fill-state-template.handlebars b/packages/codegen/src/templates/fill-state-template.handlebars index 4cda86fe..f3dbf949 100644 --- a/packages/codegen/src/templates/fill-state-template.handlebars +++ b/packages/codegen/src/templates/fill-state-template.handlebars @@ -31,7 +31,7 @@ export const fillState = async ( log(`Filling state for subgraph entities in range: [${startBlock}, ${endBlock}]`); // Check that there are no existing diffs in this range - const existingStates = await indexer.getIPLDBlocks({ block: { blockNumber: Between(startBlock, endBlock) } }); + const existingStates = await indexer.getStates({ block: { blockNumber: Between(startBlock, endBlock) } }); if (existingStates.length > 0) { log('found existing state(s) in the given range'); process.exit(1); @@ -93,12 +93,12 @@ export const fillState = async ( // Persist subgraph state to the DB await indexer.dumpSubgraphState(blockHash, true); + await indexer.updateStateSyncStatusIndexedBlock(blockNumber); // Create checkpoints await indexer.processCheckpoint(blockHash); + await indexer.updateStateSyncStatusCheckpointBlock(blockNumber); } - // TODO: Push state to IPFS - log(`Filled state for subgraph entities in range: [${startBlock}, ${endBlock}]`); }; diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 58417cae..1971beac 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -20,19 +20,19 @@ export async function createInitialState (indexer: Indexer, contractAddress: str assert(blockHash); assert(contractAddress); - // Store the desired initial state in an IPLDBlock. - const ipldBlockData: any = { + // Store an empty State. + const stateData: any = { state: {} }; // Use updateStateForElementaryType to update initial state with an elementary property. - // Eg. const ipldBlockData = updateStateForElementaryType(ipldBlockData, '_totalBalance', result.value.toString()); + // Eg. const stateData = updateStateForElementaryType(stateData, '_totalBalance', result.value.toString()); // Use updateStateForMappingType to update initial state with a nested property. - // Eg. const ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString()); + // Eg. const stateData = updateStateForMappingType(stateData, '_allowances', [owner, spender], allowance.value.toString()); // Return initial state data to be saved. - return ipldBlockData; + return stateData; } /** diff --git a/packages/codegen/src/templates/import-state-template.handlebars b/packages/codegen/src/templates/import-state-template.handlebars index 157813ab..60cef1cc 100644 --- a/packages/codegen/src/templates/import-state-template.handlebars +++ b/packages/codegen/src/templates/import-state-template.handlebars @@ -20,7 +20,7 @@ import * as codec from '@ipld/dag-cbor'; import { Database } from '../database'; import { Indexer } from '../indexer'; import { EventWatcher } from '../events'; -import { IPLDBlock } from '../entity/IPLDBlock'; +import { State } from '../entity/State'; const log = debug('vulcanize:import-state'); @@ -106,18 +106,18 @@ export const main = async (): Promise => { const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash); assert(block); - // Fill the IPLDBlocks. - for (const checkpoint of importData.ipldCheckpoints) { - let ipldBlock = new IPLDBlock(); + // Fill the States. + for (const checkpoint of importData.stateCheckpoints) { + let state = new State(); - ipldBlock = Object.assign(ipldBlock, checkpoint); - ipldBlock.block = block; + state = Object.assign(state, checkpoint); + state.block = block; - ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data)); + state.data = Buffer.from(codec.encode(state.data)); - ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock); + state = await indexer.saveOrUpdateState(state); {{#if (subgraphPath)}} - await graphWatcher.updateEntitiesFromIPLDState(ipldBlock); + await graphWatcher.updateEntitiesFromState(state); {{/if}} } @@ -126,12 +126,12 @@ export const main = async (): Promise => { await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber); await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber); - await indexer.updateIPLDStatusHooksBlock(block.blockNumber); - await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber); + await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber); + await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber); - // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block. - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init); - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged); + // The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block. + await indexer.removeStates(block.blockNumber, StateKind.Init); + await indexer.removeStates(block.blockNumber, StateKind.DiffStaged); log(`Import completed for snapshot block at height ${block.blockNumber}`); }; diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 877bc96e..a9c83f5c 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -8,11 +8,12 @@ import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm'; import JSONbig from 'json-bigint'; import { ethers } from 'ethers'; import _ from 'lodash'; +{{#if (subgraphPath)}} import { SelectionNode } from 'graphql'; +{{/if}} import { JsonFragment } from '@ethersproject/abi'; import { BaseProvider } from '@ethersproject/providers'; -import * as codec from '@ipld/dag-cbor'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; import { @@ -29,10 +30,8 @@ import { {{#if (subgraphPath)}} BlockHeight, {{/if}} - IPFSClient, StateKind, - IpldStatus as IpldStatusInterface, - ResultIPLDBlock + StateStatus } from '@cerc-io/util'; {{#if (subgraphPath)}} import { GraphWatcher } from '@cerc-io/graph-node'; @@ -46,9 +45,9 @@ import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; {{#each subgraphEntities as | subgraphEntity |}} import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.className}}'; @@ -103,8 +102,6 @@ export class Indexer implements IndexerInterface { _storageLayoutMap: Map _contractMap: Map - _ipfsClient: IPFSClient - {{#if (subgraphPath)}} _entityTypesMap: Map _relationsMap: Map @@ -120,8 +117,7 @@ export class Indexer implements IndexerInterface { this._ethClient = ethClient; this._ethProvider = ethProvider; this._serverConfig = serverConfig; - this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); - this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient); + this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); {{#if (subgraphPath)}} this._graphWatcher = graphWatcher; {{/if}} @@ -170,7 +166,7 @@ export class Indexer implements IndexerInterface { async init (): Promise { await this._baseIndexer.fetchContracts(); - await this._baseIndexer.fetchIPLDStatus(); + await this._baseIndexer.fetchStateStatus(); } getResultEvent (event: Event): ResultEvent { @@ -208,26 +204,6 @@ export class Indexer implements IndexerInterface { }; } - getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock { - const block = ipldBlock.block; - - const data = codec.decode(Buffer.from(ipldBlock.data)) as any; - - return { - block: { - cid: block.cid, - hash: block.blockHash, - number: block.blockNumber, - timestamp: block.blockTimestamp, - parentHash: block.parentHash - }, - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, - data: JSON.stringify(data) - }; - } - {{#each queries as | query |}} async {{query.name}} (blockHash: string, contractAddress: string {{~#each query.params}}, {{this.name~}}: {{this.type~}} {{/each}} @@ -324,10 +300,6 @@ export class Indexer implements IndexerInterface { ); } - async pushToIPFS (data: any): Promise { - await this._baseIndexer.pushToIPFS(data); - } - async processInitialState (contractAddress: string, blockHash: string): Promise { // Call initial state hook. return createInitialState(this, contractAddress, blockHash); @@ -362,32 +334,28 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + return this._db.getPrevState(blockHash, contractAddress, kind); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + return this._db.getLatestState(contractAddress, kind, blockNumber); } - async getIPLDBlocksByHash (blockHash: string): Promise { - return this._baseIndexer.getIPLDBlocksByHash(blockHash); + async getStatesByHash (blockHash: string): Promise { + return this._baseIndexer.getStatesByHash(blockHash); } - async getIPLDBlockByCid (cid: string): Promise { - return this._baseIndexer.getIPLDBlockByCid(cid); + async getStateByCID (cid: string): Promise { + return this._baseIndexer.getStateByCID(cid); } - async getIPLDBlocks (where: FindConditions): Promise { - return this._db.getIPLDBlocks(where); + async getStates (where: FindConditions): Promise { + return this._db.getStates(where); } - getIPLDData (ipldBlock: IPLDBlock): any { - return this._baseIndexer.getIPLDData(ipldBlock); - } - - isIPFSConfigured (): boolean { - return this._baseIndexer.isIPFSConfigured(); + getStateData (state: State): any { + return this._baseIndexer.getStateData(state); } // Method used to create auto diffs (diff_staged). @@ -425,12 +393,12 @@ export class Indexer implements IndexerInterface { await this._baseIndexer.createInit(this, blockHash, blockNumber); } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock); + async saveOrUpdateState (state: State): Promise { + return this._baseIndexer.saveOrUpdateState(state); } - async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise { - await this._baseIndexer.removeIPLDBlocks(blockNumber, kind); + async removeStates (blockNumber: number, kind: StateKind): Promise { + await this._baseIndexer.removeStates(blockNumber, kind); } {{#if (subgraphPath)}} @@ -466,8 +434,8 @@ export class Indexer implements IndexerInterface { async processBlock (blockProgress: BlockProgress): Promise { // Call a function to create initial state for contracts. await this._baseIndexer.createInit(this, blockProgress.blockHash, blockProgress.blockNumber); - {{#if (subgraphPath)}} + this._graphWatcher.updateEntityCacheFrothyBlocks(blockProgress); {{/if}} } @@ -499,16 +467,16 @@ export class Indexer implements IndexerInterface { }; } - async getIPLDStatus (): Promise { - return this._db.getIPLDStatus(); + async getStateSyncStatus (): Promise { + return this._db.getStateSyncStatus(); } - async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -520,29 +488,12 @@ export class Indexer implements IndexerInterface { return res; } - async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force); - await dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - - return res; - } - - async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise { - const dbTx = await this._db.createTransactionRunner(); - let res; - - try { - res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -564,16 +515,16 @@ export class Indexer implements IndexerInterface { return latestCanonicalBlock; } - async getLatestHooksProcessedBlock (): Promise { - return this._baseIndexer.getLatestHooksProcessedBlock(); + async getLatestStateIndexedBlock (): Promise { + return this._baseIndexer.getLatestStateIndexedBlock(); } async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { - await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + this._baseIndexer.updateStateStatusMap(address, stateStatus); } cacheContract (contract: Contract): void { diff --git a/packages/codegen/src/templates/inspect-cid-template.handlebars b/packages/codegen/src/templates/inspect-cid-template.handlebars index 129f0b47..63571bd8 100644 --- a/packages/codegen/src/templates/inspect-cid-template.handlebars +++ b/packages/codegen/src/templates/inspect-cid-template.handlebars @@ -71,12 +71,12 @@ const main = async (): Promise => { await graphWatcher.init(); {{/if}} - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); - const ipldData = await indexer.getIPLDData(ipldBlock); + const stateData = await indexer.getStateData(state); - log(util.inspect(ipldData, false, null)); + log(util.inspect(stateData, false, null)); }; main().catch(err => { diff --git a/packages/codegen/src/templates/job-runner-template.handlebars b/packages/codegen/src/templates/job-runner-template.handlebars index 99f38982..5c25a80b 100644 --- a/packages/codegen/src/templates/job-runner-template.handlebars +++ b/packages/codegen/src/templates/job-runner-template.handlebars @@ -20,7 +20,6 @@ import { QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, QUEUE_HOOKS, - QUEUE_IPFS, JOB_KIND_PRUNE, JobQueueConfig, DEFAULT_CONFIG_PATH, @@ -54,21 +53,11 @@ export class JobRunner { await this.subscribeEventProcessingQueue(); await this.subscribeBlockCheckpointQueue(); await this.subscribeHooksQueue(); - await this.subscribeIPFSQueue(); } async subscribeBlockProcessingQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseJobRunner.processBlock(job); - - const { data: { kind } } = job; - - // If it's a pruning job: Create a hooks job. - if (kind === JOB_KIND_PRUNE) { - await this.createHooksJob(); - } - - await this._jobQueue.markComplete(job); }); } @@ -80,165 +69,15 @@ export class JobRunner { async subscribeHooksQueue (): Promise { await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) { - // Create hooks job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) { - log(`Hooks for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process the hooks for the given block number. - await this._indexer.processCanonicalBlock(blockHash, blockNumber); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusHooksBlock(blockNumber); - - // Create a checkpoint job after completion of a hook job. - await this.createCheckpointJob(blockHash, blockNumber); - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processHooks(job); }); } async subscribeBlockCheckpointQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestCheckpointBlockNumber >= 0) { - if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) { - // Create a checkpoint job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) { - log(`Checkpoints for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process checkpoints for the given block. - await this._indexer.processCheckpoint(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber); - - // Create an IPFS job after completion of a checkpoint job. - if (this._indexer.isIPFSConfigured()) { - await this.createIPFSPutJob(blockHash, blockNumber); - } - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processCheckpoint(job); }); } - - async subscribeIPFSQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestIPFSBlockNumber >= 0) { - if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) { - // Create a IPFS job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) { - log(`IPFS for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Get IPLDBlocks for the given blocHash. - const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash); - - // Push all the IPLDBlocks to IPFS. - for (const ipldBlock of ipldBlocks) { - const data = this._indexer.getIPLDData(ipldBlock); - await this._indexer.pushToIPFS(data); - } - - // Update the IPLD status. - await this._indexer.updateIPLDStatusIPFSBlock(blockNumber); - - await this._jobQueue.markComplete(job); - }); - } - - async createHooksJob (blockHash?: string, blockNumber?: number): Promise { - if (!blockNumber || !blockHash) { - // Get the latest canonical block - const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock(); - - // Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block. - blockHash = latestCanonicalBlock.parentHash; - blockNumber = latestCanonicalBlock.blockNumber - 1; - } - - await this._jobQueue.pushJob( - QUEUE_HOOKS, - { - blockHash, - blockNumber - } - ); - } - - async createCheckpointJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_BLOCK_CHECKPOINT, - { - blockHash, - blockNumber - } - ); - } - - async createIPFSPutJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_IPFS, - { - blockHash, - blockNumber - } - ); - } } export const main = async (): Promise => { diff --git a/packages/codegen/src/templates/readme-template.handlebars b/packages/codegen/src/templates/readme-template.handlebars index ef73a7d7..10228260 100644 --- a/packages/codegen/src/templates/readme-template.handlebars +++ b/packages/codegen/src/templates/readme-template.handlebars @@ -8,12 +8,6 @@ yarn ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - ``` - * Create a postgres12 database for the watcher: ```bash @@ -47,7 +41,7 @@ * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint. - * Update the `server` config with state checkpoint settings and provide the IPFS API address. + * Update the `server` config with state checkpoint settings. ## Customize @@ -59,11 +53,11 @@ * Generating state: - * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object. - * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated. - * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object. ## Run @@ -134,14 +128,14 @@ GQL console: http://localhost:{{port}}/graphql ``` `cid`: CID of the checkpoint for which to verify. - + {{/if}} * To reset the watcher to a previous block number: - * Reset state: + * Reset watcher: ```bash - yarn reset state --block-number + yarn reset watcher --block-number ``` * Reset job-queue: @@ -150,6 +144,12 @@ GQL console: http://localhost:{{port}}/graphql yarn reset job-queue --block-number ``` + * Reset state: + + ```bash + yarn reset state --block-number + ``` + * `block-number`: Block number to which to reset the watcher. * To export and import the watcher state: diff --git a/packages/codegen/src/templates/reset-ipld-state-template.handlebars b/packages/codegen/src/templates/reset-ipld-state-template.handlebars deleted file mode 100644 index 47c9a02d..00000000 --- a/packages/codegen/src/templates/reset-ipld-state-template.handlebars +++ /dev/null @@ -1,66 +0,0 @@ -// -// Copyright 2022 Vulcanize, Inc. -// - -import debug from 'debug'; - -import { getConfig } from '@cerc-io/util'; - -import { Database } from '../../database'; - -const log = debug('vulcanize:reset-ipld-state'); - -export const command = 'ipld-state'; - -export const desc = 'Reset IPLD state in the given range'; - -export const builder = { - blockNumber: { - type: 'number' - } -}; - -export const handler = async (argv: any): Promise => { - const { blockNumber } = argv; - const config = await getConfig(argv.configFile); - - // Initialize database - const db = new Database(config.database); - await db.init(); - - // Create a DB transaction - const dbTx = await db.createTransactionRunner(); - - console.time('time:reset-ipld-state'); - try { - // Delete all IPLDBlock entries in the given range - await db.removeIPLDBlocksAfterBlock(dbTx, blockNumber); - - // Reset the IPLD status. - const ipldStatus = await db.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockNumber) { - await db.updateIPLDStatusHooksBlock(dbTx, blockNumber, true); - } - - if (ipldStatus.latestCheckpointBlockNumber > blockNumber) { - await db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockNumber) { - await db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, true); - } - } - - dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - console.timeEnd('time:reset-ipld-state'); - - log(`Reset ipld-state successfully to block ${blockNumber}`); -}; diff --git a/packages/codegen/src/templates/reset-state-template.handlebars b/packages/codegen/src/templates/reset-state-template.handlebars index 7a12988c..e5a9918f 100644 --- a/packages/codegen/src/templates/reset-state-template.handlebars +++ b/packages/codegen/src/templates/reset-state-template.handlebars @@ -1,32 +1,18 @@ // -// Copyright 2021 Vulcanize, Inc. +// Copyright 2022 Vulcanize, Inc. // -{{#if (subgraphPath)}} -import path from 'path'; -{{/if}} import debug from 'debug'; -import { MoreThan } from 'typeorm'; -import assert from 'assert'; -import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util'; -{{#if (subgraphPath)}} -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; -{{/if}} +import { getConfig } from '@cerc-io/util'; import { Database } from '../../database'; -import { Indexer } from '../../indexer'; -import { BlockProgress } from '../../entity/BlockProgress'; - -{{#each queries as | query |}} -import { {{query.entityName}} } from '../../entity/{{query.entityName}}'; -{{/each}} const log = debug('vulcanize:reset-state'); export const command = 'state'; -export const desc = 'Reset state to block number'; +export const desc = 'Reset State to a given block number'; export const builder = { blockNumber: { @@ -35,87 +21,34 @@ export const builder = { }; export const handler = async (argv: any): Promise => { + const { blockNumber } = argv; const config = await getConfig(argv.configFile); - await resetJobs(config); - const { ethClient, ethProvider } = await initClients(config); - // Initialize database. + // Initialize database const db = new Database(config.database); await db.init(); - {{#if (subgraphPath)}} - - const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*')); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - {{/if}} - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); - await indexer.init(); - {{#if (subgraphPath)}} - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - {{/if}} - - const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false); - assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`); - assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`); - const [blockProgress] = blockProgresses; + // Create a DB transaction const dbTx = await db.createTransactionRunner(); + console.time('time:reset-state'); try { - const entities = [BlockProgress - {{~#each queries as | query |~}} - , {{query.entityName}} - {{~/each~}} - ]; + // Delete all State entries in the given range + await db.removeStatesAfterBlock(dbTx, blockNumber); - const removeEntitiesPromise = entities.map(async entityClass => { - return db.removeEntities(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) }); - }); + // Reset the stateSyncStatus. + const stateSyncStatus = await db.getStateSyncStatus(); - await Promise.all(removeEntitiesPromise); - - const syncStatus = await indexer.getSyncStatus(); - assert(syncStatus, 'Missing syncStatus'); - - if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { - await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true); - } - - if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) { - await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); - } - - const ipldStatus = await indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true); + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockNumber) { + await db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, true); } - if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true); + if (stateSyncStatus.latestCheckpointBlockNumber > blockNumber) { + await db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, true); } } - await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true); - dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -123,6 +56,7 @@ export const handler = async (argv: any): Promise => { } finally { await dbTx.release(); } + console.timeEnd('time:reset-state'); - log('Reset state successfully'); + log(`Reset State successfully to block ${blockNumber}`); }; diff --git a/packages/codegen/src/templates/reset-watcher-template.handlebars b/packages/codegen/src/templates/reset-watcher-template.handlebars new file mode 100644 index 00000000..77e05404 --- /dev/null +++ b/packages/codegen/src/templates/reset-watcher-template.handlebars @@ -0,0 +1,124 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +{{#if (subgraphPath)}} +import path from 'path'; +{{/if}} +import debug from 'debug'; +import { MoreThan } from 'typeorm'; +import assert from 'assert'; + +import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util'; +{{#if (subgraphPath)}} +import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; +{{/if}} + +import { Database } from '../../database'; +import { Indexer } from '../../indexer'; +import { BlockProgress } from '../../entity/BlockProgress'; + +{{#each queries as | query |}} +import { {{query.entityName}} } from '../../entity/{{query.entityName}}'; +{{/each}} + +const log = debug('vulcanize:reset-watcher'); + +export const command = 'watcher'; + +export const desc = 'Reset watcher to a block number'; + +export const builder = { + blockNumber: { + type: 'number' + } +}; + +export const handler = async (argv: any): Promise => { + const config = await getConfig(argv.configFile); + await resetJobs(config); + const { ethClient, ethProvider } = await initClients(config); + + // Initialize database. + const db = new Database(config.database); + await db.init(); + {{#if (subgraphPath)}} + + const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*')); + await graphDb.init(); + + const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); + {{/if}} + + const jobQueueConfig = config.jobQueue; + assert(jobQueueConfig, 'Missing job queue config'); + + const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; + assert(dbConnectionString, 'Missing job queue db connection string'); + + const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); + await jobQueue.start(); + + const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}}); + await indexer.init(); + {{#if (subgraphPath)}} + + graphWatcher.setIndexer(indexer); + await graphWatcher.init(); + {{/if}} + + const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false); + assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`); + assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`); + const [blockProgress] = blockProgresses; + + const dbTx = await db.createTransactionRunner(); + + try { + const entities = [BlockProgress + {{~#each queries as | query |~}} + , {{query.entityName}} + {{~/each~}} + ]; + + const removeEntitiesPromise = entities.map(async entityClass => { + return db.removeEntities(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) }); + }); + + await Promise.all(removeEntitiesPromise); + + const syncStatus = await indexer.getSyncStatus(); + assert(syncStatus, 'Missing syncStatus'); + + if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true); + } + + if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) { + await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); + } + + const stateSyncStatus = await indexer.getStateSyncStatus(); + + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true); + } + + if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true); + } + } + + await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true); + + dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + + log('Reset watcher successfully'); +}; diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index c26d859f..97d932b9 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -126,9 +126,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getStateByCID').inc(1); - const ipldBlock = await indexer.getIPLDBlockByCid(cid); + const state = await indexer.getStateByCID(cid); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { @@ -136,9 +136,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1); - const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); + const state = await indexer.getPrevState(blockHash, contractAddress, kind); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getSyncStatus: async () => { diff --git a/packages/codegen/src/visitor.ts b/packages/codegen/src/visitor.ts index 812d5eca..eb9bc60e 100644 --- a/packages/codegen/src/visitor.ts +++ b/packages/codegen/src/visitor.ts @@ -210,13 +210,14 @@ export class Visitor { } /** - * Writes the reset.ts, job-queue.ts, state.ts files generated from templates to respective streams. + * Writes the reset.ts, job-queue.ts, watcher.ts, state.ts files generated from templates to respective streams. * @param resetOutStream A writable output stream to write the reset file to. * @param resetJQOutStream A writable output stream to write the reset job-queue file to. + * @param resetWatcherOutStream A writable output stream to write the reset watcher file to. * @param resetStateOutStream A writable output stream to write the reset state file to. */ - exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetStateOutStream: Writable, resetIPLDStateOutStream: Writable | undefined, subgraphPath: string): void { - this._reset.exportReset(resetOutStream, resetJQOutStream, resetStateOutStream, resetIPLDStateOutStream, subgraphPath); + exportReset (resetOutStream: Writable, resetJQOutStream: Writable, resetWatcherOutStream: Writable, resetStateOutStream: Writable, subgraphPath: string): void { + this._reset.exportReset(resetOutStream, resetJQOutStream, resetWatcherOutStream, resetStateOutStream, subgraphPath); } /** diff --git a/packages/eden-watcher/README.md b/packages/eden-watcher/README.md index 5bbdf8b9..bf674c7e 100644 --- a/packages/eden-watcher/README.md +++ b/packages/eden-watcher/README.md @@ -8,12 +8,6 @@ yarn ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - ``` - * Create a postgres12 database for the watcher: ```bash @@ -47,7 +41,7 @@ * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint. - * Update the `server` config with state checkpoint settings and provide the IPFS API address. + * Update the `server` config with state checkpoint settings. ## Customize @@ -59,11 +53,11 @@ * Generating state: - * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object. - * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated. - * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object. ## Run @@ -136,10 +130,10 @@ GQL console: http://localhost:3012/graphql * To reset the watcher to a previous block number: - * Reset state: + * Reset watcher: ```bash - yarn reset state --block-number + yarn reset watcher --block-number ``` * Reset job-queue: @@ -148,6 +142,12 @@ GQL console: http://localhost:3012/graphql yarn reset job-queue --block-number ``` + * Reset state: + + ```bash + yarn reset state --block-number + ``` + * `block-number`: Block number to which to reset the watcher. * To export and import the watcher state: diff --git a/packages/eden-watcher/environments/local.toml b/packages/eden-watcher/environments/local.toml index da973af5..970eef70 100644 --- a/packages/eden-watcher/environments/local.toml +++ b/packages/eden-watcher/environments/local.toml @@ -9,9 +9,6 @@ # Checkpoint interval in number of blocks. checkpointInterval = 2000 - # IPFS API address (can be taken from the output on running the IPFS daemon). - # ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" - subgraphPath = "../graph-node/test/subgraph/eden" # Disable creation of state from subgraph entity updates diff --git a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts index 9fd9b815..27960003 100644 --- a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts +++ b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts @@ -54,12 +54,12 @@ export const handler = async (argv: any): Promise => { graphWatcher.setIndexer(indexer); await graphWatcher.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); - const data = indexer.getIPLDData(ipldBlock); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); + const data = indexer.getStateData(state); - log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`); - await verifyCheckpointData(graphDb, ipldBlock.block, data); + log(`Verifying checkpoint data for contract ${state.contractAddress}`); + await verifyCheckpointData(graphDb, state.block, data); log('Checkpoint data verified'); await db.close(); diff --git a/packages/eden-watcher/src/cli/export-state.ts b/packages/eden-watcher/src/cli/export-state.ts index e772e9b1..ec4f4281 100644 --- a/packages/eden-watcher/src/cli/export-state.ts +++ b/packages/eden-watcher/src/cli/export-state.ts @@ -70,16 +70,16 @@ const main = async (): Promise => { const exportData: any = { snapshotBlock: {}, contracts: [], - ipldCheckpoints: [] + stateCheckpoints: [] }; const contracts = await db.getContracts(); - let block = await indexer.getLatestHooksProcessedBlock(); + let block = await indexer.getLatestStateIndexedBlock(); assert(block); if (argv.blockNumber) { if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`); + throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); } const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); @@ -116,19 +116,15 @@ const main = async (): Promise => { if (contract.checkpoint) { await indexer.createCheckpoint(contract.address, block.blockHash); - const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(ipldBlock); + const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); + assert(state); - const data = indexer.getIPLDData(ipldBlock); + const data = indexer.getStateData(state); - if (indexer.isIPFSConfigured()) { - await indexer.pushToIPFS(data); - } - - exportData.ipldCheckpoints.push({ - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, + exportData.stateCheckpoints.push({ + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, data }); } diff --git a/packages/eden-watcher/src/cli/import-state.ts b/packages/eden-watcher/src/cli/import-state.ts index 6d8751e0..ebb96f44 100644 --- a/packages/eden-watcher/src/cli/import-state.ts +++ b/packages/eden-watcher/src/cli/import-state.ts @@ -18,7 +18,7 @@ import * as codec from '@ipld/dag-cbor'; import { Database } from '../database'; import { Indexer } from '../indexer'; import { EventWatcher } from '../events'; -import { IPLDBlock } from '../entity/IPLDBlock'; +import { State } from '../entity/State'; const log = debug('vulcanize:import-state'); @@ -100,17 +100,17 @@ export const main = async (): Promise => { const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash); assert(block); - // Fill the IPLDBlocks. - for (const checkpoint of importData.ipldCheckpoints) { - let ipldBlock = new IPLDBlock(); + // Fill the States. + for (const checkpoint of importData.stateCheckpoints) { + let state = new State(); - ipldBlock = Object.assign(ipldBlock, checkpoint); - ipldBlock.block = block; + state = Object.assign(state, checkpoint); + state.block = block; - ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data)); + state.data = Buffer.from(codec.encode(state.data)); - ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock); - await graphWatcher.updateEntitiesFromIPLDState(ipldBlock); + state = await indexer.saveOrUpdateState(state); + await graphWatcher.updateEntitiesFromState(state); } // Mark snapshot block as completely processed. @@ -118,12 +118,12 @@ export const main = async (): Promise => { await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber); await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber); - await indexer.updateIPLDStatusHooksBlock(block.blockNumber); - await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber); + await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber); + await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber); - // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block. - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init); - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged); + // The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block. + await indexer.removeStates(block.blockNumber, StateKind.Init); + await indexer.removeStates(block.blockNumber, StateKind.DiffStaged); log(`Import completed for snapshot block at height ${block.blockNumber}`); }; diff --git a/packages/eden-watcher/src/cli/inspect-cid.ts b/packages/eden-watcher/src/cli/inspect-cid.ts index fa4de97d..24e57bae 100644 --- a/packages/eden-watcher/src/cli/inspect-cid.ts +++ b/packages/eden-watcher/src/cli/inspect-cid.ts @@ -63,12 +63,12 @@ const main = async (): Promise => { graphWatcher.setIndexer(indexer); await graphWatcher.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); - const ipldData = await indexer.getIPLDData(ipldBlock); + const stateData = await indexer.getStateData(state); - log(util.inspect(ipldData, false, null)); + log(util.inspect(stateData, false, null)); }; main().catch(err => { diff --git a/packages/eden-watcher/src/cli/reset-cmds/ipld-state.ts b/packages/eden-watcher/src/cli/reset-cmds/ipld-state.ts deleted file mode 100644 index 47c9a02d..00000000 --- a/packages/eden-watcher/src/cli/reset-cmds/ipld-state.ts +++ /dev/null @@ -1,66 +0,0 @@ -// -// Copyright 2022 Vulcanize, Inc. -// - -import debug from 'debug'; - -import { getConfig } from '@cerc-io/util'; - -import { Database } from '../../database'; - -const log = debug('vulcanize:reset-ipld-state'); - -export const command = 'ipld-state'; - -export const desc = 'Reset IPLD state in the given range'; - -export const builder = { - blockNumber: { - type: 'number' - } -}; - -export const handler = async (argv: any): Promise => { - const { blockNumber } = argv; - const config = await getConfig(argv.configFile); - - // Initialize database - const db = new Database(config.database); - await db.init(); - - // Create a DB transaction - const dbTx = await db.createTransactionRunner(); - - console.time('time:reset-ipld-state'); - try { - // Delete all IPLDBlock entries in the given range - await db.removeIPLDBlocksAfterBlock(dbTx, blockNumber); - - // Reset the IPLD status. - const ipldStatus = await db.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockNumber) { - await db.updateIPLDStatusHooksBlock(dbTx, blockNumber, true); - } - - if (ipldStatus.latestCheckpointBlockNumber > blockNumber) { - await db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockNumber) { - await db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, true); - } - } - - dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - console.timeEnd('time:reset-ipld-state'); - - log(`Reset ipld-state successfully to block ${blockNumber}`); -}; diff --git a/packages/eden-watcher/src/cli/reset-cmds/state.ts b/packages/eden-watcher/src/cli/reset-cmds/state.ts index 052804c7..0ed58e20 100644 --- a/packages/eden-watcher/src/cli/reset-cmds/state.ts +++ b/packages/eden-watcher/src/cli/reset-cmds/state.ts @@ -1,42 +1,18 @@ // -// Copyright 2021 Vulcanize, Inc. +// Copyright 2022 Vulcanize, Inc. // -import path from 'path'; import debug from 'debug'; -import { MoreThan } from 'typeorm'; -import assert from 'assert'; -import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util'; -import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; +import { getConfig } from '@cerc-io/util'; import { Database } from '../../database'; -import { Indexer } from '../../indexer'; -import { BlockProgress } from '../../entity/BlockProgress'; -import { Producer } from '../../entity/Producer'; -import { ProducerSet } from '../../entity/ProducerSet'; -import { ProducerSetChange } from '../../entity/ProducerSetChange'; -import { ProducerRewardCollectorChange } from '../../entity/ProducerRewardCollectorChange'; -import { RewardScheduleEntry } from '../../entity/RewardScheduleEntry'; -import { RewardSchedule } from '../../entity/RewardSchedule'; -import { ProducerEpoch } from '../../entity/ProducerEpoch'; -import { Block } from '../../entity/Block'; -import { Epoch } from '../../entity/Epoch'; -import { SlotClaim } from '../../entity/SlotClaim'; -import { Slot } from '../../entity/Slot'; -import { Staker } from '../../entity/Staker'; -import { Network } from '../../entity/Network'; -import { Distributor } from '../../entity/Distributor'; -import { Distribution } from '../../entity/Distribution'; -import { Claim } from '../../entity/Claim'; -import { Slash } from '../../entity/Slash'; -import { Account } from '../../entity/Account'; const log = debug('vulcanize:reset-state'); export const command = 'state'; -export const desc = 'Reset state to block number'; +export const desc = 'Reset State to a given block number'; export const builder = { blockNumber: { @@ -45,77 +21,34 @@ export const builder = { }; export const handler = async (argv: any): Promise => { + const { blockNumber } = argv; const config = await getConfig(argv.configFile); - await resetJobs(config); - const { ethClient, ethProvider } = await initClients(config); - // Initialize database. + // Initialize database const db = new Database(config.database); await db.init(); - const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*')); - await graphDb.init(); - - const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); - - const jobQueueConfig = config.jobQueue; - assert(jobQueueConfig, 'Missing job queue config'); - - const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; - assert(dbConnectionString, 'Missing job queue db connection string'); - - const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); - await jobQueue.start(); - - const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher); - await indexer.init(); - - graphWatcher.setIndexer(indexer); - await graphWatcher.init(); - - const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false); - assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`); - assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`); - const [blockProgress] = blockProgresses; - + // Create a DB transaction const dbTx = await db.createTransactionRunner(); + console.time('time:reset-state'); try { - const entities = [BlockProgress, Producer, ProducerSet, ProducerSetChange, ProducerRewardCollectorChange, RewardScheduleEntry, RewardSchedule, ProducerEpoch, Block, Epoch, SlotClaim, Slot, Staker, Network, Distributor, Distribution, Claim, Slash, Account]; + // Delete all State entries after the given block + await db.removeStatesAfterBlock(dbTx, blockNumber); - for (const entity of entities) { - await db.deleteEntitiesByConditions(dbTx, entity, { blockNumber: MoreThan(argv.blockNumber) }); - } + // Reset the stateSyncStatus. + const stateSyncStatus = await db.getStateSyncStatus(); - const syncStatus = await indexer.getSyncStatus(); - assert(syncStatus, 'Missing syncStatus'); - - if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { - await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true); - } - - if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) { - await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); - } - - const ipldStatus = await indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true); + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockNumber) { + await db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, true); } - if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true); + if (stateSyncStatus.latestCheckpointBlockNumber > blockNumber) { + await db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, true); } } - await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true); - dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -123,6 +56,7 @@ export const handler = async (argv: any): Promise => { } finally { await dbTx.release(); } + console.timeEnd('time:reset-state'); - log('Reset state successfully'); + log(`Reset state successfully to block ${blockNumber}`); }; diff --git a/packages/eden-watcher/src/cli/reset-cmds/watcher.ts b/packages/eden-watcher/src/cli/reset-cmds/watcher.ts new file mode 100644 index 00000000..28faa5c9 --- /dev/null +++ b/packages/eden-watcher/src/cli/reset-cmds/watcher.ts @@ -0,0 +1,124 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import path from 'path'; +import debug from 'debug'; +import { MoreThan } from 'typeorm'; +import assert from 'assert'; + +import { getConfig, initClients, resetJobs, JobQueue } from '@cerc-io/util'; +import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node'; + +import { Database } from '../../database'; +import { Indexer } from '../../indexer'; +import { BlockProgress } from '../../entity/BlockProgress'; +import { Producer } from '../../entity/Producer'; +import { ProducerSet } from '../../entity/ProducerSet'; +import { ProducerSetChange } from '../../entity/ProducerSetChange'; +import { ProducerRewardCollectorChange } from '../../entity/ProducerRewardCollectorChange'; +import { RewardScheduleEntry } from '../../entity/RewardScheduleEntry'; +import { RewardSchedule } from '../../entity/RewardSchedule'; +import { ProducerEpoch } from '../../entity/ProducerEpoch'; +import { Block } from '../../entity/Block'; +import { Epoch } from '../../entity/Epoch'; +import { SlotClaim } from '../../entity/SlotClaim'; +import { Slot } from '../../entity/Slot'; +import { Staker } from '../../entity/Staker'; +import { Network } from '../../entity/Network'; +import { Distributor } from '../../entity/Distributor'; +import { Distribution } from '../../entity/Distribution'; +import { Claim } from '../../entity/Claim'; +import { Slash } from '../../entity/Slash'; +import { Account } from '../../entity/Account'; + +const log = debug('vulcanize:reset-watcher'); + +export const command = 'watcher'; + +export const desc = 'Reset watcher to a block number'; + +export const builder = { + blockNumber: { + type: 'number' + } +}; + +export const handler = async (argv: any): Promise => { + const config = await getConfig(argv.configFile); + await resetJobs(config); + const { ethClient, ethProvider } = await initClients(config); + + // Initialize database. + const db = new Database(config.database); + await db.init(); + + const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, '../../entity/*')); + await graphDb.init(); + + const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); + + const jobQueueConfig = config.jobQueue; + assert(jobQueueConfig, 'Missing job queue config'); + + const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig; + assert(dbConnectionString, 'Missing job queue db connection string'); + + const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs }); + await jobQueue.start(); + + const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher); + await indexer.init(); + + graphWatcher.setIndexer(indexer); + await graphWatcher.init(); + + const blockProgresses = await indexer.getBlocksAtHeight(argv.blockNumber, false); + assert(blockProgresses.length, `No blocks at specified block number ${argv.blockNumber}`); + assert(!blockProgresses.some(block => !block.isComplete), `Incomplete block at block number ${argv.blockNumber} with unprocessed events`); + const [blockProgress] = blockProgresses; + + const dbTx = await db.createTransactionRunner(); + + try { + const entities = [BlockProgress, Producer, ProducerSet, ProducerSetChange, ProducerRewardCollectorChange, RewardScheduleEntry, RewardSchedule, ProducerEpoch, Block, Epoch, SlotClaim, Slot, Staker, Network, Distributor, Distribution, Claim, Slash, Account]; + + for (const entity of entities) { + await db.deleteEntitiesByConditions(dbTx, entity, { blockNumber: MoreThan(argv.blockNumber) }); + } + + const syncStatus = await indexer.getSyncStatus(); + assert(syncStatus, 'Missing syncStatus'); + + if (syncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateSyncStatusIndexedBlock(blockProgress.blockHash, blockProgress.blockNumber, true); + } + + if (syncStatus.latestCanonicalBlockNumber > blockProgress.blockNumber) { + await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); + } + + const stateSyncStatus = await indexer.getStateSyncStatus(); + + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true); + } + + if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true); + } + } + + await indexer.updateSyncStatusChainHead(blockProgress.blockHash, blockProgress.blockNumber, true); + + dbTx.commitTransaction(); + } catch (error) { + await dbTx.rollbackTransaction(); + throw error; + } finally { + await dbTx.release(); + } + + log('Reset watcher successfully'); +}; diff --git a/packages/eden-watcher/src/database.ts b/packages/eden-watcher/src/database.ts index 2a93a88c..5b7ebceb 100644 --- a/packages/eden-watcher/src/database.ts +++ b/packages/eden-watcher/src/database.ts @@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; export class Database implements DatabaseInterface { _config: ConnectionOptions; @@ -39,75 +39,69 @@ export class Database implements DatabaseInterface { return this._baseDatabase.close(); } - getNewIPLDBlock (): IPLDBlock { - return new IPLDBlock(); + getNewState (): State { + return new State(); } - async getIPLDBlocks (where: FindConditions): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getStates (where: FindConditions): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getIPLDBlocks(repo, where); + return this._baseDatabase.getStates(repo, where); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber); + return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind); + return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind); } - // Fetch all diff IPLDBlocks after the specified block number. - async getDiffIPLDBlocksInRange (contractAddress: string, startblock: number, endBlock: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + // Fetch all diff States after the specified block number. + async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startblock, endBlock); + return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock); } - async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock); + return this._baseDatabase.saveOrUpdateState(repo, state); } - async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind); + await this._baseDatabase.removeStates(repo, blockNumber, kind); } - async removeIPLDBlocksAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocksAfterBlock(repo, blockNumber); + await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber); } - async getIPLDStatus (): Promise { - const repo = this._conn.getRepository(IpldStatus); + async getStateSyncStatus (): Promise { + const repo = this._conn.getRepository(StateSyncStatus); - return this._baseDatabase.getIPLDStatus(repo); + return this._baseDatabase.getStateSyncStatus(repo); } - async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force); } - async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force); - } - - async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); - - return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force); } async getContracts (): Promise { diff --git a/packages/eden-watcher/src/entity/IPLDBlock.ts b/packages/eden-watcher/src/entity/State.ts similarity index 96% rename from packages/eden-watcher/src/entity/IPLDBlock.ts rename to packages/eden-watcher/src/entity/State.ts index ddcf2660..10c1bf63 100644 --- a/packages/eden-watcher/src/entity/IPLDBlock.ts +++ b/packages/eden-watcher/src/entity/State.ts @@ -12,7 +12,7 @@ import { BlockProgress } from './BlockProgress'; @Index(['cid'], { unique: true }) @Index(['block', 'contractAddress']) @Index(['block', 'contractAddress', 'kind'], { unique: true }) -export class IPLDBlock { +export class State { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/eden-watcher/src/entity/IpldStatus.ts b/packages/eden-watcher/src/entity/StateSyncStatus.ts similarity index 66% rename from packages/eden-watcher/src/entity/IpldStatus.ts rename to packages/eden-watcher/src/entity/StateSyncStatus.ts index e122bc45..fae389bc 100644 --- a/packages/eden-watcher/src/entity/IpldStatus.ts +++ b/packages/eden-watcher/src/entity/StateSyncStatus.ts @@ -5,16 +5,13 @@ import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; @Entity() -export class IpldStatus { +export class StateSyncStatus { @PrimaryGeneratedColumn() id!: number; @Column('integer') - latestHooksBlockNumber!: number; + latestIndexedBlockNumber!: number; @Column('integer', { nullable: true }) latestCheckpointBlockNumber!: number; - - @Column('integer', { nullable: true }) - latestIPFSBlockNumber!: number; } diff --git a/packages/eden-watcher/src/fill-state.ts b/packages/eden-watcher/src/fill-state.ts index b5da9d1f..e1994321 100644 --- a/packages/eden-watcher/src/fill-state.ts +++ b/packages/eden-watcher/src/fill-state.ts @@ -31,7 +31,7 @@ export const fillState = async ( log(`Filling state for subgraph entities in range: [${startBlock}, ${endBlock}]`); // Check that there are no existing diffs in this range - const existingStates = await indexer.getIPLDBlocks({ block: { blockNumber: Between(startBlock, endBlock) } }); + const existingStates = await indexer.getStates({ block: { blockNumber: Between(startBlock, endBlock) } }); if (existingStates.length > 0) { log('found existing state(s) in the given range'); process.exit(1); @@ -97,26 +97,11 @@ export const fillState = async ( // Persist subgraph state to the DB await indexer.dumpSubgraphState(blockHash, true); - await indexer.updateIPLDStatusHooksBlock(blockNumber); + await indexer.updateStateSyncStatusIndexedBlock(blockNumber); // Create checkpoints await indexer.processCheckpoint(blockHash); - await indexer.updateIPLDStatusCheckpointBlock(blockNumber); - - // TODO: Push state to IPFS in separate process. - if (indexer.isIPFSConfigured()) { - // Get IPLDBlocks for the given blocHash. - const ipldBlocks = await indexer.getIPLDBlocksByHash(blockHash); - - // Push all the IPLDBlocks to IPFS. - for (const ipldBlock of ipldBlocks) { - const data = indexer.getIPLDData(ipldBlock); - await indexer.pushToIPFS(data); - } - - // Update the IPLD status. - await indexer.updateIPLDStatusIPFSBlock(blockNumber); - } + await indexer.updateStateSyncStatusCheckpointBlock(blockNumber); console.timeEnd(`time:fill-state-${blockNumber}`); } diff --git a/packages/eden-watcher/src/hooks.ts b/packages/eden-watcher/src/hooks.ts index fa42a3f3..3dc50420 100644 --- a/packages/eden-watcher/src/hooks.ts +++ b/packages/eden-watcher/src/hooks.ts @@ -2,15 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import { IPLDBlockInterface, StateKind } from '@cerc-io/util'; import assert from 'assert'; -import * as codec from '@ipld/dag-cbor'; -import _ from 'lodash'; import { Indexer, ResultEvent } from './indexer'; -const IPLD_BATCH_BLOCKS = 10000; - /** * Hook function to store an initial state. * @param indexer Indexer instance. @@ -23,13 +18,13 @@ export async function createInitialState (indexer: Indexer, contractAddress: str assert(blockHash); assert(contractAddress); - // Store an empty state in an IPLDBlock. - const ipldBlockData: any = { + // Store an empty State. + const stateData: any = { state: {} }; // Return initial state data to be saved. - return ipldBlockData; + return stateData; } /** @@ -56,64 +51,11 @@ export async function createStateCheckpoint (indexer: Indexer, contractAddress: assert(blockHash); assert(contractAddress); - // TODO: Pass blockProgress instead of blockHash to hook method. - const block = await indexer.getBlockProgress(blockHash); - assert(block); + // Use indexer.createStateCheckpoint() method to create a custom checkpoint. - // Fetch the latest 'checkpoint' | 'init' for the contract to fetch diffs after it. - let prevNonDiffBlock: IPLDBlockInterface; - let diffStartBlockNumber: number; - const checkpointBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Checkpoint, block.blockNumber - 1); - - if (checkpointBlock) { - const checkpointBlockNumber = checkpointBlock.block.blockNumber; - - prevNonDiffBlock = checkpointBlock; - diffStartBlockNumber = checkpointBlockNumber; - - // Update IPLD status map with the latest checkpoint info. - // Essential while importing state as checkpoint at the snapshot block is added by import-state CLI. - // (job-runner won't have the updated ipld status) - indexer.updateIPLDStatusMap(contractAddress, { checkpoint: checkpointBlockNumber }); - } else { - // There should be an initial state at least. - const initBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Init); - assert(initBlock, 'No initial state found'); - - prevNonDiffBlock = initBlock; - // Take block number previous to initial state block to include any diff state at that block. - diffStartBlockNumber = initBlock.block.blockNumber - 1; - } - - const prevNonDiffBlockData = codec.decode(Buffer.from(prevNonDiffBlock.data)) as any; - const data = { - state: prevNonDiffBlockData.state - }; - - console.time('time:hooks#createStateCheckpoint'); - - // Fetching and merging all diff blocks after the latest 'checkpoint' | 'init' in batch. - for (let i = diffStartBlockNumber; i < block.blockNumber;) { - const endBlockHeight = Math.min(i + IPLD_BATCH_BLOCKS, block.blockNumber); - console.time(`time:hooks#createStateCheckpoint-batch-merge-diff-${i}-${endBlockHeight}`); - const diffBlocks = await indexer.getDiffIPLDBlocksInRange(contractAddress, i, endBlockHeight); - - // Merge all diff blocks after previous checkpoint. - for (const diffBlock of diffBlocks) { - const diff = codec.decode(Buffer.from(diffBlock.data)) as any; - data.state = _.merge(data.state, diff.state); - } - - console.timeEnd(`time:hooks#createStateCheckpoint-batch-merge-diff-${i}-${endBlockHeight}`); - i = endBlockHeight; - } - - console.time('time:hooks#createStateCheckpoint-db-save-checkpoint'); - await indexer.createStateCheckpoint(contractAddress, blockHash, data); - console.timeEnd('time:hooks#createStateCheckpoint-db-save-checkpoint'); - - console.timeEnd('time:hooks#createStateCheckpoint'); - return true; + // Return false to update the state created by this hook by auto-generated checkpoint state. + // Return true to disable update of the state created by this hook by auto-generated checkpoint state. + return false; } /** diff --git a/packages/eden-watcher/src/indexer.ts b/packages/eden-watcher/src/indexer.ts index 903bf92f..da69f04f 100644 --- a/packages/eden-watcher/src/indexer.ts +++ b/packages/eden-watcher/src/indexer.ts @@ -12,7 +12,6 @@ import { SelectionNode } from 'graphql'; import { JsonFragment } from '@ethersproject/abi'; import { BaseProvider } from '@ethersproject/providers'; -import * as codec from '@ipld/dag-cbor'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; import { @@ -23,12 +22,10 @@ import { Where, QueryOptions, BlockHeight, - IPFSClient, StateKind, IndexerInterface, - IpldStatus as IpldStatusInterface, - ValueResult, - ResultIPLDBlock + StateStatus, + ValueResult } from '@cerc-io/util'; import { GraphWatcher } from '@cerc-io/graph-node'; @@ -36,9 +33,9 @@ import { Database } from './database'; import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import EdenNetworkArtifacts from './artifacts/EdenNetwork.json'; import MerkleDistributorArtifacts from './artifacts/MerkleDistributor.json'; import DistributorGovernanceArtifacts from './artifacts/DistributorGovernance.json'; @@ -103,8 +100,6 @@ export class Indexer implements IndexerInterface { _storageLayoutMap: Map _contractMap: Map - _ipfsClient: IPFSClient - _entityTypesMap: Map _relationsMap: Map @@ -118,8 +113,7 @@ export class Indexer implements IndexerInterface { this._ethClient = ethClient; this._ethProvider = ethProvider; this._serverConfig = serverConfig; - this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); - this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient); + this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); this._graphWatcher = graphWatcher; this._abiMap = new Map(); @@ -170,7 +164,7 @@ export class Indexer implements IndexerInterface { async init (): Promise { await this._baseIndexer.fetchContracts(); - await this._baseIndexer.fetchIPLDStatus(); + await this._baseIndexer.fetchStateStatus(); } getResultEvent (event: Event): ResultEvent { @@ -208,26 +202,6 @@ export class Indexer implements IndexerInterface { }; } - getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock { - const block = ipldBlock.block; - - const data = codec.decode(Buffer.from(ipldBlock.data)) as any; - - return { - block: { - cid: block.cid, - hash: block.blockHash, - number: block.blockNumber, - timestamp: block.blockTimestamp, - parentHash: block.parentHash - }, - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, - data: JSON.stringify(data) - }; - } - async getStorageValue (storageLayout: StorageLayout, blockHash: string, contractAddress: string, variable: string, ...mappingKeys: MappingKey[]): Promise { return this._baseIndexer.getStorageValue( storageLayout, @@ -238,10 +212,6 @@ export class Indexer implements IndexerInterface { ); } - async pushToIPFS (data: any): Promise { - await this._baseIndexer.pushToIPFS(data); - } - async processInitialState (contractAddress: string, blockHash: string): Promise { // Call initial state hook. return createInitialState(this, contractAddress, blockHash); @@ -280,36 +250,32 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + return this._db.getPrevState(blockHash, contractAddress, kind); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + return this._db.getLatestState(contractAddress, kind, blockNumber); } - async getIPLDBlocksByHash (blockHash: string): Promise { - return this._baseIndexer.getIPLDBlocksByHash(blockHash); + async getStatesByHash (blockHash: string): Promise { + return this._baseIndexer.getStatesByHash(blockHash); } - async getIPLDBlockByCid (cid: string): Promise { - return this._baseIndexer.getIPLDBlockByCid(cid); + async getStateByCID (cid: string): Promise { + return this._baseIndexer.getStateByCID(cid); } - async getIPLDBlocks (where: FindConditions): Promise { - return this._db.getIPLDBlocks(where); + async getStates (where: FindConditions): Promise { + return this._db.getStates(where); } - async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise { - return this._db.getDiffIPLDBlocksInRange(contractAddress, startBlock, endBlock); + async getDiffStatesInRange (contractAddress: string, startBlock: number, endBlock: number): Promise { + return this._db.getDiffStatesInRange(contractAddress, startBlock, endBlock); } - getIPLDData (ipldBlock: IPLDBlock): any { - return this._baseIndexer.getIPLDData(ipldBlock); - } - - isIPFSConfigured (): boolean { - return this._baseIndexer.isIPFSConfigured(); + getStateData (state: State): any { + return this._baseIndexer.getStateData(state); } // Method used to create auto diffs (diff_staged). @@ -351,12 +317,12 @@ export class Indexer implements IndexerInterface { await this._baseIndexer.createInit(this, blockHash, blockNumber); } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock); + async saveOrUpdateState (state: State): Promise { + return this._baseIndexer.saveOrUpdateState(state); } - async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise { - await this._baseIndexer.removeIPLDBlocks(blockNumber, kind); + async removeStates (blockNumber: number, kind: StateKind): Promise { + await this._baseIndexer.removeStates(blockNumber, kind); } async getSubgraphEntity (entity: new () => Entity, id: string, block: BlockHeight, selections: ReadonlyArray = []): Promise { @@ -436,16 +402,16 @@ export class Indexer implements IndexerInterface { }; } - async getIPLDStatus (): Promise { - return this._db.getIPLDStatus(); + async getStateSyncStatus (): Promise { + return this._db.getStateSyncStatus(); } - async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -457,29 +423,12 @@ export class Indexer implements IndexerInterface { return res; } - async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force); - await dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - - return res; - } - - async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise { - const dbTx = await this._db.createTransactionRunner(); - let res; - - try { - res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -501,16 +450,16 @@ export class Indexer implements IndexerInterface { return latestCanonicalBlock; } - async getLatestHooksProcessedBlock (): Promise { - return this._baseIndexer.getLatestHooksProcessedBlock(); + async getLatestStateIndexedBlock (): Promise { + return this._baseIndexer.getLatestStateIndexedBlock(); } async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { - await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + this._baseIndexer.updateStateStatusMap(address, stateStatus); } cacheContract (contract: Contract): void { diff --git a/packages/eden-watcher/src/job-runner.ts b/packages/eden-watcher/src/job-runner.ts index 240b3f18..09db8431 100644 --- a/packages/eden-watcher/src/job-runner.ts +++ b/packages/eden-watcher/src/job-runner.ts @@ -18,8 +18,6 @@ import { QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, QUEUE_HOOKS, - QUEUE_IPFS, - JOB_KIND_PRUNE, JobQueueConfig, DEFAULT_CONFIG_PATH, initClients, @@ -50,22 +48,12 @@ export class JobRunner { await this.subscribeEventProcessingQueue(); await this.subscribeBlockCheckpointQueue(); await this.subscribeHooksQueue(); - await this.subscribeIPFSQueue(); this._baseJobRunner.handleShutdown(); } async subscribeBlockProcessingQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseJobRunner.processBlock(job); - - const { data: { kind } } = job; - - // If it's a pruning job: Create a hooks job. - if (kind === JOB_KIND_PRUNE) { - await this.createHooksJob(); - } - - await this._jobQueue.markComplete(job); }); } @@ -77,165 +65,15 @@ export class JobRunner { async subscribeHooksQueue (): Promise { await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) { - // Create hooks job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) { - log(`Hooks for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process the hooks for the given block number. - await this._indexer.processCanonicalBlock(blockHash, blockNumber); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusHooksBlock(blockNumber); - - // Create a checkpoint job after completion of a hook job. - await this.createCheckpointJob(blockHash, blockNumber); - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processHooks(job); }); } async subscribeBlockCheckpointQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestCheckpointBlockNumber >= 0) { - if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) { - // Create a checkpoint job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) { - log(`Checkpoints for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process checkpoints for the given block. - await this._indexer.processCheckpoint(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber); - - // Create an IPFS job after completion of a checkpoint job. - if (this._indexer.isIPFSConfigured()) { - await this.createIPFSPutJob(blockHash, blockNumber); - } - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processCheckpoint(job); }); } - - async subscribeIPFSQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestIPFSBlockNumber >= 0) { - if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) { - // Create a IPFS job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) { - log(`IPFS for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Get IPLDBlocks for the given blocHash. - const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash); - - // Push all the IPLDBlocks to IPFS. - for (const ipldBlock of ipldBlocks) { - const data = this._indexer.getIPLDData(ipldBlock); - await this._indexer.pushToIPFS(data); - } - - // Update the IPLD status. - await this._indexer.updateIPLDStatusIPFSBlock(blockNumber); - - await this._jobQueue.markComplete(job); - }); - } - - async createHooksJob (blockHash?: string, blockNumber?: number): Promise { - if (!blockNumber || !blockHash) { - // Get the latest canonical block - const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock(); - - // Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block. - blockHash = latestCanonicalBlock.parentHash; - blockNumber = latestCanonicalBlock.blockNumber - 1; - } - - await this._jobQueue.pushJob( - QUEUE_HOOKS, - { - blockHash, - blockNumber - } - ); - } - - async createCheckpointJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_BLOCK_CHECKPOINT, - { - blockHash, - blockNumber - } - ); - } - - async createIPFSPutJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_IPFS, - { - blockHash, - blockNumber - } - ); - } } export const main = async (): Promise => { diff --git a/packages/eden-watcher/src/resolvers.ts b/packages/eden-watcher/src/resolvers.ts index 5e8fb008..5cd789fa 100644 --- a/packages/eden-watcher/src/resolvers.ts +++ b/packages/eden-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql'; -import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -442,9 +442,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getStateByCID').inc(1); - const ipldBlock = await indexer.getIPLDBlockByCid(cid); + const state = await indexer.getStateByCID(cid); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { @@ -452,9 +452,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1); - const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); + const state = await indexer.getPrevState(blockHash, contractAddress, kind); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getSyncStatus: async () => { diff --git a/packages/eden-watcher/src/schema.gql b/packages/eden-watcher/src/schema.gql index 257270e7..d5188884 100644 --- a/packages/eden-watcher/src/schema.gql +++ b/packages/eden-watcher/src/schema.gql @@ -203,7 +203,7 @@ type RoleRevokedEvent { sender: String! } -type ResultIPLDBlock { +type ResultState { block: _Block_! contractAddress: String! cid: String! @@ -248,8 +248,8 @@ type Query { claim(id: String!, block: Block_height): Claim! slash(id: String!, block: Block_height): Slash! account(id: String!, block: Block_height): Account! - getStateByCID(cid: String!): ResultIPLDBlock - getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock + getStateByCID(cid: String!): ResultState + getState(blockHash: String!, contractAddress: String!, kind: String): ResultState getSyncStatus: SyncStatus } diff --git a/packages/erc20-watcher/src/cli/reset-cmds/state.ts b/packages/erc20-watcher/src/cli/reset-cmds/watcher.ts similarity index 93% rename from packages/erc20-watcher/src/cli/reset-cmds/state.ts rename to packages/erc20-watcher/src/cli/reset-cmds/watcher.ts index 6dd70016..b21aaf60 100644 --- a/packages/erc20-watcher/src/cli/reset-cmds/state.ts +++ b/packages/erc20-watcher/src/cli/reset-cmds/watcher.ts @@ -14,11 +14,11 @@ import { BlockProgress } from '../../entity/BlockProgress'; import { Allowance } from '../../entity/Allowance'; import { Balance } from '../../entity/Balance'; -const log = debug('vulcanize:reset-state'); +const log = debug('vulcanize:reset-watcher'); -export const command = 'state'; +export const command = 'watcher'; -export const desc = 'Reset state to block number'; +export const desc = 'Reset watcher to a block number'; export const builder = { blockNumber: { @@ -78,5 +78,5 @@ export const handler = async (argv: any): Promise => { await dbTx.release(); } - log('Reset state successfully'); + log('Reset watcher successfully'); }; diff --git a/packages/erc20-watcher/src/database.ts b/packages/erc20-watcher/src/database.ts index f278d888..ff69b49c 100644 --- a/packages/erc20-watcher/src/database.ts +++ b/packages/erc20-watcher/src/database.ts @@ -14,8 +14,8 @@ import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; -import { IpldStatus } from './entity/IpldStatus'; +import { State } from './entity/State'; +import { StateSyncStatus } from './entity/StateSyncStatus'; export class Database implements DatabaseInterface { _config: ConnectionOptions @@ -41,45 +41,57 @@ export class Database implements DatabaseInterface { return this._baseDatabase.close(); } - getNewIPLDBlock (): IPLDBlock { - return new IPLDBlock(); + getNewState (): State { + return new State(); } - async getIPLDBlocks (where: FindConditions): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getStates (where: FindConditions): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getIPLDBlocks(repo, where); + return this._baseDatabase.getStates(repo, where); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber); + return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber); } - // Fetch all diff IPLDBlocks after the specified block number. - async getDiffIPLDBlocksInRange (contractAddress: string, startblock: number, endBlock: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startblock, endBlock); + return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind); } - async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + // Fetch all diff States after the specified block number. + async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock); + return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock); } - async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind); + return this._baseDatabase.saveOrUpdateState(repo, state); } - async getIPLDStatus (): Promise { - const repo = this._conn.getRepository(IpldStatus); + async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.getIPLDStatus(repo); + await this._baseDatabase.removeStates(repo, blockNumber, kind); + } + + async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise { + const repo = dbTx.manager.getRepository(State); + + await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber); + } + + async getStateSyncStatus (): Promise { + const repo = this._conn.getRepository(StateSyncStatus); + + return this._baseDatabase.getStateSyncStatus(repo); } async getBalance ({ blockHash, token, owner }: { blockHash: string, token: string, owner: string }): Promise { diff --git a/packages/erc20-watcher/src/entity/IpldStatus.ts b/packages/erc20-watcher/src/entity/IpldStatus.ts deleted file mode 100644 index fb81069e..00000000 --- a/packages/erc20-watcher/src/entity/IpldStatus.ts +++ /dev/null @@ -1,20 +0,0 @@ -// -// Copyright 2022 Vulcanize, Inc. -// - -import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; - -@Entity() -export class IpldStatus { - @PrimaryGeneratedColumn() - id!: number; - - @Column('integer') - latestHooksBlockNumber!: number; - - @Column('integer', { nullable: true }) - latestCheckpointBlockNumber!: number; - - @Column('integer', { nullable: true }) - latestIPFSBlockNumber!: number; -} diff --git a/packages/erc20-watcher/src/entity/IPLDBlock.ts b/packages/erc20-watcher/src/entity/State.ts similarity index 96% rename from packages/erc20-watcher/src/entity/IPLDBlock.ts rename to packages/erc20-watcher/src/entity/State.ts index bff1118c..7cf3e49b 100644 --- a/packages/erc20-watcher/src/entity/IPLDBlock.ts +++ b/packages/erc20-watcher/src/entity/State.ts @@ -12,7 +12,7 @@ import { BlockProgress } from './BlockProgress'; @Index(['cid'], { unique: true }) @Index(['block', 'contractAddress']) @Index(['block', 'contractAddress', 'kind'], { unique: true }) -export class IPLDBlock { +export class State { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/graph-test-watcher/src/entity/IpldStatus.ts b/packages/erc20-watcher/src/entity/StateSyncStatus.ts similarity index 66% rename from packages/graph-test-watcher/src/entity/IpldStatus.ts rename to packages/erc20-watcher/src/entity/StateSyncStatus.ts index e122bc45..fae389bc 100644 --- a/packages/graph-test-watcher/src/entity/IpldStatus.ts +++ b/packages/erc20-watcher/src/entity/StateSyncStatus.ts @@ -5,16 +5,13 @@ import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; @Entity() -export class IpldStatus { +export class StateSyncStatus { @PrimaryGeneratedColumn() id!: number; @Column('integer') - latestHooksBlockNumber!: number; + latestIndexedBlockNumber!: number; @Column('integer', { nullable: true }) latestCheckpointBlockNumber!: number; - - @Column('integer', { nullable: true }) - latestIPFSBlockNumber!: number; } diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 7a6877f6..62558464 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -12,16 +12,17 @@ import { BaseProvider } from '@ethersproject/providers'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; -import { IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, JobQueue, Where, QueryOptions, ServerConfig, IPFSClient, IpldStatus as IpldStatusInterface } from '@cerc-io/util'; +import { IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, JobQueue, Where, QueryOptions, ServerConfig, StateStatus } from '@cerc-io/util'; import { Database } from './database'; import { Event } from './entity/Event'; import { fetchTokenDecimals, fetchTokenName, fetchTokenSymbol, fetchTokenTotalSupply } from './utils'; import { SyncStatus } from './entity/SyncStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import artifacts from './artifacts/ERC20.json'; import { BlockProgress } from './entity/BlockProgress'; import { Contract } from './entity/Contract'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; const log = debug('vulcanize:indexer'); const JSONbigNative = JSONbig({ useNativeBigInt: true }); @@ -64,8 +65,7 @@ export class Indexer implements IndexerInterface { this._ethProvider = ethProvider; this._serverConfig = serverConfig; this._serverMode = serverConfig.mode; - const ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); - this._baseIndexer = new BaseIndexer(serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, ipfsClient); + this._baseIndexer = new BaseIndexer(serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); const { abi, storageLayout } = artifacts; @@ -250,8 +250,16 @@ export class Indexer implements IndexerInterface { ); } - getIPLDData (ipldBlock: IPLDBlock): any { - return this._baseIndexer.getIPLDData(ipldBlock); + async processCanonicalBlock (blockHash: string, blockNumber: number): Promise { + // TODO Implement + } + + async processCheckpoint (blockHash: string): Promise { + // TODO Implement + } + + getStateData (state: State): any { + return this._baseIndexer.getStateData(state); } async triggerIndexingOnEvent (event: Event): Promise { @@ -298,6 +306,30 @@ export class Indexer implements IndexerInterface { return { eventName, eventInfo }; } + async getStateSyncStatus (): Promise { + return this._db.getStateSyncStatus(); + } + + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { + // TODO Implement + return {} as StateSyncStatus; + } + + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + // TODO Implement + return {} as StateSyncStatus; + } + + async getLatestCanonicalBlock (): Promise { + const syncStatus = await this.getSyncStatus(); + assert(syncStatus); + + const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash); + assert(latestCanonicalBlock); + + return latestCanonicalBlock; + } + async getEventsByFilter (blockHash: string, contract: string, name?: string): Promise> { return this._baseIndexer.getEventsByFilter(blockHash, contract, name); } @@ -310,8 +342,8 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { - await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + this._baseIndexer.updateStateStatusMap(address, stateStatus); } cacheContract (contract: Contract): void { diff --git a/packages/erc721-watcher/README.md b/packages/erc721-watcher/README.md index 7f75e1f8..2df4ef94 100644 --- a/packages/erc721-watcher/README.md +++ b/packages/erc721-watcher/README.md @@ -8,12 +8,6 @@ yarn ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - ``` - * Create a postgres12 database for the watcher: ```bash @@ -42,9 +36,9 @@ ``` * The following core services should be setup and running on localhost: - + * `vulcanize/go-ethereum` [v1.10.18-statediff-4.0.2-alpha](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.18-statediff-4.0.2-alpha) on port 8545 - + * `vulcanize/ipld-eth-server` [v4.0.3-alpha](https://github.com/vulcanize/ipld-eth-server/releases/tag/v4.0.3-alpha) with native GQL API enabled, on port 8082 * In the [config file](./environments/local.toml): @@ -53,7 +47,7 @@ * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint. - * Update the `server` config with state checkpoint settings and provide the IPFS API address. + * Update the `server` config with state checkpoint settings. ## Customize @@ -65,11 +59,11 @@ * Generating state: - * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object. - * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated. - * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object. ## Run @@ -136,10 +130,10 @@ GQL console: http://localhost:3006/graphql * To reset the watcher to a previous block number: - * Reset state: + * Reset watcher: ```bash - yarn reset state --block-number + yarn reset watcher --block-number ``` * Reset job-queue: diff --git a/packages/erc721-watcher/demo.md b/packages/erc721-watcher/demo.md index 08a28c28..4b207775 100644 --- a/packages/erc721-watcher/demo.md +++ b/packages/erc721-watcher/demo.md @@ -26,14 +26,14 @@ ```bash docker-compose version - + # docker-compose version 1.29.2, build 5becea4c ``` * Run the stack-orchestrator ```bash - cd stack-orchestrator/helper-scripts + cd stack-orchestrator/helper-scripts ``` ```bash @@ -45,22 +45,11 @@ -p ../config.sh ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - - # API server listening on /ip4/127.0.0.1/tcp/5001 - ``` - The IPFS API address can be seen in the output. - -* In the [config file](./environments/local.toml) update the `server.ipfsApiAddr` config with the IPFS API address. - * Create a postgres12 database for the watcher: ```bash sudo su - postgres - + # If database already exists # dropdb erc721-watcher @@ -135,7 +124,7 @@ ``` * Get the signer account address and export to a shell variable: - + ```bash yarn account ``` @@ -262,11 +251,11 @@ * A Transfer event to SIGNER_ADDRESS shall be visible in the subscription at endpoint. - * An auto-generated `diff_staged` IPLDBlock should be added with parent CID pointing to the initial checkpoint IPLDBlock. + * An auto-generated `diff_staged` `State` should be added with parent CID pointing to the initial `checkpoint` `State`. * Custom property `transferCount` should be 1 initially. -* Run the `getState` query at the endpoint to get the latest IPLDBlock for NFT_ADDRESS: +* Run the `getState` query at the endpoint to get the latest `State` for NFT_ADDRESS: ```graphql query { @@ -291,7 +280,7 @@ } ``` - * `diff` IPLDBlocks get created corresponding to the `diff_staged` blocks when their respective eth_blocks reach the pruned region. + * `diff` States get created corresponding to the `diff_staged` blocks when their respective eth_blocks reach the pruned region. * `data` contains the default state and also the custom state property `transferCount` that is indexed in [hooks.ts](./src/hooks.ts) file. @@ -360,9 +349,9 @@ * A Transfer event to $RECIPIENT_ADDRESS shall be visible in the subscription at endpoint. - * An auto-generated `diff_staged` IPLDBlock should be added with parent CID pointing to the previous IPLDBlock. + * An auto-generated `diff_staged` State should be added with parent CID pointing to the previous State. - * Custom property `transferCount` should be incremented after transfer. This can be checked in the `getState` query and in IPFS webUI mentioned in the later steps. + * Custom property `transferCount` should be incremented after transfer. This can be checked in the `getState` query. * Get the latest blockHash and replace the blockHash in the above `eth_call` query. The result should be different and the token should be transferred to the recipient. @@ -380,11 +369,7 @@ * The latest checkpoint should have the aggregate of state diffs since the last checkpoint. - * The IPLDBlock entries can be seen in pg-admin in table ipld_block. - -* All the diff and checkpoint IPLDBlocks should pushed to IPFS. - -* Open IPFS WebUI http://127.0.0.1:5001/webui and search for IPLDBlocks using their CIDs. + * The `State` entries can be seen in pg-admin in table `state`. * The state should have auto indexed data and also custom property `transferCount` according to code in [hooks](./src/hooks.ts) file `handleEvent` method. diff --git a/packages/erc721-watcher/environments/local.toml b/packages/erc721-watcher/environments/local.toml index 7747a449..6af96571 100644 --- a/packages/erc721-watcher/environments/local.toml +++ b/packages/erc721-watcher/environments/local.toml @@ -9,9 +9,6 @@ # Checkpoint interval in number of blocks. checkpointInterval = 2000 - # IPFS API address (can be taken from the output on running the IPFS daemon). - ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" - # Boolean to filter logs by contract. filterLogs = false diff --git a/packages/erc721-watcher/src/cli/export-state.ts b/packages/erc721-watcher/src/cli/export-state.ts index fe88e1ba..55b80846 100644 --- a/packages/erc721-watcher/src/cli/export-state.ts +++ b/packages/erc721-watcher/src/cli/export-state.ts @@ -61,16 +61,16 @@ const main = async (): Promise => { const exportData: any = { snapshotBlock: {}, contracts: [], - ipldCheckpoints: [] + stateCheckpoints: [] }; const contracts = await db.getContracts(); - let block = await indexer.getLatestHooksProcessedBlock(); + let block = await indexer.getLatestStateIndexedBlock(); assert(block); if (argv.blockNumber) { if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`); + throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); } const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); @@ -107,19 +107,15 @@ const main = async (): Promise => { if (contract.checkpoint) { await indexer.createCheckpoint(contract.address, block.blockHash); - const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(ipldBlock); + const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); + assert(state); - const data = indexer.getIPLDData(ipldBlock); + const data = indexer.getStateData(state); - if (indexer.isIPFSConfigured()) { - await indexer.pushToIPFS(data); - } - - exportData.ipldCheckpoints.push({ - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, + exportData.stateCheckpoints.push({ + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, data }); } diff --git a/packages/erc721-watcher/src/cli/import-state.ts b/packages/erc721-watcher/src/cli/import-state.ts index 36073451..250d09bd 100644 --- a/packages/erc721-watcher/src/cli/import-state.ts +++ b/packages/erc721-watcher/src/cli/import-state.ts @@ -17,7 +17,7 @@ import * as codec from '@ipld/dag-cbor'; import { Database } from '../database'; import { Indexer } from '../indexer'; import { EventWatcher } from '../events'; -import { IPLDBlock } from '../entity/IPLDBlock'; +import { State } from '../entity/State'; const log = debug('vulcanize:import-state'); @@ -91,16 +91,16 @@ export const main = async (): Promise => { const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash); assert(block); - // Fill the IPLDBlocks. - for (const checkpoint of importData.ipldCheckpoints) { - let ipldBlock = new IPLDBlock(); + // Fill the States. + for (const checkpoint of importData.stateCheckpoints) { + let state = new State(); - ipldBlock = Object.assign(ipldBlock, checkpoint); - ipldBlock.block = block; + state = Object.assign(state, checkpoint); + state.block = block; - ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data)); + state.data = Buffer.from(codec.encode(state.data)); - ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock); + state = await indexer.saveOrUpdateState(state); } // Mark snapshot block as completely processed. @@ -108,12 +108,12 @@ export const main = async (): Promise => { await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber); await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber); - await indexer.updateIPLDStatusHooksBlock(block.blockNumber); - await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber); + await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber); + await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber); - // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block. - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init); - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged); + // The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block. + await indexer.removeStates(block.blockNumber, StateKind.Init); + await indexer.removeStates(block.blockNumber, StateKind.DiffStaged); log(`Import completed for snapshot block at height ${block.blockNumber}`); }; diff --git a/packages/erc721-watcher/src/cli/inspect-cid.ts b/packages/erc721-watcher/src/cli/inspect-cid.ts index 2b48fa32..c32ba47c 100644 --- a/packages/erc721-watcher/src/cli/inspect-cid.ts +++ b/packages/erc721-watcher/src/cli/inspect-cid.ts @@ -53,12 +53,12 @@ const main = async (): Promise => { const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue); await indexer.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); - const ipldData = await indexer.getIPLDData(ipldBlock); + const stateData = await indexer.getStateData(state); - log(util.inspect(ipldData, false, null)); + log(util.inspect(stateData, false, null)); }; main().catch(err => { diff --git a/packages/erc721-watcher/src/cli/reset-cmds/state.ts b/packages/erc721-watcher/src/cli/reset-cmds/watcher.ts similarity index 82% rename from packages/erc721-watcher/src/cli/reset-cmds/state.ts rename to packages/erc721-watcher/src/cli/reset-cmds/watcher.ts index 68b15384..d1181020 100644 --- a/packages/erc721-watcher/src/cli/reset-cmds/state.ts +++ b/packages/erc721-watcher/src/cli/reset-cmds/watcher.ts @@ -27,11 +27,11 @@ import { _Balances } from '../../entity/_Balances'; import { _TokenApprovals } from '../../entity/_TokenApprovals'; import { _OperatorApprovals } from '../../entity/_OperatorApprovals'; -const log = debug('vulcanize:reset-state'); +const log = debug('vulcanize:reset-watcher'); -export const command = 'state'; +export const command = 'watcher'; -export const desc = 'Reset state to block number'; +export const desc = 'Reset watcher to a block number'; export const builder = { blockNumber: { @@ -85,19 +85,15 @@ export const handler = async (argv: any): Promise => { await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); } - const ipldStatus = await indexer.getIPLDStatus(); + const stateSyncStatus = await indexer.getStateSyncStatus(); - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true); + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true); } - if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true); + if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true); } } @@ -111,5 +107,5 @@ export const handler = async (argv: any): Promise => { await dbTx.release(); } - log('Reset state successfully'); + log('Reset watcher successfully'); }; diff --git a/packages/erc721-watcher/src/database.ts b/packages/erc721-watcher/src/database.ts index a8c57866..f56ba5cf 100644 --- a/packages/erc721-watcher/src/database.ts +++ b/packages/erc721-watcher/src/database.ts @@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import { SupportsInterface } from './entity/SupportsInterface'; import { BalanceOf } from './entity/BalanceOf'; import { OwnerOf } from './entity/OwnerOf'; @@ -300,69 +300,69 @@ export class Database implements DatabaseInterface { return repo.save(entity); } - getNewIPLDBlock (): IPLDBlock { - return new IPLDBlock(); + getNewState (): State { + return new State(); } - async getIPLDBlocks (where: FindConditions): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getStates (where: FindConditions): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getIPLDBlocks(repo, where); + return this._baseDatabase.getStates(repo, where); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber); + return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind); + return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind); } - // Fetch all diff IPLDBlocks after the specified block number. - async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + // Fetch all diff States after the specified block number. + async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock); + return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock); } - async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock); + return this._baseDatabase.saveOrUpdateState(repo, state); } - async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind); + await this._baseDatabase.removeStates(repo, blockNumber, kind); } - async getIPLDStatus (): Promise { - const repo = this._conn.getRepository(IpldStatus); + async removeStatesAfterBlock (dbTx: QueryRunner, blockNumber: number): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.getIPLDStatus(repo); + await this._baseDatabase.removeStatesAfterBlock(repo, blockNumber); } - async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async getStateSyncStatus (): Promise { + const repo = this._conn.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force); + return this._baseDatabase.getStateSyncStatus(repo); } - async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force); } - async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force); } async getContracts (): Promise { diff --git a/packages/graph-test-watcher/src/entity/IPLDBlock.ts b/packages/erc721-watcher/src/entity/State.ts similarity index 96% rename from packages/graph-test-watcher/src/entity/IPLDBlock.ts rename to packages/erc721-watcher/src/entity/State.ts index ddcf2660..10c1bf63 100644 --- a/packages/graph-test-watcher/src/entity/IPLDBlock.ts +++ b/packages/erc721-watcher/src/entity/State.ts @@ -12,7 +12,7 @@ import { BlockProgress } from './BlockProgress'; @Index(['cid'], { unique: true }) @Index(['block', 'contractAddress']) @Index(['block', 'contractAddress', 'kind'], { unique: true }) -export class IPLDBlock { +export class State { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/mobymask-watcher/src/entity/IpldStatus.ts b/packages/erc721-watcher/src/entity/StateSyncStatus.ts similarity index 62% rename from packages/mobymask-watcher/src/entity/IpldStatus.ts rename to packages/erc721-watcher/src/entity/StateSyncStatus.ts index d99eebf0..fae389bc 100644 --- a/packages/mobymask-watcher/src/entity/IpldStatus.ts +++ b/packages/erc721-watcher/src/entity/StateSyncStatus.ts @@ -5,16 +5,13 @@ import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; @Entity() -export class IpldStatus { +export class StateSyncStatus { @PrimaryGeneratedColumn() id!: number; @Column('integer') - latestHooksBlockNumber!: number; + latestIndexedBlockNumber!: number; - @Column('integer') + @Column('integer', { nullable: true }) latestCheckpointBlockNumber!: number; - - @Column('integer') - latestIPFSBlockNumber!: number; } diff --git a/packages/erc721-watcher/src/hooks.ts b/packages/erc721-watcher/src/hooks.ts index a4091cb1..a8a03e92 100644 --- a/packages/erc721-watcher/src/hooks.ts +++ b/packages/erc721-watcher/src/hooks.ts @@ -21,19 +21,19 @@ export async function createInitialState (indexer: Indexer, contractAddress: str assert(blockHash); assert(contractAddress); - // Store the desired initial state in an IPLDBlock. - const ipldBlockData: any = { + // Store an empty State. + const stateData: any = { state: {} }; // Use updateStateForElementaryType to update initial state with an elementary property. - // Eg. const ipldBlockData = updateStateForElementaryType(ipldBlockData, '_totalBalance', result.value.toString()); + // Eg. const stateData = updateStateForElementaryType(stateData, '_totalBalance', result.value.toString()); // Use updateStateForMappingType to update initial state with a nested property. - // Eg. const ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString()); + // Eg. const stateData = updateStateForMappingType(stateData, '_allowances', [owner, spender], allowance.value.toString()); // Return initial state data to be saved. - return ipldBlockData; + return stateData; } /** diff --git a/packages/erc721-watcher/src/indexer.ts b/packages/erc721-watcher/src/indexer.ts index 6bc45666..f73c7287 100644 --- a/packages/erc721-watcher/src/indexer.ts +++ b/packages/erc721-watcher/src/indexer.ts @@ -10,7 +10,6 @@ import { ethers } from 'ethers'; import { JsonFragment } from '@ethersproject/abi'; import { BaseProvider } from '@ethersproject/providers'; -import * as codec from '@ipld/dag-cbor'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; import { @@ -25,10 +24,8 @@ import { updateStateForElementaryType, updateStateForMappingType, BlockHeight, - IPFSClient, StateKind, - IpldStatus as IpldStatusInterface, - ResultIPLDBlock + StateStatus } from '@cerc-io/util'; import ERC721Artifacts from './artifacts/ERC721.json'; @@ -37,9 +34,9 @@ import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import { TransferCount } from './entity/TransferCount'; const log = debug('vulcanize:indexer'); @@ -82,8 +79,6 @@ export class Indexer implements IndexerInterface { _storageLayoutMap: Map _contractMap: Map - _ipfsClient: IPFSClient - constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue) { assert(db); assert(ethClient); @@ -92,8 +87,7 @@ export class Indexer implements IndexerInterface { this._ethClient = ethClient; this._ethProvider = ethProvider; this._serverConfig = serverConfig; - this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); - this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient); + this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); this._abiMap = new Map(); this._storageLayoutMap = new Map(); @@ -121,7 +115,7 @@ export class Indexer implements IndexerInterface { async init (): Promise { await this._baseIndexer.fetchContracts(); - await this._baseIndexer.fetchIPLDStatus(); + await this._baseIndexer.fetchStateStatus(); } getResultEvent (event: Event): ResultEvent { @@ -159,26 +153,6 @@ export class Indexer implements IndexerInterface { }; } - getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock { - const block = ipldBlock.block; - - const data = codec.decode(Buffer.from(ipldBlock.data)) as any; - - return { - block: { - cid: block.cid, - hash: block.blockHash, - number: block.blockNumber, - timestamp: block.blockTimestamp, - parentHash: block.parentHash - }, - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, - data: JSON.stringify(data) - }; - } - async supportsInterface (blockHash: string, contractAddress: string, interfaceId: string): Promise { const entity = await this._db.getSupportsInterface({ blockHash, contractAddress, interfaceId }); if (entity) { @@ -674,10 +648,6 @@ export class Indexer implements IndexerInterface { ); } - async pushToIPFS (data: any): Promise { - await this._baseIndexer.pushToIPFS(data); - } - async processInitialState (contractAddress: string, blockHash: string): Promise { // Call initial state hook. return createInitialState(this, contractAddress, blockHash); @@ -708,28 +678,24 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + return this._db.getPrevState(blockHash, contractAddress, kind); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + return this._db.getLatestState(contractAddress, kind, blockNumber); } - async getIPLDBlocksByHash (blockHash: string): Promise { - return this._baseIndexer.getIPLDBlocksByHash(blockHash); + async getStatesByHash (blockHash: string): Promise { + return this._baseIndexer.getStatesByHash(blockHash); } - async getIPLDBlockByCid (cid: string): Promise { - return this._baseIndexer.getIPLDBlockByCid(cid); + async getStateByCID (cid: string): Promise { + return this._baseIndexer.getStateByCID(cid); } - getIPLDData (ipldBlock: IPLDBlock): any { - return this._baseIndexer.getIPLDData(ipldBlock); - } - - isIPFSConfigured (): boolean { - return this._baseIndexer.isIPFSConfigured(); + getStateData (state: State): any { + return this._baseIndexer.getStateData(state); } // Method used to create auto diffs (diff_staged). @@ -761,12 +727,12 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.createCheckpoint(this, contractAddress, block); } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock); + async saveOrUpdateState (state: State): Promise { + return this._baseIndexer.saveOrUpdateState(state); } - async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise { - await this._baseIndexer.removeIPLDBlocks(blockNumber, kind); + async removeStates (blockNumber: number, kind: StateKind): Promise { + await this._baseIndexer.removeStates(blockNumber, kind); } async triggerIndexingOnEvent (event: Event): Promise { @@ -803,16 +769,16 @@ export class Indexer implements IndexerInterface { }; } - async getIPLDStatus (): Promise { - return this._db.getIPLDStatus(); + async getStateSyncStatus (): Promise { + return this._db.getStateSyncStatus(); } - async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -824,29 +790,12 @@ export class Indexer implements IndexerInterface { return res; } - async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force); - await dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - - return res; - } - - async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise { - const dbTx = await this._db.createTransactionRunner(); - let res; - - try { - res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -868,16 +817,16 @@ export class Indexer implements IndexerInterface { return latestCanonicalBlock; } - async getLatestHooksProcessedBlock (): Promise { - return this._baseIndexer.getLatestHooksProcessedBlock(); + async getLatestStateIndexedBlock (): Promise { + return this._baseIndexer.getLatestStateIndexedBlock(); } async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { - await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + this._baseIndexer.updateStateStatusMap(address, stateStatus); } cacheContract (contract: Contract): void { diff --git a/packages/erc721-watcher/src/job-runner.ts b/packages/erc721-watcher/src/job-runner.ts index 42c5c6ed..2fc66644 100644 --- a/packages/erc721-watcher/src/job-runner.ts +++ b/packages/erc721-watcher/src/job-runner.ts @@ -17,8 +17,6 @@ import { QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, QUEUE_HOOKS, - QUEUE_IPFS, - JOB_KIND_PRUNE, JobQueueConfig, DEFAULT_CONFIG_PATH, initClients, @@ -48,22 +46,12 @@ export class JobRunner { await this.subscribeEventProcessingQueue(); await this.subscribeBlockCheckpointQueue(); await this.subscribeHooksQueue(); - await this.subscribeIPFSQueue(); this._baseJobRunner.handleShutdown(); } async subscribeBlockProcessingQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseJobRunner.processBlock(job); - - const { data: { kind } } = job; - - // If it's a pruning job: Create a hooks job. - if (kind === JOB_KIND_PRUNE) { - await this.createHooksJob(); - } - - await this._jobQueue.markComplete(job); }); } @@ -75,165 +63,15 @@ export class JobRunner { async subscribeHooksQueue (): Promise { await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) { - // Create hooks job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) { - log(`Hooks for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process the hooks for the given block number. - await this._indexer.processCanonicalBlock(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusHooksBlock(blockNumber); - - // Create a checkpoint job after completion of a hook job. - await this.createCheckpointJob(blockHash, blockNumber); - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processHooks(job); }); } async subscribeBlockCheckpointQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestCheckpointBlockNumber >= 0) { - if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) { - // Create a checkpoint job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) { - log(`Checkpoints for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process checkpoints for the given block. - await this._indexer.processCheckpoint(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber); - - // Create an IPFS job after completion of a checkpoint job. - if (this._indexer.isIPFSConfigured()) { - await this.createIPFSPutJob(blockHash, blockNumber); - } - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processCheckpoint(job); }); } - - async subscribeIPFSQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestIPFSBlockNumber >= 0) { - if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) { - // Create a IPFS job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) { - log(`IPFS for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Get IPLDBlocks for the given blocHash. - const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash); - - // Push all the IPLDBlocks to IPFS. - for (const ipldBlock of ipldBlocks) { - const data = this._indexer.getIPLDData(ipldBlock); - await this._indexer.pushToIPFS(data); - } - - // Update the IPLD status. - await this._indexer.updateIPLDStatusIPFSBlock(blockNumber); - - await this._jobQueue.markComplete(job); - }); - } - - async createHooksJob (blockHash?: string, blockNumber?: number): Promise { - if (!blockNumber || !blockHash) { - // Get the latest canonical block - const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock(); - - // Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block. - blockHash = latestCanonicalBlock.parentHash; - blockNumber = latestCanonicalBlock.blockNumber - 1; - } - - await this._jobQueue.pushJob( - QUEUE_HOOKS, - { - blockHash, - blockNumber - } - ); - } - - async createCheckpointJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_BLOCK_CHECKPOINT, - { - blockHash, - blockNumber - } - ); - } - - async createIPFSPutJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_IPFS, - { - blockHash, - blockNumber - } - ); - } } export const main = async (): Promise => { diff --git a/packages/erc721-watcher/src/resolvers.ts b/packages/erc721-watcher/src/resolvers.ts index 3f430641..1dca46ac 100644 --- a/packages/erc721-watcher/src/resolvers.ts +++ b/packages/erc721-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight } from '@cerc-io/util'; +import { ValueResult, BlockHeight, getResultState } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -161,17 +161,17 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getStateByCID: async (_: any, { cid }: { cid: string }) => { log('getStateByCID', cid); - const ipldBlock = await indexer.getIPLDBlockByCid(cid); + const state = await indexer.getStateByCID(cid); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); - const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); + const state = await indexer.getPrevState(blockHash, contractAddress, kind); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getSyncStatus: async () => { diff --git a/packages/erc721-watcher/src/schema.gql b/packages/erc721-watcher/src/schema.gql index dd358945..7b15b148 100644 --- a/packages/erc721-watcher/src/schema.gql +++ b/packages/erc721-watcher/src/schema.gql @@ -77,7 +77,7 @@ type TransferEvent { tokenId: BigInt! } -type ResultIPLDBlock { +type ResultState { block: _Block_! contractAddress: String! cid: String! @@ -109,8 +109,8 @@ type Query { _balances(blockHash: String!, contractAddress: String!, key0: String!): ResultBigInt! _tokenApprovals(blockHash: String!, contractAddress: String!, key0: BigInt!): ResultString! _operatorApprovals(blockHash: String!, contractAddress: String!, key0: String!, key1: String!): ResultBoolean! - getStateByCID(cid: String!): ResultIPLDBlock - getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock + getStateByCID(cid: String!): ResultState + getState(blockHash: String!, contractAddress: String!, kind: String): ResultState getSyncStatus: SyncStatus transferCount(id: String!, block: Block_height): TransferCount! } diff --git a/packages/graph-node/README.md b/packages/graph-node/README.md index a28b2487..a6d73347 100644 --- a/packages/graph-node/README.md +++ b/packages/graph-node/README.md @@ -69,13 +69,13 @@ ``` The queries will be fired if the corresponding entities are updated. - + * Run the CLI: ```bash ./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10 ``` - + * For comparing entities after fetching updated entity ids from watcher database: * Set the watcher config file path and entities directory. @@ -90,13 +90,13 @@ [queries.names] author = "Author" blog = "Blog" - + [watcher] configPath = "../../graph-test-watcher/environments/local.toml" entitiesDir = "../../graph-test-watcher/dist/entity/*" ``` - - * To verify diff IPLD state generated at each block, set the watcher endpoint and `verifyState` flag to true + + * To verify `diff` State state generated at each block, set the watcher endpoint and `verifyState` flag to true ```toml [watcher] @@ -105,7 +105,7 @@ endpoint = "gqlEndpoint2" verifyState = true ``` - + * Run the CLI with `fetch-ids` flag set to true:\ ```bash diff --git a/packages/graph-node/src/cli/compare/compare-blocks.ts b/packages/graph-node/src/cli/compare/compare-blocks.ts index c6b6a767..efd88867 100644 --- a/packages/graph-node/src/cli/compare/compare-blocks.ts +++ b/packages/graph-node/src/cli/compare/compare-blocks.ts @@ -12,7 +12,17 @@ import _ from 'lodash'; import { getConfig as getWatcherConfig, wait } from '@cerc-io/util'; import { GraphQLClient } from '@cerc-io/ipld-eth-client'; -import { checkGQLEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig, checkGQLEntitiesInIPLDState } from './utils'; +import { + checkGQLEntityInState, + compareQuery, + Config, + getStatesByBlock, + checkStateMetaData, + combineState, + getClients, + getConfig, + checkGQLEntitiesInState +} from './utils'; import { Database } from '../../database'; import { getSubgraphConfig } from '../../utils'; @@ -117,8 +127,14 @@ export const main = async (): Promise => { await db.init(); if (config.watcher.verifyState) { - const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath); - subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address); + // Use provided contracts if available; else read from subraph config. + if (config.watcher.contracts) { + subgraphContracts = config.watcher.contracts; + } else { + const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath); + subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address); + } + const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string; subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint }); } @@ -134,7 +150,7 @@ export const main = async (): Promise => { const block = { number: blockNumber }; const updatedEntityIds: { [entityName: string]: string[] } = {}; const updatedEntities: Set = new Set(); - let ipldStateByBlock = {}; + let stateByBlock = {}; assert(db); console.time(`time:compare-block-${blockNumber}`); @@ -166,18 +182,18 @@ export const main = async (): Promise => { assert(db); const [block] = await db.getBlocksAtHeight(blockNumber, false); assert(subgraphGQLClient); - const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash); + const contractStatesByBlock = await getStatesByBlock(subgraphGQLClient, subgraphContracts, block.blockHash); - // Check meta data for each IPLD block found - contractIPLDsByBlock.flat().forEach(contractIPLD => { - const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson); - if (ipldMetaDataDiff) { - log('Results mismatch for IPLD meta data:', ipldMetaDataDiff); + // Check meta data for each State entry found + contractStatesByBlock.flat().forEach(contractStateEntry => { + const stateMetaDataDiff = checkStateMetaData(contractStateEntry, contractLatestStateCIDMap, rawJson); + if (stateMetaDataDiff) { + log('Results mismatch for State meta data:', stateMetaDataDiff); diffFound = true; } }); - ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat()); + stateByBlock = combineState(contractStatesByBlock.flat()); } await blockDelay; @@ -205,10 +221,10 @@ export const main = async (): Promise => { ); if (config.watcher.verifyState) { - const ipldDiff = await checkGQLEntityInIPLDState(ipldStateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields); + const stateDiff = await checkGQLEntityInState(stateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields); - if (ipldDiff) { - log('Results mismatch for IPLD state:', ipldDiff); + if (stateDiff) { + log('Results mismatch for State:', stateDiff); diffFound = true; } } @@ -236,10 +252,10 @@ export const main = async (): Promise => { )); if (config.watcher.verifyState) { - const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields); + const stateDiff = await checkGQLEntitiesInState(stateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields); - if (ipldDiff) { - log('Results mismatch for IPLD state:', ipldDiff); + if (stateDiff) { + log('Results mismatch for State:', stateDiff); diffFound = true; } } diff --git a/packages/graph-node/src/cli/compare/utils.ts b/packages/graph-node/src/cli/compare/utils.ts index d6aecd1a..c617fffc 100644 --- a/packages/graph-node/src/cli/compare/utils.ts +++ b/packages/graph-node/src/cli/compare/utils.ts @@ -21,7 +21,7 @@ import { DEFAULT_LIMIT } from '../../database'; const log = debug('vulcanize:compare-utils'); -const IPLD_STATE_QUERY = ` +const STATE_QUERY = ` query getState($blockHash: String!, $contractAddress: String!, $kind: String){ getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){ block { @@ -63,7 +63,8 @@ export interface Config { entitiesDir: string; verifyState: boolean; endpoint: keyof EndpointConfig; - skipFields: EntitySkipFields[] + skipFields: EntitySkipFields[]; + contracts: string[]; } cache: { endpoint: keyof EndpointConfig; @@ -169,24 +170,24 @@ export const getClients = async (config: Config, timeDiff: boolean, queryDir?: s }; }; -export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => { - // Fetch IPLD states for all contracts +export const getStatesByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => { + // Fetch States for all contracts return Promise.all(contracts.map(async contract => { const { getState } = await client.query( - gql(IPLD_STATE_QUERY), + gql(STATE_QUERY), { blockHash, contractAddress: contract } ); - const stateIPLDs = []; + const states = []; // If 'checkpoint' is found at the same block, fetch 'diff' as well if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) { // Check if 'init' present at the same block const { getState: getInitState } = await client.query( - gql(IPLD_STATE_QUERY), + gql(STATE_QUERY), { blockHash, contractAddress: contract, @@ -195,13 +196,13 @@ export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[] ); if (getInitState && getInitState.block.hash === blockHash) { - // Append the 'init' IPLD block to the result - stateIPLDs.push(getInitState); + // Append the 'init' state to the result + states.push(getInitState); } - // Check if 'diff' present at the same block + // Check if 'diff' state present at the same block const { getState: getDiffState } = await client.query( - gql(IPLD_STATE_QUERY), + gql(STATE_QUERY), { blockHash, contractAddress: contract, @@ -210,25 +211,25 @@ export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[] ); if (getDiffState && getDiffState.block.hash === blockHash) { - // Append the 'diff' IPLD block to the result - stateIPLDs.push(getDiffState); + // Append the 'diff' state to the result + states.push(getDiffState); } } - // Append the IPLD block to the result - stateIPLDs.push(getState); + // Append the state to the result + states.push(getState); - return stateIPLDs; + return states; })); }; -export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map, rawJson: boolean) => { - // Return if IPLD for a contract not found - if (!contractIPLD) { +export const checkStateMetaData = (contractState: {[key: string]: any}, contractLatestStateCIDMap: Map, rawJson: boolean) => { + // Return if State for a contract not found + if (!contractState) { return; } - const { contractAddress, cid, kind, block } = contractIPLD; + const { contractAddress, cid, kind, block } = contractState; const parentCIDs = contractLatestStateCIDMap.get(contractAddress); assert(parentCIDs); @@ -246,7 +247,7 @@ export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLa contractLatestStateCIDMap.set(contractAddress, nextParentCIDs); // Actual meta data from the GQL result - const data = JSON.parse(contractIPLD.data); + const data = JSON.parse(contractState.data); // If parentCID not initialized (is empty at start) // Take the expected parentCID from the actual data itself @@ -279,13 +280,13 @@ export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLa return compareObjects(expectedMetaData, data.meta, rawJson); }; -export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => { - const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => { - if (!contractIPLD) { +export const combineState = (contractStateEntries: {[key: string]: any}[]): {[key: string]: any} => { + const contractStates: {[key: string]: any}[] = contractStateEntries.map(contractStateEntry => { + if (!contractStateEntry) { return {}; } - const data = JSON.parse(contractIPLD.data); + const data = JSON.parse(contractStateEntry.data); // Apply default limit and sort by id on array type relation fields. Object.values(data.state) @@ -309,18 +310,18 @@ export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: return data.state; }); - return contractIPLDStates.reduce((acc, state) => _.merge(acc, state)); + return contractStates.reduce((acc, state) => _.merge(acc, state)); }; -export const checkGQLEntityInIPLDState = async ( - ipldState: {[key: string]: any}, +export const checkGQLEntityInState = async ( + state: {[key: string]: any}, entityName: string, entityResult: {[key: string]: any}, id: string, rawJson: boolean, skipFields: EntitySkipFields[] = [] ): Promise => { - const ipldEntity = ipldState[entityName][id]; + const stateEntity = state[entityName][id]; // Filter __typename key in GQL result. entityResult = omitDeep(entityResult, '__typename'); @@ -329,24 +330,24 @@ export const checkGQLEntityInIPLDState = async ( skipFields.forEach(({ entity, fields }) => { if (entityName === entity) { omitDeep(entityResult, fields); - omitDeep(ipldEntity, fields); + omitDeep(stateEntity, fields); } }); - const diff = compareObjects(entityResult, ipldEntity, rawJson); + const diff = compareObjects(entityResult, stateEntity, rawJson); return diff; }; -export const checkGQLEntitiesInIPLDState = async ( - ipldState: {[key: string]: any}, +export const checkGQLEntitiesInState = async ( + state: {[key: string]: any}, entityName: string, entitiesResult: any[], rawJson: boolean, skipFields: EntitySkipFields[] = [] ): Promise => { // Form entities from state to compare with GQL result - const stateEntities = ipldState[entityName]; + const stateEntities = state[entityName]; for (const entityResult of entitiesResult) { const stateEntity = stateEntities[entityResult.id]; diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index 1fcc7fd0..4a9ac12c 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -31,8 +31,6 @@ import { Block, fromEntityValue, fromStateEntityValues, toEntityValue } from './ export const DEFAULT_LIMIT = 100; -const log = debug('vulcanize:graph-node-database'); - interface CachedEntities { frothyBlocks: Map< string, @@ -641,7 +639,7 @@ export class Database { }, {}); } - fromIPLDState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any { + fromState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any { const repo = this._conn.getRepository(entity); const entityFields = repo.metadata.columns; @@ -678,7 +676,7 @@ export class Database { }, {}); } - cacheUpdatedEntity (entityName: string, entity: any, pruned = false): void { + cacheUpdatedEntity (entityName: string, entity: any, pruned = false): void { const repo = this._conn.getRepository(entityName); const tableName = repo.metadata.tableName; diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index 10e67093..af5be5f7 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -12,7 +12,7 @@ import { SelectionNode } from 'graphql'; import { ResultObject } from '@vulcanize/assemblyscript/lib/loader'; import { EthClient } from '@cerc-io/ipld-eth-client'; -import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util'; +import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, StateInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util'; import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils'; import { Context, GraphData, instantiate } from './loader'; @@ -349,9 +349,9 @@ export class GraphWatcher { } } - async updateEntitiesFromIPLDState (ipldBlock: IPLDBlockInterface) { + async updateEntitiesFromState (state: StateInterface) { assert(this._indexer); - const data = this._indexer.getIPLDData(ipldBlock); + const data = this._indexer.getStateData(state); for (const [entityName, entities] of Object.entries(data.state)) { // Get relations for subgraph entity @@ -363,13 +363,13 @@ export class GraphWatcher { const relations = result ? result[1] : {}; - log(`Updating entities from IPLD state for entity ${entityName}`); - console.time(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`); + log(`Updating entities from State for entity ${entityName}`); + console.time(`time:watcher#GraphWatcher-updateEntitiesFromState-update-entity-${entityName}`); for (const [id, entityData] of Object.entries(entities as any)) { - const dbData = this._database.fromIPLDState(ipldBlock.block, entityName, entityData, relations); + const dbData = this._database.fromState(state.block, entityName, entityData, relations); await this._database.saveEntity(entityName, dbData); } - console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`); + console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromState-update-entity-${entityName}`); } } diff --git a/packages/graph-node/test/utils/indexer.ts b/packages/graph-node/test/utils/indexer.ts index 589d5f11..82dba32a 100644 --- a/packages/graph-node/test/utils/indexer.ts +++ b/packages/graph-node/test/utils/indexer.ts @@ -9,8 +9,9 @@ import { ServerConfig as ServerConfigInterface, ValueResult, ContractInterface, - IpldStatus as IpldStatusInterface, - IPLDBlockInterface + StateStatus, + StateSyncStatusInterface, + StateInterface } from '@cerc-io/util'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { GetStorageAt, getStorageValue, MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; @@ -107,7 +108,7 @@ export class Indexer implements IndexerInterface { assert(blockHash); assert(blockNumber); - return new SyncStatus(); + return {} as SyncStatusInterface; } async updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number, force?: boolean): Promise { @@ -115,7 +116,7 @@ export class Indexer implements IndexerInterface { assert(blockHash); assert(force); - return new SyncStatus(); + return {} as SyncStatusInterface; } async updateSyncStatusCanonicalBlock (blockHash: string, blockNumber: number, force?: boolean): Promise { @@ -123,7 +124,7 @@ export class Indexer implements IndexerInterface { assert(blockHash); assert(force); - return new SyncStatus(); + return {} as SyncStatusInterface; } async markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise { @@ -157,6 +158,22 @@ export class Indexer implements IndexerInterface { assert(event); } + async getStateSyncStatus (): Promise { + return undefined; + } + + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { + return {} as StateSyncStatusInterface; + } + + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + return {} as StateSyncStatusInterface; + } + + async getLatestCanonicalBlock (): Promise { + return {} as BlockProgressInterface; + } + isWatchedContract (address : string): ContractInterface | undefined { return undefined; } @@ -165,36 +182,20 @@ export class Indexer implements IndexerInterface { return undefined; } - getIPLDData (ipldBlock: IPLDBlockInterface): any { + async processCanonicalBlock (blockHash: string, blockNumber: number): Promise { return undefined; } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { + async processCheckpoint (blockHash: string): Promise { return undefined; } -} -class SyncStatus implements SyncStatusInterface { - id: number; - chainHeadBlockHash: string; - chainHeadBlockNumber: number; - latestIndexedBlockHash: string; - latestIndexedBlockNumber: number; - latestCanonicalBlockHash: string; - latestCanonicalBlockNumber: number; - initialIndexedBlockHash: string; - initialIndexedBlockNumber: number; + getStateData (state: StateInterface): any { + return undefined; + } - constructor () { - this.id = 0; - this.chainHeadBlockHash = '0'; - this.chainHeadBlockNumber = 0; - this.latestIndexedBlockHash = '0'; - this.latestIndexedBlockNumber = 0; - this.latestCanonicalBlockHash = '0'; - this.latestCanonicalBlockNumber = 0; - this.initialIndexedBlockHash = '0'; - this.initialIndexedBlockNumber = 0; + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + return undefined; } } @@ -205,7 +206,6 @@ class ServerConfig implements ServerConfigInterface { kind: string; checkpointing: boolean; checkpointInterval: number; - ipfsApiAddr: string; subgraphPath: string; disableSubgraphState: boolean; wasmRestartBlocksInterval: number; @@ -220,7 +220,6 @@ class ServerConfig implements ServerConfigInterface { this.kind = ''; this.checkpointing = false; this.checkpointInterval = 0; - this.ipfsApiAddr = ''; this.subgraphPath = ''; this.disableSubgraphState = false; this.wasmRestartBlocksInterval = 0; diff --git a/packages/graph-test-watcher/README.md b/packages/graph-test-watcher/README.md index b7fe4406..eef1f12c 100644 --- a/packages/graph-test-watcher/README.md +++ b/packages/graph-test-watcher/README.md @@ -8,12 +8,6 @@ yarn ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - ``` - * Create a postgres12 database for the watcher: ```bash @@ -47,7 +41,7 @@ * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint. - * Update the `server` config with state checkpoint settings and provide the IPFS API address. + * Update the `server` config with state checkpoint settings. ## Customize @@ -59,11 +53,11 @@ * Generating state: - * Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `State` using the `Indexer` object. - * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated. - * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object. * The existing example hooks in [hooks.ts](./src/hooks.ts) are for an `ERC20` contract. @@ -138,10 +132,10 @@ GQL console: http://localhost:3008/graphql * To reset the watcher to a previous block number: - * Reset state: + * Reset watcher: ```bash - yarn reset state --block-number + yarn reset watcher --block-number ``` * Reset job-queue: diff --git a/packages/graph-test-watcher/environments/local.toml b/packages/graph-test-watcher/environments/local.toml index 66942412..24cba88e 100644 --- a/packages/graph-test-watcher/environments/local.toml +++ b/packages/graph-test-watcher/environments/local.toml @@ -9,9 +9,6 @@ # Checkpoint interval in number of blocks. checkpointInterval = 2000 - # IPFS API address (can be taken from the output on running the IPFS daemon). - ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" - subgraphPath = "../graph-node/test/subgraph/example1/build" wasmRestartBlocksInterval = 20 diff --git a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts index 9fd9b815..27960003 100644 --- a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts +++ b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts @@ -54,12 +54,12 @@ export const handler = async (argv: any): Promise => { graphWatcher.setIndexer(indexer); await graphWatcher.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); - const data = indexer.getIPLDData(ipldBlock); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); + const data = indexer.getStateData(state); - log(`Verifying checkpoint data for contract ${ipldBlock.contractAddress}`); - await verifyCheckpointData(graphDb, ipldBlock.block, data); + log(`Verifying checkpoint data for contract ${state.contractAddress}`); + await verifyCheckpointData(graphDb, state.block, data); log('Checkpoint data verified'); await db.close(); diff --git a/packages/graph-test-watcher/src/cli/export-state.ts b/packages/graph-test-watcher/src/cli/export-state.ts index e772e9b1..ec4f4281 100644 --- a/packages/graph-test-watcher/src/cli/export-state.ts +++ b/packages/graph-test-watcher/src/cli/export-state.ts @@ -70,16 +70,16 @@ const main = async (): Promise => { const exportData: any = { snapshotBlock: {}, contracts: [], - ipldCheckpoints: [] + stateCheckpoints: [] }; const contracts = await db.getContracts(); - let block = await indexer.getLatestHooksProcessedBlock(); + let block = await indexer.getLatestStateIndexedBlock(); assert(block); if (argv.blockNumber) { if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`); + throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); } const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); @@ -116,19 +116,15 @@ const main = async (): Promise => { if (contract.checkpoint) { await indexer.createCheckpoint(contract.address, block.blockHash); - const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(ipldBlock); + const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); + assert(state); - const data = indexer.getIPLDData(ipldBlock); + const data = indexer.getStateData(state); - if (indexer.isIPFSConfigured()) { - await indexer.pushToIPFS(data); - } - - exportData.ipldCheckpoints.push({ - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, + exportData.stateCheckpoints.push({ + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, data }); } diff --git a/packages/graph-test-watcher/src/cli/import-state.ts b/packages/graph-test-watcher/src/cli/import-state.ts index 6d8751e0..ebb96f44 100644 --- a/packages/graph-test-watcher/src/cli/import-state.ts +++ b/packages/graph-test-watcher/src/cli/import-state.ts @@ -18,7 +18,7 @@ import * as codec from '@ipld/dag-cbor'; import { Database } from '../database'; import { Indexer } from '../indexer'; import { EventWatcher } from '../events'; -import { IPLDBlock } from '../entity/IPLDBlock'; +import { State } from '../entity/State'; const log = debug('vulcanize:import-state'); @@ -100,17 +100,17 @@ export const main = async (): Promise => { const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash); assert(block); - // Fill the IPLDBlocks. - for (const checkpoint of importData.ipldCheckpoints) { - let ipldBlock = new IPLDBlock(); + // Fill the States. + for (const checkpoint of importData.stateCheckpoints) { + let state = new State(); - ipldBlock = Object.assign(ipldBlock, checkpoint); - ipldBlock.block = block; + state = Object.assign(state, checkpoint); + state.block = block; - ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data)); + state.data = Buffer.from(codec.encode(state.data)); - ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock); - await graphWatcher.updateEntitiesFromIPLDState(ipldBlock); + state = await indexer.saveOrUpdateState(state); + await graphWatcher.updateEntitiesFromState(state); } // Mark snapshot block as completely processed. @@ -118,12 +118,12 @@ export const main = async (): Promise => { await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber); await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber); - await indexer.updateIPLDStatusHooksBlock(block.blockNumber); - await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber); + await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber); + await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber); - // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block. - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init); - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged); + // The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block. + await indexer.removeStates(block.blockNumber, StateKind.Init); + await indexer.removeStates(block.blockNumber, StateKind.DiffStaged); log(`Import completed for snapshot block at height ${block.blockNumber}`); }; diff --git a/packages/graph-test-watcher/src/cli/inspect-cid.ts b/packages/graph-test-watcher/src/cli/inspect-cid.ts index fa4de97d..24e57bae 100644 --- a/packages/graph-test-watcher/src/cli/inspect-cid.ts +++ b/packages/graph-test-watcher/src/cli/inspect-cid.ts @@ -63,12 +63,12 @@ const main = async (): Promise => { graphWatcher.setIndexer(indexer); await graphWatcher.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); - const ipldData = await indexer.getIPLDData(ipldBlock); + const stateData = await indexer.getStateData(state); - log(util.inspect(ipldData, false, null)); + log(util.inspect(stateData, false, null)); }; main().catch(err => { diff --git a/packages/graph-test-watcher/src/cli/reset-cmds/state.ts b/packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts similarity index 81% rename from packages/graph-test-watcher/src/cli/reset-cmds/state.ts rename to packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts index 8b876a0b..4206d38f 100644 --- a/packages/graph-test-watcher/src/cli/reset-cmds/state.ts +++ b/packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts @@ -20,11 +20,11 @@ import { Author } from '../../entity/Author'; import { Blog } from '../../entity/Blog'; import { Category } from '../../entity/Category'; -const log = debug('vulcanize:reset-state'); +const log = debug('vulcanize:reset-watcher'); -export const command = 'state'; +export const command = 'watcher'; -export const desc = 'Reset state to block number'; +export const desc = 'Reset watcher to a block number'; export const builder = { blockNumber: { @@ -86,18 +86,15 @@ export const handler = async (argv: any): Promise => { await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); } - const ipldStatus = await indexer.getIPLDStatus(); - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true); + const stateSyncStatus = await indexer.getStateSyncStatus(); + + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true); } - if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true); + if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true); } } @@ -111,5 +108,5 @@ export const handler = async (argv: any): Promise => { await dbTx.release(); } - log('Reset state successfully'); + log('Reset watcher successfully'); }; diff --git a/packages/graph-test-watcher/src/database.ts b/packages/graph-test-watcher/src/database.ts index d1fd69ff..3e6cdb44 100644 --- a/packages/graph-test-watcher/src/database.ts +++ b/packages/graph-test-watcher/src/database.ts @@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import { GetMethod } from './entity/GetMethod'; import { _Test } from './entity/_Test'; @@ -73,69 +73,63 @@ export class Database implements DatabaseInterface { return repo.save(entity); } - getNewIPLDBlock (): IPLDBlock { - return new IPLDBlock(); + getNewState (): State { + return new State(); } - async getIPLDBlocks (where: FindConditions): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getStates (where: FindConditions): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getIPLDBlocks(repo, where); + return this._baseDatabase.getStates(repo, where); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber); + return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind); + return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind); } - // Fetch all diff IPLDBlocks after the specified block number. - async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + // Fetch all diff States after the specified block number. + async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock); + return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock); } - async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock); + return this._baseDatabase.saveOrUpdateState(repo, state); } - async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind); + await this._baseDatabase.removeStates(repo, blockNumber, kind); } - async getIPLDStatus (): Promise { - const repo = this._conn.getRepository(IpldStatus); + async getStateSyncStatus (): Promise { + const repo = this._conn.getRepository(StateSyncStatus); - return this._baseDatabase.getIPLDStatus(repo); + return this._baseDatabase.getStateSyncStatus(repo); } - async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force); } - async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force); - } - - async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); - - return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force); } async getContracts (): Promise { diff --git a/packages/erc721-watcher/src/entity/IPLDBlock.ts b/packages/graph-test-watcher/src/entity/State.ts similarity index 89% rename from packages/erc721-watcher/src/entity/IPLDBlock.ts rename to packages/graph-test-watcher/src/entity/State.ts index 0fd42a19..10c1bf63 100644 --- a/packages/erc721-watcher/src/entity/IPLDBlock.ts +++ b/packages/graph-test-watcher/src/entity/State.ts @@ -3,14 +3,16 @@ // import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm'; + import { StateKind } from '@cerc-io/util'; + import { BlockProgress } from './BlockProgress'; @Entity() @Index(['cid'], { unique: true }) @Index(['block', 'contractAddress']) @Index(['block', 'contractAddress', 'kind'], { unique: true }) -export class IPLDBlock { +export class State { @PrimaryGeneratedColumn() id!: number; @@ -23,7 +25,10 @@ export class IPLDBlock { @Column('varchar') cid!: string; - @Column({ type: 'enum', enum: StateKind }) + @Column({ + type: 'enum', + enum: StateKind + }) kind!: StateKind; @Column('bytea') diff --git a/packages/erc721-watcher/src/entity/IpldStatus.ts b/packages/graph-test-watcher/src/entity/StateSyncStatus.ts similarity index 62% rename from packages/erc721-watcher/src/entity/IpldStatus.ts rename to packages/graph-test-watcher/src/entity/StateSyncStatus.ts index d99eebf0..fae389bc 100644 --- a/packages/erc721-watcher/src/entity/IpldStatus.ts +++ b/packages/graph-test-watcher/src/entity/StateSyncStatus.ts @@ -5,16 +5,13 @@ import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; @Entity() -export class IpldStatus { +export class StateSyncStatus { @PrimaryGeneratedColumn() id!: number; @Column('integer') - latestHooksBlockNumber!: number; + latestIndexedBlockNumber!: number; - @Column('integer') + @Column('integer', { nullable: true }) latestCheckpointBlockNumber!: number; - - @Column('integer') - latestIPFSBlockNumber!: number; } diff --git a/packages/graph-test-watcher/src/hooks.ts b/packages/graph-test-watcher/src/hooks.ts index 709c785e..3e61d0a3 100644 --- a/packages/graph-test-watcher/src/hooks.ts +++ b/packages/graph-test-watcher/src/hooks.ts @@ -18,13 +18,13 @@ export async function createInitialState (indexer: Indexer, contractAddress: str assert(blockHash); assert(contractAddress); - // Store an empty state in an IPLDBlock. - const ipldBlockData: any = { + // Store an empty State. + const stateData: any = { state: {} }; // Return initial state data to be saved. - return ipldBlockData; + return stateData; } /** diff --git a/packages/graph-test-watcher/src/indexer.ts b/packages/graph-test-watcher/src/indexer.ts index 8aa94a79..874c6831 100644 --- a/packages/graph-test-watcher/src/indexer.ts +++ b/packages/graph-test-watcher/src/indexer.ts @@ -12,7 +12,6 @@ import { SelectionNode } from 'graphql'; import { JsonFragment } from '@ethersproject/abi'; import { BaseProvider } from '@ethersproject/providers'; -import * as codec from '@ipld/dag-cbor'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { StorageLayout, MappingKey } from '@cerc-io/solidity-mapper'; import { @@ -25,11 +24,9 @@ import { Where, QueryOptions, BlockHeight, - IPFSClient, StateKind, IndexerInterface, - IpldStatus as IpldStatusInterface, - ResultIPLDBlock + StateStatus } from '@cerc-io/util'; import { GraphWatcher } from '@cerc-io/graph-node'; @@ -37,9 +34,9 @@ import { Database } from './database'; import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import Example1Artifacts from './artifacts/Example.json'; import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint } from './hooks'; import { Author } from './entity/Author'; @@ -87,8 +84,6 @@ export class Indexer implements IndexerInterface { _storageLayoutMap: Map _contractMap: Map - _ipfsClient: IPFSClient - _entityTypesMap: Map _relationsMap: Map @@ -102,8 +97,7 @@ export class Indexer implements IndexerInterface { this._ethClient = ethClient; this._ethProvider = ethProvider; this._serverConfig = serverConfig; - this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); - this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient); + this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); this._graphWatcher = graphWatcher; this._abiMap = new Map(); @@ -140,7 +134,7 @@ export class Indexer implements IndexerInterface { async init (): Promise { await this._baseIndexer.fetchContracts(); - await this._baseIndexer.fetchIPLDStatus(); + await this._baseIndexer.fetchStateStatus(); } getResultEvent (event: Event): ResultEvent { @@ -178,26 +172,6 @@ export class Indexer implements IndexerInterface { }; } - getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock { - const block = ipldBlock.block; - - const data = codec.decode(Buffer.from(ipldBlock.data)) as any; - - return { - block: { - cid: block.cid, - hash: block.blockHash, - number: block.blockNumber, - timestamp: block.blockTimestamp, - parentHash: block.parentHash - }, - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, - data: JSON.stringify(data) - }; - } - async getMethod (blockHash: string, contractAddress: string): Promise { const entity = await this._db.getGetMethod({ blockHash, contractAddress }); if (entity) { @@ -273,10 +247,6 @@ export class Indexer implements IndexerInterface { ); } - async pushToIPFS (data: any): Promise { - await this._baseIndexer.pushToIPFS(data); - } - async processInitialState (contractAddress: string, blockHash: string): Promise { // Call initial state hook. return createInitialState(this, contractAddress, blockHash); @@ -309,32 +279,28 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + return this._db.getPrevState(blockHash, contractAddress, kind); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + return this._db.getLatestState(contractAddress, kind, blockNumber); } - async getIPLDBlocksByHash (blockHash: string): Promise { - return this._baseIndexer.getIPLDBlocksByHash(blockHash); + async getStatesByHash (blockHash: string): Promise { + return this._baseIndexer.getStatesByHash(blockHash); } - async getIPLDBlockByCid (cid: string): Promise { - return this._baseIndexer.getIPLDBlockByCid(cid); + async getStateByCID (cid: string): Promise { + return this._baseIndexer.getStateByCID(cid); } - async getIPLDBlocks (where: FindConditions): Promise { - return this._db.getIPLDBlocks(where); + async getStates (where: FindConditions): Promise { + return this._db.getStates(where); } - getIPLDData (ipldBlock: IPLDBlock): any { - return this._baseIndexer.getIPLDData(ipldBlock); - } - - isIPFSConfigured (): boolean { - return this._baseIndexer.isIPFSConfigured(); + getStateData (state: State): any { + return this._baseIndexer.getStateData(state); } // Method used to create auto diffs (diff_staged). @@ -366,12 +332,12 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.createCheckpoint(this, contractAddress, block); } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock); + async saveOrUpdateState (state: State): Promise { + return this._baseIndexer.saveOrUpdateState(state); } - async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise { - await this._baseIndexer.removeIPLDBlocks(blockNumber, kind); + async removeStates (blockNumber: number, kind: StateKind): Promise { + await this._baseIndexer.removeStates(blockNumber, kind); } async getSubgraphEntity ( @@ -432,16 +398,16 @@ export class Indexer implements IndexerInterface { }; } - async getIPLDStatus (): Promise { - return this._db.getIPLDStatus(); + async getStateSyncStatus (): Promise { + return this._db.getStateSyncStatus(); } - async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -453,29 +419,12 @@ export class Indexer implements IndexerInterface { return res; } - async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force); - await dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - - return res; - } - - async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise { - const dbTx = await this._db.createTransactionRunner(); - let res; - - try { - res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -497,16 +446,16 @@ export class Indexer implements IndexerInterface { return latestCanonicalBlock; } - async getLatestHooksProcessedBlock (): Promise { - return this._baseIndexer.getLatestHooksProcessedBlock(); + async getLatestStateIndexedBlock (): Promise { + return this._baseIndexer.getLatestStateIndexedBlock(); } async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { - await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + this._baseIndexer.updateStateStatusMap(address, stateStatus); } cacheContract (contract: Contract): void { diff --git a/packages/graph-test-watcher/src/job-runner.ts b/packages/graph-test-watcher/src/job-runner.ts index 240b3f18..09db8431 100644 --- a/packages/graph-test-watcher/src/job-runner.ts +++ b/packages/graph-test-watcher/src/job-runner.ts @@ -18,8 +18,6 @@ import { QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, QUEUE_HOOKS, - QUEUE_IPFS, - JOB_KIND_PRUNE, JobQueueConfig, DEFAULT_CONFIG_PATH, initClients, @@ -50,22 +48,12 @@ export class JobRunner { await this.subscribeEventProcessingQueue(); await this.subscribeBlockCheckpointQueue(); await this.subscribeHooksQueue(); - await this.subscribeIPFSQueue(); this._baseJobRunner.handleShutdown(); } async subscribeBlockProcessingQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseJobRunner.processBlock(job); - - const { data: { kind } } = job; - - // If it's a pruning job: Create a hooks job. - if (kind === JOB_KIND_PRUNE) { - await this.createHooksJob(); - } - - await this._jobQueue.markComplete(job); }); } @@ -77,165 +65,15 @@ export class JobRunner { async subscribeHooksQueue (): Promise { await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) { - // Create hooks job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) { - log(`Hooks for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process the hooks for the given block number. - await this._indexer.processCanonicalBlock(blockHash, blockNumber); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusHooksBlock(blockNumber); - - // Create a checkpoint job after completion of a hook job. - await this.createCheckpointJob(blockHash, blockNumber); - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processHooks(job); }); } async subscribeBlockCheckpointQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestCheckpointBlockNumber >= 0) { - if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) { - // Create a checkpoint job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) { - log(`Checkpoints for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process checkpoints for the given block. - await this._indexer.processCheckpoint(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber); - - // Create an IPFS job after completion of a checkpoint job. - if (this._indexer.isIPFSConfigured()) { - await this.createIPFSPutJob(blockHash, blockNumber); - } - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processCheckpoint(job); }); } - - async subscribeIPFSQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestIPFSBlockNumber >= 0) { - if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) { - // Create a IPFS job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) { - log(`IPFS for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Get IPLDBlocks for the given blocHash. - const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash); - - // Push all the IPLDBlocks to IPFS. - for (const ipldBlock of ipldBlocks) { - const data = this._indexer.getIPLDData(ipldBlock); - await this._indexer.pushToIPFS(data); - } - - // Update the IPLD status. - await this._indexer.updateIPLDStatusIPFSBlock(blockNumber); - - await this._jobQueue.markComplete(job); - }); - } - - async createHooksJob (blockHash?: string, blockNumber?: number): Promise { - if (!blockNumber || !blockHash) { - // Get the latest canonical block - const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock(); - - // Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block. - blockHash = latestCanonicalBlock.parentHash; - blockNumber = latestCanonicalBlock.blockNumber - 1; - } - - await this._jobQueue.pushJob( - QUEUE_HOOKS, - { - blockHash, - blockNumber - } - ); - } - - async createCheckpointJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_BLOCK_CHECKPOINT, - { - blockHash, - blockNumber - } - ); - } - - async createIPFSPutJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_IPFS, - { - blockHash, - blockNumber - } - ); - } } export const main = async (): Promise => { diff --git a/packages/graph-test-watcher/src/resolvers.ts b/packages/graph-test-watcher/src/resolvers.ts index 0c3ec210..4ea13311 100644 --- a/packages/graph-test-watcher/src/resolvers.ts +++ b/packages/graph-test-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer, getResultState } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -153,9 +153,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getStateByCID').inc(1); - const ipldBlock = await indexer.getIPLDBlockByCid(cid); + const state = await indexer.getStateByCID(cid); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { @@ -163,9 +163,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1); - const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); + const state = await indexer.getPrevState(blockHash, contractAddress, kind); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getSyncStatus: async () => { diff --git a/packages/graph-test-watcher/src/schema.gql b/packages/graph-test-watcher/src/schema.gql index 77649d95..337bd259 100644 --- a/packages/graph-test-watcher/src/schema.gql +++ b/packages/graph-test-watcher/src/schema.gql @@ -65,7 +65,7 @@ type TestEvent { param3: BigInt! } -type ResultIPLDBlock { +type ResultState { block: Block! contractAddress: String! cid: String! @@ -88,8 +88,8 @@ type Query { blog(id: String!, block: Block_height): Blog! author(id: String!, block: Block_height): Author! category(id: String!, block: Block_height): Category! - getStateByCID(cid: String!): ResultIPLDBlock - getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock + getStateByCID(cid: String!): ResultState + getState(blockHash: String!, contractAddress: String!, kind: String): ResultState getSyncStatus: SyncStatus } diff --git a/packages/mobymask-watcher/README.md b/packages/mobymask-watcher/README.md index 04c3a00b..c998fb51 100644 --- a/packages/mobymask-watcher/README.md +++ b/packages/mobymask-watcher/README.md @@ -8,12 +8,6 @@ yarn ``` -* Run the IPFS (go-ipfs version 0.12.2) daemon: - - ```bash - ipfs daemon - ``` - * Create a postgres12 database for the watcher: ```bash @@ -47,7 +41,7 @@ * Update the `upstream` config and provide the `ipld-eth-server` GQL API endpoint. - * Update the `server` config with state checkpoint settings and provide the IPFS API address. + * Update the `server` config with state checkpoint settings. ## Customize @@ -59,11 +53,11 @@ * Generating state: - * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial state `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createInitialState` (triggered if the watcher passes the start block, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial `State` using the `Indexer` object. - * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in a `diff` `State` using the `Indexer` object. The default state (if exists) is updated. - * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `IPLDBlock` using the `Indexer` object. + * Edit the custom hook function `createStateCheckpoint` (triggered just before default and CLI checkpoint) in [hooks.ts](./src/hooks.ts) to save the state in a `checkpoint` `State` using the `Indexer` object. ## Run @@ -130,10 +124,10 @@ GQL console: http://localhost:3010/graphql * To reset the watcher to a previous block number: - * Reset state: + * Reset watcher: ```bash - yarn reset state --block-number + yarn reset watcher --block-number ``` * Reset job-queue: diff --git a/packages/mobymask-watcher/environments/local.toml b/packages/mobymask-watcher/environments/local.toml index 603a0b59..3a4e2910 100644 --- a/packages/mobymask-watcher/environments/local.toml +++ b/packages/mobymask-watcher/environments/local.toml @@ -9,9 +9,6 @@ # Checkpoint interval in number of blocks. checkpointInterval = 2000 - # IPFS API address (can be taken from the output on running the IPFS daemon). - # ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001" - # Boolean to filter logs by contract. filterLogs = true diff --git a/packages/mobymask-watcher/src/cli/export-state.ts b/packages/mobymask-watcher/src/cli/export-state.ts index fe88e1ba..55b80846 100644 --- a/packages/mobymask-watcher/src/cli/export-state.ts +++ b/packages/mobymask-watcher/src/cli/export-state.ts @@ -61,16 +61,16 @@ const main = async (): Promise => { const exportData: any = { snapshotBlock: {}, contracts: [], - ipldCheckpoints: [] + stateCheckpoints: [] }; const contracts = await db.getContracts(); - let block = await indexer.getLatestHooksProcessedBlock(); + let block = await indexer.getLatestStateIndexedBlock(); assert(block); if (argv.blockNumber) { if (argv.blockNumber > block.blockNumber) { - throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest hooks processed block height ${block.blockNumber}`); + throw new Error(`Export snapshot block height ${argv.blockNumber} should be less than latest state indexed block height ${block.blockNumber}`); } const blocksAtSnapshotHeight = await indexer.getBlocksAtHeight(argv.blockNumber, false); @@ -107,19 +107,15 @@ const main = async (): Promise => { if (contract.checkpoint) { await indexer.createCheckpoint(contract.address, block.blockHash); - const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber); - assert(ipldBlock); + const state = await indexer.getLatestState(contract.address, StateKind.Checkpoint, block.blockNumber); + assert(state); - const data = indexer.getIPLDData(ipldBlock); + const data = indexer.getStateData(state); - if (indexer.isIPFSConfigured()) { - await indexer.pushToIPFS(data); - } - - exportData.ipldCheckpoints.push({ - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, + exportData.stateCheckpoints.push({ + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, data }); } diff --git a/packages/mobymask-watcher/src/cli/import-state.ts b/packages/mobymask-watcher/src/cli/import-state.ts index 36073451..250d09bd 100644 --- a/packages/mobymask-watcher/src/cli/import-state.ts +++ b/packages/mobymask-watcher/src/cli/import-state.ts @@ -17,7 +17,7 @@ import * as codec from '@ipld/dag-cbor'; import { Database } from '../database'; import { Indexer } from '../indexer'; import { EventWatcher } from '../events'; -import { IPLDBlock } from '../entity/IPLDBlock'; +import { State } from '../entity/State'; const log = debug('vulcanize:import-state'); @@ -91,16 +91,16 @@ export const main = async (): Promise => { const block = await indexer.getBlockProgress(importData.snapshotBlock.blockHash); assert(block); - // Fill the IPLDBlocks. - for (const checkpoint of importData.ipldCheckpoints) { - let ipldBlock = new IPLDBlock(); + // Fill the States. + for (const checkpoint of importData.stateCheckpoints) { + let state = new State(); - ipldBlock = Object.assign(ipldBlock, checkpoint); - ipldBlock.block = block; + state = Object.assign(state, checkpoint); + state.block = block; - ipldBlock.data = Buffer.from(codec.encode(ipldBlock.data)); + state.data = Buffer.from(codec.encode(state.data)); - ipldBlock = await indexer.saveOrUpdateIPLDBlock(ipldBlock); + state = await indexer.saveOrUpdateState(state); } // Mark snapshot block as completely processed. @@ -108,12 +108,12 @@ export const main = async (): Promise => { await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); await indexer.updateSyncStatusChainHead(block.blockHash, block.blockNumber); await indexer.updateSyncStatusIndexedBlock(block.blockHash, block.blockNumber); - await indexer.updateIPLDStatusHooksBlock(block.blockNumber); - await indexer.updateIPLDStatusCheckpointBlock(block.blockNumber); + await indexer.updateStateSyncStatusIndexedBlock(block.blockNumber); + await indexer.updateStateSyncStatusCheckpointBlock(block.blockNumber); - // The 'diff_staged' and 'init' IPLD blocks are unnecessary as checkpoints have been already created for the snapshot block. - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.Init); - await indexer.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged); + // The 'diff_staged' and 'init' State entries are unnecessary as checkpoints have been already created for the snapshot block. + await indexer.removeStates(block.blockNumber, StateKind.Init); + await indexer.removeStates(block.blockNumber, StateKind.DiffStaged); log(`Import completed for snapshot block at height ${block.blockNumber}`); }; diff --git a/packages/mobymask-watcher/src/cli/inspect-cid.ts b/packages/mobymask-watcher/src/cli/inspect-cid.ts index 2b48fa32..c32ba47c 100644 --- a/packages/mobymask-watcher/src/cli/inspect-cid.ts +++ b/packages/mobymask-watcher/src/cli/inspect-cid.ts @@ -53,12 +53,12 @@ const main = async (): Promise => { const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue); await indexer.init(); - const ipldBlock = await indexer.getIPLDBlockByCid(argv.cid); - assert(ipldBlock, 'IPLDBlock for the provided CID doesn\'t exist.'); + const state = await indexer.getStateByCID(argv.cid); + assert(state, 'State for the provided CID doesn\'t exist.'); - const ipldData = await indexer.getIPLDData(ipldBlock); + const stateData = await indexer.getStateData(state); - log(util.inspect(ipldData, false, null)); + log(util.inspect(stateData, false, null)); }; main().catch(err => { diff --git a/packages/mobymask-watcher/src/cli/reset-cmds/state.ts b/packages/mobymask-watcher/src/cli/reset-cmds/watcher.ts similarity index 79% rename from packages/mobymask-watcher/src/cli/reset-cmds/state.ts rename to packages/mobymask-watcher/src/cli/reset-cmds/watcher.ts index 5bb02e11..fd611873 100644 --- a/packages/mobymask-watcher/src/cli/reset-cmds/state.ts +++ b/packages/mobymask-watcher/src/cli/reset-cmds/watcher.ts @@ -18,11 +18,11 @@ import { IsRevoked } from '../../entity/IsRevoked'; import { IsPhisher } from '../../entity/IsPhisher'; import { IsMember } from '../../entity/IsMember'; -const log = debug('vulcanize:reset-state'); +const log = debug('vulcanize:reset-watcher'); -export const command = 'state'; +export const command = 'watcher'; -export const desc = 'Reset state to block number'; +export const desc = 'Reset watcher to a block number'; export const builder = { blockNumber: { @@ -76,19 +76,15 @@ export const handler = async (argv: any): Promise => { await indexer.updateSyncStatusCanonicalBlock(blockProgress.blockHash, blockProgress.blockNumber, true); } - const ipldStatus = await indexer.getIPLDStatus(); + const stateSyncStatus = await indexer.getStateSyncStatus(); - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true); + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusIndexedBlock(blockProgress.blockNumber, true); } - if (ipldStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusCheckpointBlock(blockProgress.blockNumber, true); - } - - if (ipldStatus.latestIPFSBlockNumber > blockProgress.blockNumber) { - await indexer.updateIPLDStatusIPFSBlock(blockProgress.blockNumber, true); + if (stateSyncStatus.latestCheckpointBlockNumber > blockProgress.blockNumber) { + await indexer.updateStateSyncStatusCheckpointBlock(blockProgress.blockNumber, true); } } @@ -102,5 +98,5 @@ export const handler = async (argv: any): Promise => { await dbTx.release(); } - log('Reset state successfully'); + log('Reset watcher successfully'); }; diff --git a/packages/mobymask-watcher/src/database.ts b/packages/mobymask-watcher/src/database.ts index b000e6f5..3d51b117 100644 --- a/packages/mobymask-watcher/src/database.ts +++ b/packages/mobymask-watcher/src/database.ts @@ -11,9 +11,9 @@ import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, W import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import { MultiNonce } from './entity/MultiNonce'; import { _Owner } from './entity/_Owner'; import { IsRevoked } from './entity/IsRevoked'; @@ -132,69 +132,63 @@ export class Database implements DatabaseInterface { return repo.save(entity); } - getNewIPLDBlock (): IPLDBlock { - return new IPLDBlock(); + getNewState (): State { + return new State(); } - async getIPLDBlocks (where: FindConditions): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getStates (where: FindConditions): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getIPLDBlocks(repo, where); + return this._baseDatabase.getStates(repo, where); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber); + return this._baseDatabase.getLatestState(repo, contractAddress, kind, blockNumber); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - const repo = this._conn.getRepository(IPLDBlock); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getPrevIPLDBlock(repo, blockHash, contractAddress, kind); + return this._baseDatabase.getPrevState(repo, blockHash, contractAddress, kind); } - // Fetch all diff IPLDBlocks after the specified block number. - async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise { - const repo = this._conn.getRepository(IPLDBlock); + // Fetch all diff States after the specified block number. + async getDiffStatesInRange (contractAddress: string, startblock: number, endBlock: number): Promise { + const repo = this._conn.getRepository(State); - return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock); + return this._baseDatabase.getDiffStatesInRange(repo, contractAddress, startblock, endBlock); } - async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async saveOrUpdateState (dbTx: QueryRunner, state: State): Promise { + const repo = dbTx.manager.getRepository(State); - return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock); + return this._baseDatabase.saveOrUpdateState(repo, state); } - async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { - const repo = dbTx.manager.getRepository(IPLDBlock); + async removeStates (dbTx: QueryRunner, blockNumber: number, kind: string): Promise { + const repo = dbTx.manager.getRepository(State); - await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind); + await this._baseDatabase.removeStates(repo, blockNumber, kind); } - async getIPLDStatus (): Promise { - const repo = this._conn.getRepository(IpldStatus); + async getStateSyncStatus (): Promise { + const repo = this._conn.getRepository(StateSyncStatus); - return this._baseDatabase.getIPLDStatus(repo); + return this._baseDatabase.getStateSyncStatus(repo); } - async updateIPLDStatusHooksBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusHooksBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusIndexedBlock(repo, blockNumber, force); } - async updateIPLDStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); + async updateStateSyncStatusCheckpointBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { + const repo = queryRunner.manager.getRepository(StateSyncStatus); - return this._baseDatabase.updateIPLDStatusCheckpointBlock(repo, blockNumber, force); - } - - async updateIPLDStatusIPFSBlock (queryRunner: QueryRunner, blockNumber: number, force?: boolean): Promise { - const repo = queryRunner.manager.getRepository(IpldStatus); - - return this._baseDatabase.updateIPLDStatusIPFSBlock(repo, blockNumber, force); + return this._baseDatabase.updateStateSyncStatusCheckpointBlock(repo, blockNumber, force); } async getContracts (): Promise { diff --git a/packages/mobymask-watcher/src/entity/IPLDBlock.ts b/packages/mobymask-watcher/src/entity/IPLDBlock.ts deleted file mode 100644 index 0fd42a19..00000000 --- a/packages/mobymask-watcher/src/entity/IPLDBlock.ts +++ /dev/null @@ -1,31 +0,0 @@ -// -// Copyright 2021 Vulcanize, Inc. -// - -import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm'; -import { StateKind } from '@cerc-io/util'; -import { BlockProgress } from './BlockProgress'; - -@Entity() -@Index(['cid'], { unique: true }) -@Index(['block', 'contractAddress']) -@Index(['block', 'contractAddress', 'kind'], { unique: true }) -export class IPLDBlock { - @PrimaryGeneratedColumn() - id!: number; - - @ManyToOne(() => BlockProgress, { onDelete: 'CASCADE' }) - block!: BlockProgress; - - @Column('varchar', { length: 42 }) - contractAddress!: string; - - @Column('varchar') - cid!: string; - - @Column({ type: 'enum', enum: StateKind }) - kind!: StateKind; - - @Column('bytea') - data!: Buffer; -} diff --git a/packages/mobymask-watcher/src/entity/State.ts b/packages/mobymask-watcher/src/entity/State.ts new file mode 100644 index 00000000..10c1bf63 --- /dev/null +++ b/packages/mobymask-watcher/src/entity/State.ts @@ -0,0 +1,36 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm'; + +import { StateKind } from '@cerc-io/util'; + +import { BlockProgress } from './BlockProgress'; + +@Entity() +@Index(['cid'], { unique: true }) +@Index(['block', 'contractAddress']) +@Index(['block', 'contractAddress', 'kind'], { unique: true }) +export class State { + @PrimaryGeneratedColumn() + id!: number; + + @ManyToOne(() => BlockProgress, { onDelete: 'CASCADE' }) + block!: BlockProgress; + + @Column('varchar', { length: 42 }) + contractAddress!: string; + + @Column('varchar') + cid!: string; + + @Column({ + type: 'enum', + enum: StateKind + }) + kind!: StateKind; + + @Column('bytea') + data!: Buffer; +} diff --git a/packages/mobymask-watcher/src/entity/StateSyncStatus.ts b/packages/mobymask-watcher/src/entity/StateSyncStatus.ts new file mode 100644 index 00000000..fae389bc --- /dev/null +++ b/packages/mobymask-watcher/src/entity/StateSyncStatus.ts @@ -0,0 +1,17 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm'; + +@Entity() +export class StateSyncStatus { + @PrimaryGeneratedColumn() + id!: number; + + @Column('integer') + latestIndexedBlockNumber!: number; + + @Column('integer', { nullable: true }) + latestCheckpointBlockNumber!: number; +} diff --git a/packages/mobymask-watcher/src/hooks.ts b/packages/mobymask-watcher/src/hooks.ts index c35bb49c..2166a765 100644 --- a/packages/mobymask-watcher/src/hooks.ts +++ b/packages/mobymask-watcher/src/hooks.ts @@ -25,19 +25,19 @@ export async function createInitialState (indexer: Indexer, contractAddress: str assert(blockHash); assert(contractAddress); - // Store the desired initial state in an IPLDBlock. - const ipldBlockData: any = { + // Store an empty State. + const stateData: any = { state: {} }; // Use updateStateForElementaryType to update initial state with an elementary property. - // Eg. const ipldBlockData = updateStateForElementaryType(ipldBlockData, '_totalBalance', result.value.toString()); + // Eg. const stateData = updateStateForElementaryType(stateData, '_totalBalance', result.value.toString()); // Use updateStateForMappingType to update initial state with a nested property. - // Eg. const ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString()); + // Eg. const stateData = updateStateForMappingType(stateData, '_allowances', [owner, spender], allowance.value.toString()); // Return initial state data to be saved. - return ipldBlockData; + return stateData; } /** diff --git a/packages/mobymask-watcher/src/indexer.ts b/packages/mobymask-watcher/src/indexer.ts index 21c2e0b4..743cbfe4 100644 --- a/packages/mobymask-watcher/src/indexer.ts +++ b/packages/mobymask-watcher/src/indexer.ts @@ -10,7 +10,6 @@ import { ethers } from 'ethers'; import { JsonFragment } from '@ethersproject/abi'; import { JsonRpcProvider } from '@ethersproject/providers'; -import * as codec from '@ipld/dag-cbor'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; import { @@ -25,11 +24,9 @@ import { updateStateForElementaryType, updateStateForMappingType, BlockHeight, - IPFSClient, StateKind, - IpldStatus as IpldStatusInterface, - getFullTransaction, - ResultIPLDBlock + StateStatus, + getFullTransaction } from '@cerc-io/util'; import PhisherRegistryArtifacts from './artifacts/PhisherRegistry.json'; @@ -38,9 +35,9 @@ import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint import { Contract } from './entity/Contract'; import { Event } from './entity/Event'; import { SyncStatus } from './entity/SyncStatus'; -import { IpldStatus } from './entity/IpldStatus'; +import { StateSyncStatus } from './entity/StateSyncStatus'; import { BlockProgress } from './entity/BlockProgress'; -import { IPLDBlock } from './entity/IPLDBlock'; +import { State } from './entity/State'; import { IsMember } from './entity/IsMember'; import { IsPhisher } from './entity/IsPhisher'; import { IsRevoked } from './entity/IsRevoked'; @@ -87,8 +84,6 @@ export class Indexer implements IndexerInterface { _storageLayoutMap: Map _contractMap: Map - _ipfsClient: IPFSClient - constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: JsonRpcProvider, jobQueue: JobQueue) { assert(db); assert(ethClient); @@ -97,8 +92,7 @@ export class Indexer implements IndexerInterface { this._ethClient = ethClient; this._ethProvider = ethProvider; this._serverConfig = serverConfig; - this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); - this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, this._ipfsClient); + this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); this._abiMap = new Map(); this._storageLayoutMap = new Map(); @@ -126,7 +120,7 @@ export class Indexer implements IndexerInterface { async init (): Promise { await this._baseIndexer.fetchContracts(); - await this._baseIndexer.fetchIPLDStatus(); + await this._baseIndexer.fetchStateStatus(); } getResultEvent (event: Event): ResultEvent { @@ -164,26 +158,6 @@ export class Indexer implements IndexerInterface { }; } - getResultIPLDBlock (ipldBlock: IPLDBlock): ResultIPLDBlock { - const block = ipldBlock.block; - - const data = codec.decode(Buffer.from(ipldBlock.data)) as any; - - return { - block: { - cid: block.cid, - hash: block.blockHash, - number: block.blockNumber, - timestamp: block.blockTimestamp, - parentHash: block.parentHash - }, - contractAddress: ipldBlock.contractAddress, - cid: ipldBlock.cid, - kind: ipldBlock.kind, - data: JSON.stringify(data) - }; - } - async multiNonce (blockHash: string, contractAddress: string, key0: string, key1: bigint, diff = false): Promise { let entity = await this._db.getMultiNonce({ blockHash, contractAddress, key0, key1 }); @@ -401,10 +375,6 @@ export class Indexer implements IndexerInterface { ); } - async pushToIPFS (data: any): Promise { - await this._baseIndexer.pushToIPFS(data); - } - async processInitialState (contractAddress: string, blockHash: string): Promise { // Call initial state hook. return createInitialState(this, contractAddress, blockHash); @@ -435,28 +405,24 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.processCLICheckpoint(this, contractAddress, blockHash); } - async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { - return this._db.getPrevIPLDBlock(blockHash, contractAddress, kind); + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + return this._db.getPrevState(blockHash, contractAddress, kind); } - async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - return this._db.getLatestIPLDBlock(contractAddress, kind, blockNumber); + async getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + return this._db.getLatestState(contractAddress, kind, blockNumber); } - async getIPLDBlocksByHash (blockHash: string): Promise { - return this._baseIndexer.getIPLDBlocksByHash(blockHash); + async getStatesByHash (blockHash: string): Promise { + return this._baseIndexer.getStatesByHash(blockHash); } - async getIPLDBlockByCid (cid: string): Promise { - return this._baseIndexer.getIPLDBlockByCid(cid); + async getStateByCID (cid: string): Promise { + return this._baseIndexer.getStateByCID(cid); } - getIPLDData (ipldBlock: IPLDBlock): any { - return this._baseIndexer.getIPLDData(ipldBlock); - } - - isIPFSConfigured (): boolean { - return this._baseIndexer.isIPFSConfigured(); + getStateData (state: State): any { + return this._baseIndexer.getStateData(state); } // Method used to create auto diffs (diff_staged). @@ -488,12 +454,12 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.createCheckpoint(this, contractAddress, block); } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { - return this._baseIndexer.saveOrUpdateIPLDBlock(ipldBlock); + async saveOrUpdateState (state: State): Promise { + return this._baseIndexer.saveOrUpdateState(state); } - async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise { - await this._baseIndexer.removeIPLDBlocks(blockNumber, kind); + async removeStates (blockNumber: number, kind: StateKind): Promise { + await this._baseIndexer.removeStates(blockNumber, kind); } async triggerIndexingOnEvent (event: Event): Promise { @@ -530,16 +496,16 @@ export class Indexer implements IndexerInterface { }; } - async getIPLDStatus (): Promise { - return this._db.getIPLDStatus(); + async getStateSyncStatus (): Promise { + return this._db.getStateSyncStatus(); } - async updateIPLDStatusHooksBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusHooksBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusIndexedBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -551,29 +517,12 @@ export class Indexer implements IndexerInterface { return res; } - async updateIPLDStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.updateIPLDStatusCheckpointBlock(dbTx, blockNumber, force); - await dbTx.commitTransaction(); - } catch (error) { - await dbTx.rollbackTransaction(); - throw error; - } finally { - await dbTx.release(); - } - - return res; - } - - async updateIPLDStatusIPFSBlock (blockNumber: number, force?: boolean): Promise { - const dbTx = await this._db.createTransactionRunner(); - let res; - - try { - res = await this._db.updateIPLDStatusIPFSBlock(dbTx, blockNumber, force); + res = await this._db.updateStateSyncStatusCheckpointBlock(dbTx, blockNumber, force); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -595,16 +544,16 @@ export class Indexer implements IndexerInterface { return latestCanonicalBlock; } - async getLatestHooksProcessedBlock (): Promise { - return this._baseIndexer.getLatestHooksProcessedBlock(); + async getLatestStateIndexedBlock (): Promise { + return this._baseIndexer.getLatestStateIndexedBlock(); } async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } - async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise { - await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + this._baseIndexer.updateStateStatusMap(address, stateStatus); } cacheContract (contract: Contract): void { diff --git a/packages/mobymask-watcher/src/job-runner.ts b/packages/mobymask-watcher/src/job-runner.ts index 42c5c6ed..2fc66644 100644 --- a/packages/mobymask-watcher/src/job-runner.ts +++ b/packages/mobymask-watcher/src/job-runner.ts @@ -17,8 +17,6 @@ import { QUEUE_EVENT_PROCESSING, QUEUE_BLOCK_CHECKPOINT, QUEUE_HOOKS, - QUEUE_IPFS, - JOB_KIND_PRUNE, JobQueueConfig, DEFAULT_CONFIG_PATH, initClients, @@ -48,22 +46,12 @@ export class JobRunner { await this.subscribeEventProcessingQueue(); await this.subscribeBlockCheckpointQueue(); await this.subscribeHooksQueue(); - await this.subscribeIPFSQueue(); this._baseJobRunner.handleShutdown(); } async subscribeBlockProcessingQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseJobRunner.processBlock(job); - - const { data: { kind } } = job; - - // If it's a pruning job: Create a hooks job. - if (kind === JOB_KIND_PRUNE) { - await this.createHooksJob(); - } - - await this._jobQueue.markComplete(job); }); } @@ -75,165 +63,15 @@ export class JobRunner { async subscribeHooksQueue (): Promise { await this._jobQueue.subscribe(QUEUE_HOOKS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - - if (ipldStatus) { - if (ipldStatus.latestHooksBlockNumber < (blockNumber - 1)) { - // Create hooks job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createHooksJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Hooks for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestHooksBlockNumber > (blockNumber - 1)) { - log(`Hooks for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process the hooks for the given block number. - await this._indexer.processCanonicalBlock(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusHooksBlock(blockNumber); - - // Create a checkpoint job after completion of a hook job. - await this.createCheckpointJob(blockHash, blockNumber); - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processHooks(job); }); } async subscribeBlockCheckpointQueue (): Promise { await this._jobQueue.subscribe(QUEUE_BLOCK_CHECKPOINT, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - // Get the current IPLD Status. - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestCheckpointBlockNumber >= 0) { - if (ipldStatus.latestCheckpointBlockNumber < (blockNumber - 1)) { - // Create a checkpoint job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createCheckpointJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestCheckpointBlockNumber > (blockNumber - 1)) { - log(`Checkpoints for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Process checkpoints for the given block. - await this._indexer.processCheckpoint(blockHash); - - // Update the IPLD status. - await this._indexer.updateIPLDStatusCheckpointBlock(blockNumber); - - // Create an IPFS job after completion of a checkpoint job. - if (this._indexer.isIPFSConfigured()) { - await this.createIPFSPutJob(blockHash, blockNumber); - } - - await this._jobQueue.markComplete(job); + await this._baseJobRunner.processCheckpoint(job); }); } - - async subscribeIPFSQueue (): Promise { - await this._jobQueue.subscribe(QUEUE_IPFS, async (job) => { - const { data: { blockHash, blockNumber } } = job; - - const ipldStatus = await this._indexer.getIPLDStatus(); - assert(ipldStatus); - - if (ipldStatus.latestIPFSBlockNumber >= 0) { - if (ipldStatus.latestIPFSBlockNumber < (blockNumber - 1)) { - // Create a IPFS job for parent block. - const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); - await this.createIPFSPutJob(parentBlock.blockHash, parentBlock.blockNumber); - - const message = `IPFS for blockNumber ${blockNumber - 1} not processed yet, aborting`; - log(message); - - throw new Error(message); - } - - if (ipldStatus.latestIPFSBlockNumber > (blockNumber - 1)) { - log(`IPFS for blockNumber ${blockNumber} already processed`); - - return; - } - } - - // Get IPLDBlocks for the given blocHash. - const ipldBlocks = await this._indexer.getIPLDBlocksByHash(blockHash); - - // Push all the IPLDBlocks to IPFS. - for (const ipldBlock of ipldBlocks) { - const data = this._indexer.getIPLDData(ipldBlock); - await this._indexer.pushToIPFS(data); - } - - // Update the IPLD status. - await this._indexer.updateIPLDStatusIPFSBlock(blockNumber); - - await this._jobQueue.markComplete(job); - }); - } - - async createHooksJob (blockHash?: string, blockNumber?: number): Promise { - if (!blockNumber || !blockHash) { - // Get the latest canonical block - const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock(); - - // Create a hooks job for parent block of latestCanonicalBlock because pruning for first block is skipped as it is assumed to be a canonical block. - blockHash = latestCanonicalBlock.parentHash; - blockNumber = latestCanonicalBlock.blockNumber - 1; - } - - await this._jobQueue.pushJob( - QUEUE_HOOKS, - { - blockHash, - blockNumber - } - ); - } - - async createCheckpointJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_BLOCK_CHECKPOINT, - { - blockHash, - blockNumber - } - ); - } - - async createIPFSPutJob (blockHash: string, blockNumber: number): Promise { - await this._jobQueue.pushJob( - QUEUE_IPFS, - { - blockHash, - blockNumber - } - ); - } } export const main = async (): Promise => { diff --git a/packages/mobymask-watcher/src/resolvers.ts b/packages/mobymask-watcher/src/resolvers.ts index 90e77549..7992de60 100644 --- a/packages/mobymask-watcher/src/resolvers.ts +++ b/packages/mobymask-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util'; +import { ValueResult, gqlTotalQueryCount, gqlQueryCount, getResultState } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -126,9 +126,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getStateByCID').inc(1); - const ipldBlock = await indexer.getIPLDBlockByCid(cid); + const state = await indexer.getStateByCID(cid); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { @@ -136,9 +136,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1); - const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); + const state = await indexer.getPrevState(blockHash, contractAddress, kind); - return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; + return state && state.block.isComplete ? getResultState(state) : undefined; }, getSyncStatus: async () => { diff --git a/packages/mobymask-watcher/src/schema.gql b/packages/mobymask-watcher/src/schema.gql index b58f64ab..da107f9c 100644 --- a/packages/mobymask-watcher/src/schema.gql +++ b/packages/mobymask-watcher/src/schema.gql @@ -79,7 +79,7 @@ type PhisherStatusUpdatedEvent { isPhisher: Boolean! } -type ResultIPLDBlock { +type ResultState { block: _Block_! contractAddress: String! cid: String! @@ -102,8 +102,8 @@ type Query { isRevoked(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean! isPhisher(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean! isMember(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean! - getStateByCID(cid: String!): ResultIPLDBlock - getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock + getStateByCID(cid: String!): ResultState + getState(blockHash: String!, contractAddress: String!, kind: String): ResultState getSyncStatus: SyncStatus latestBlock: Block_height } diff --git a/packages/util/index.ts b/packages/util/index.ts index 4869babe..23ef33c3 100644 --- a/packages/util/index.ts +++ b/packages/util/index.ts @@ -12,9 +12,8 @@ export * from './src/events'; export * from './src/types'; export * from './src/indexer'; export * from './src/job-runner'; -export * from './src/ipld-helper'; +export * from './src/state-helper'; export * from './src/graph-decimal'; -export * from './src/ipfs'; export * from './src/index-block'; export * from './src/metrics'; export * from './src/gql-metrics'; diff --git a/packages/util/src/common.ts b/packages/util/src/common.ts index 50c3e05f..03bd8606 100644 --- a/packages/util/src/common.ts +++ b/packages/util/src/common.ts @@ -2,7 +2,14 @@ import debug from 'debug'; import assert from 'assert'; import { DeepPartial } from 'typeorm'; -import { QUEUE_BLOCK_PROCESSING, JOB_KIND_PRUNE, JOB_KIND_INDEX, UNKNOWN_EVENT_NAME } from './constants'; +import { + QUEUE_BLOCK_PROCESSING, + QUEUE_HOOKS, + QUEUE_BLOCK_CHECKPOINT, + JOB_KIND_PRUNE, + JOB_KIND_INDEX, + UNKNOWN_EVENT_NAME +} from './constants'; import { JobQueue } from './job-queue'; import { BlockProgressInterface, IndexerInterface, EventInterface } from './types'; import { wait } from './misc'; @@ -18,30 +25,6 @@ export interface PrefetchedBlock { events: DeepPartial[]; } -/** - * Create pruning job in QUEUE_BLOCK_PROCESSING. - * @param jobQueue - * @param latestCanonicalBlockNumber - * @param priority - */ -export const createPruningJob = async (jobQueue: JobQueue, latestCanonicalBlockNumber: number, priority = 0): Promise => { - const pruneBlockHeight = latestCanonicalBlockNumber + 1; - const newPriority = priority + 1; - - // Create a job to prune at block height (latestCanonicalBlockNumber + 1). - return jobQueue.pushJob( - QUEUE_BLOCK_PROCESSING, - { - kind: JOB_KIND_PRUNE, - pruneBlockHeight, - priority: newPriority - }, - { - priority: newPriority - } - ); -}; - /** * Method to fetch block by number and push to job queue. * @param jobQueue @@ -382,6 +365,62 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block console.timeEnd('time:common#processBatchEvents-updateBlockProgress'); }; +/** + * Create pruning job in QUEUE_BLOCK_PROCESSING. + * @param jobQueue + * @param latestCanonicalBlockNumber + * @param priority + */ +export const createPruningJob = async (jobQueue: JobQueue, latestCanonicalBlockNumber: number, priority = 0): Promise => { + const pruneBlockHeight = latestCanonicalBlockNumber + 1; + const newPriority = priority + 1; + + // Create a job to prune at block height (latestCanonicalBlockNumber + 1). + return jobQueue.pushJob( + QUEUE_BLOCK_PROCESSING, + { + kind: JOB_KIND_PRUNE, + pruneBlockHeight, + priority: newPriority + }, + { + priority: newPriority + } + ); +}; + +/** + * Create a job in QUEUE_HOOKS. + * @param jobQueue + * @param blockHash + * @param blockNumber + */ +export const createHooksJob = async (jobQueue: JobQueue, blockHash: string, blockNumber: number): Promise => { + await jobQueue.pushJob( + QUEUE_HOOKS, + { + blockHash, + blockNumber + } + ); +}; + +/** + * Create a job in QUEUE_BLOCK_CHECKPOINT. + * @param jobQueue + * @param blockHash + * @param blockNumber + */ +export const createCheckpointJob = async (jobQueue: JobQueue, blockHash: string, blockNumber: number): Promise => { + await jobQueue.pushJob( + QUEUE_BLOCK_CHECKPOINT, + { + blockHash, + blockNumber + } + ); +}; + const getPrefetchedBlocksAtHeight = (prefetchedBlocksMap: Map, blockNumber: number):any[] => { return Array.from(prefetchedBlocksMap.values()) .filter(({ block }) => Number(block.blockNumber) === blockNumber) diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index 6180d4f8..3d9061e8 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -36,7 +36,6 @@ export interface ServerConfig { kind: string; checkpointing: boolean; checkpointInterval: number; - ipfsApiAddr: string; subgraphPath: string; disableSubgraphState: boolean; wasmRestartBlocksInterval: number; diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts index f4479d7f..231815f0 100644 --- a/packages/util/src/constants.ts +++ b/packages/util/src/constants.ts @@ -3,13 +3,13 @@ // export const MAX_REORG_DEPTH = 16; +export const DIFF_MERGE_BATCH_SIZE = 10000; export const QUEUE_BLOCK_PROCESSING = 'block-processing'; export const QUEUE_EVENT_PROCESSING = 'event-processing'; export const QUEUE_CHAIN_PRUNING = 'chain-pruning'; export const QUEUE_BLOCK_CHECKPOINT = 'block-checkpoint'; export const QUEUE_HOOKS = 'hooks'; -export const QUEUE_IPFS = 'ipfs'; export const JOB_KIND_INDEX = 'index'; export const JOB_KIND_PRUNE = 'prune'; diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 5fb671f1..ab257979 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -21,7 +21,7 @@ import { SnakeNamingStrategy } from 'typeorm-naming-strategies'; import _ from 'lodash'; import { Pool } from 'pg'; -import { BlockProgressInterface, ContractInterface, EventInterface, IPLDBlockInterface, IpldStatusInterface, StateKind, SyncStatusInterface } from './types'; +import { BlockProgressInterface, ContractInterface, EventInterface, StateInterface, StateSyncStatusInterface, StateKind, SyncStatusInterface } from './types'; import { MAX_REORG_DEPTH, UNKNOWN_EVENT_NAME } from './constants'; import { blockProgressCount, eventCount } from './metrics'; @@ -558,11 +558,11 @@ export class Database { return repo.save(entity); } - async getLatestIPLDBlock (repo: Repository, contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { - let queryBuilder = repo.createQueryBuilder('ipld_block') - .leftJoinAndSelect('ipld_block.block', 'block') + async getLatestState (repo: Repository, contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise { + let queryBuilder = repo.createQueryBuilder('state') + .leftJoinAndSelect('state.block', 'block') .where('block.is_pruned = false') - .andWhere('ipld_block.contract_address = :contractAddress', { contractAddress }) + .andWhere('state.contract_address = :contractAddress', { contractAddress }) .orderBy('block.block_number', 'DESC'); // Filter out blocks after the provided block number. @@ -572,8 +572,8 @@ export class Database { // Filter using kind if specified else avoid diff_staged block. queryBuilder = kind - ? queryBuilder.andWhere('ipld_block.kind = :kind', { kind }) - : queryBuilder.andWhere('ipld_block.kind != :kind', { kind: StateKind.DiffStaged }); + ? queryBuilder.andWhere('state.kind = :kind', { kind }) + : queryBuilder.andWhere('state.kind != :kind', { kind: StateKind.DiffStaged }); // Get the first three entries. queryBuilder.limit(3); @@ -582,7 +582,7 @@ export class Database { if (results.length) { // Sort by (block number desc, id desc) to get the latest entry. - // At same height, IPLD blocks are expected in order ['init', 'diff', 'checkpoint'], + // At same height, State entries are expected in order ['init', 'diff', 'checkpoint'], // and are given preference in order ['checkpoint', 'diff', 'init'] results.sort((result1, result2) => { if (result1.block.blockNumber === result2.block.blockNumber) { @@ -596,7 +596,7 @@ export class Database { } } - async getPrevIPLDBlock (repo: Repository, blockHash: string, contractAddress: string, kind?: string): Promise { + async getPrevState (repo: Repository, blockHash: string, contractAddress: string, kind?: string): Promise { const heirerchicalQuery = ` WITH RECURSIVE cte_query AS ( @@ -605,13 +605,13 @@ export class Database { b.block_number, b.parent_hash, 1 as depth, - i.id, - i.kind + s.id, + s.kind FROM block_progress b LEFT JOIN - ipld_block i ON i.block_id = b.id - AND i.contract_address = $2 + state s ON s.block_id = b.id + AND s.contract_address = $2 WHERE b.block_hash = $1 UNION ALL @@ -620,14 +620,14 @@ export class Database { b.block_number, b.parent_hash, c.depth + 1, - i.id, - i.kind + s.id, + s.kind FROM block_progress b LEFT JOIN - ipld_block i - ON i.block_id = b.id - AND i.contract_address = $2 + state s + ON s.block_id = b.id + AND s.contract_address = $2 INNER JOIN cte_query c ON c.parent_hash = b.block_hash WHERE @@ -646,26 +646,26 @@ export class Database { ? queryResult.find((obj: any) => obj.kind === kind) : queryResult.find((obj: any) => obj.id); - let result: IPLDBlockInterface | undefined; + let result: StateInterface | undefined; if (latestRequiredResult) { result = await repo.findOne(latestRequiredResult.id, { relations: ['block'] }); } else { - // If IPLDBlock not found in frothy region get latest IPLDBlock in the pruned region. - // Filter out IPLDBlocks from pruned blocks. + // If State not found in frothy region get latest State in the pruned region. + // Filter out State entries from pruned blocks. const canonicalBlockNumber = queryResult.pop().block_number + 1; - let queryBuilder = repo.createQueryBuilder('ipld_block') - .leftJoinAndSelect('ipld_block.block', 'block') + let queryBuilder = repo.createQueryBuilder('state') + .leftJoinAndSelect('state.block', 'block') .where('block.is_pruned = false') - .andWhere('ipld_block.contract_address = :contractAddress', { contractAddress }) + .andWhere('state.contract_address = :contractAddress', { contractAddress }) .andWhere('block.block_number <= :canonicalBlockNumber', { canonicalBlockNumber }) .orderBy('block.block_number', 'DESC'); // Filter using kind if specified else order by id to give preference to checkpoint. queryBuilder = kind - ? queryBuilder.andWhere('ipld_block.kind = :kind', { kind }) - : queryBuilder.addOrderBy('ipld_block.id', 'DESC'); + ? queryBuilder.andWhere('state.kind = :kind', { kind }) + : queryBuilder.addOrderBy('state.id', 'DESC'); // Get the first entry. queryBuilder.limit(1); @@ -676,11 +676,11 @@ export class Database { return result; } - async getIPLDBlocks (repo: Repository, where: FindConditions): Promise { + async getStates (repo: Repository, where: FindConditions): Promise { return repo.find({ where, relations: ['block'] }); } - async getDiffIPLDBlocksInRange (repo: Repository, contractAddress: string, startblock: number, endBlock: number): Promise { + async getDiffStatesInRange (repo: Repository, contractAddress: string, startblock: number, endBlock: number): Promise { return repo.find({ relations: ['block'], where: { @@ -697,34 +697,34 @@ export class Database { }); } - async saveOrUpdateIPLDBlock (repo: Repository, ipldBlock: IPLDBlockInterface): Promise { + async saveOrUpdateState (repo: Repository, state: StateInterface): Promise { let updatedData: {[key: string]: any}; - console.time('time:ipld-database#saveOrUpdateIPLDBlock-DB-query'); - if (ipldBlock.id) { + console.time('time:database#saveOrUpdateState-DB-query'); + if (state.id) { // Using pg query as workaround for typeorm memory issue when saving checkpoint with large sized data. const { rows } = await this._pgPool.query(` - UPDATE ipld_block + UPDATE state SET block_id = $1, contract_address = $2, cid = $3, kind = $4, data = $5 WHERE id = $6 RETURNING * - `, [ipldBlock.block.id, ipldBlock.contractAddress, ipldBlock.cid, ipldBlock.kind, ipldBlock.data, ipldBlock.id]); + `, [state.block.id, state.contractAddress, state.cid, state.kind, state.data, state.id]); updatedData = rows[0]; } else { const { rows } = await this._pgPool.query(` - INSERT INTO ipld_block(block_id, contract_address, cid, kind, data) + INSERT INTO state(block_id, contract_address, cid, kind, data) VALUES($1, $2, $3, $4, $5) RETURNING * - `, [ipldBlock.block.id, ipldBlock.contractAddress, ipldBlock.cid, ipldBlock.kind, ipldBlock.data]); + `, [state.block.id, state.contractAddress, state.cid, state.kind, state.data]); updatedData = rows[0]; } - console.timeEnd('time:ipld-database#saveOrUpdateIPLDBlock-DB-query'); + console.timeEnd('time:database#saveOrUpdateState-DB-query'); assert(updatedData); return { - block: ipldBlock.block, + block: state.block, contractAddress: updatedData.contract_address, cid: updatedData.cid, kind: updatedData.kind, @@ -733,7 +733,7 @@ export class Database { }; } - async removeIPLDBlocks (repo: Repository, blockNumber: number, kind: string): Promise { + async removeStates (repo: Repository, blockNumber: number, kind: string): Promise { const entities = await repo.find({ relations: ['block'], where: { block: { blockNumber }, kind } }); // Delete if entities found. @@ -742,43 +742,42 @@ export class Database { } } - async removeIPLDBlocksAfterBlock (repo: Repository, blockNumber: number): Promise { + async removeStatesAfterBlock (repo: Repository, blockNumber: number): Promise { // Use raw SQL as TypeORM curently doesn't support delete via 'join' or 'using' const deleteQuery = ` DELETE FROM - ipld_block + state USING block_progress WHERE - ipld_block.block_id = block_progress.id + state.block_id = block_progress.id AND block_progress.block_number > $1; `; await repo.query(deleteQuery, [blockNumber]); } - async getIPLDStatus (repo: Repository): Promise { + async getStateSyncStatus (repo: Repository): Promise { return repo.findOne(); } - async updateIPLDStatusHooksBlock (repo: Repository, blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (repo: Repository, blockNumber: number, force?: boolean): Promise { let entity = await repo.findOne(); if (!entity) { entity = repo.create({ - latestHooksBlockNumber: blockNumber, - latestCheckpointBlockNumber: -1, - latestIPFSBlockNumber: -1 + latestIndexedBlockNumber: blockNumber, + latestCheckpointBlockNumber: -1 }); } - if (force || blockNumber > entity.latestHooksBlockNumber) { - entity.latestHooksBlockNumber = blockNumber; + if (force || blockNumber > entity.latestIndexedBlockNumber) { + entity.latestIndexedBlockNumber = blockNumber; } return repo.save(entity); } - async updateIPLDStatusCheckpointBlock (repo: Repository, blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusCheckpointBlock (repo: Repository, blockNumber: number, force?: boolean): Promise { const entity = await repo.findOne(); assert(entity); @@ -789,17 +788,6 @@ export class Database { return repo.save(entity); } - async updateIPLDStatusIPFSBlock (repo: Repository, blockNumber: number, force?: boolean): Promise { - const entity = await repo.findOne(); - assert(entity); - - if (force || blockNumber > entity.latestIPFSBlockNumber) { - entity.latestIPFSBlockNumber = blockNumber; - } - - return repo.save(entity); - } - buildQuery (repo: Repository, selectQueryBuilder: SelectQueryBuilder, where: Where = {}): SelectQueryBuilder { Object.entries(where).forEach(([field, filters]) => { filters.forEach((filter, index) => { diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index af91f947..a639d710 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -14,12 +14,20 @@ import * as codec from '@ipld/dag-cbor'; import { EthClient } from '@cerc-io/ipld-eth-client'; import { GetStorageAt, getStorageValue, StorageLayout } from '@cerc-io/solidity-mapper'; -import { BlockProgressInterface, DatabaseInterface, EventInterface, SyncStatusInterface, ContractInterface, IPLDBlockInterface, IndexerInterface, StateKind } from './types'; -import { UNKNOWN_EVENT_NAME, JOB_KIND_CONTRACT, QUEUE_EVENT_PROCESSING } from './constants'; +import { + BlockProgressInterface, + DatabaseInterface, + IndexerInterface, + EventInterface, + ContractInterface, + SyncStatusInterface, + StateInterface, + StateKind +} from './types'; +import { UNKNOWN_EVENT_NAME, JOB_KIND_CONTRACT, QUEUE_EVENT_PROCESSING, DIFF_MERGE_BATCH_SIZE } from './constants'; import { JobQueue } from './job-queue'; import { Where, QueryOptions } from './database'; import { ServerConfig } from './config'; -import { IPFSClient } from './ipfs'; const DEFAULT_MAX_EVENTS_BLOCK_RANGE = 1000; @@ -32,7 +40,7 @@ export interface ValueResult { } } -export interface IpldStatus { +export interface StateStatus { init?: number; diff?: number; checkpoint?: number; @@ -40,7 +48,7 @@ export interface IpldStatus { diff_staged?: number; } -export type ResultIPLDBlock = { +export type ResultState = { block: { cid: string; hash: string; @@ -61,25 +69,22 @@ export class Indexer { _getStorageAt: GetStorageAt; _ethProvider: ethers.providers.BaseProvider; _jobQueue: JobQueue; - _ipfsClient: IPFSClient; _watchedContracts: { [key: string]: ContractInterface } = {}; - _ipldStatusMap: { [key: string]: IpldStatus } = {}; + _stateStatusMap: { [key: string]: StateStatus } = {}; constructor ( serverConfig: ServerConfig, db: DatabaseInterface, ethClient: EthClient, ethProvider: ethers.providers.BaseProvider, - jobQueue: JobQueue, - ipfsClient: IPFSClient + jobQueue: JobQueue ) { this._serverConfig = serverConfig; this._db = db; this._ethClient = ethClient; this._ethProvider = ethProvider; this._jobQueue = jobQueue; - this._ipfsClient = ipfsClient; this._getStorageAt = this._ethClient.getStorageAt.bind(this._ethClient); } @@ -404,7 +409,7 @@ export class Indexer { async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { assert(this._db.saveContract); - this.updateIPLDStatusMap(address, {}); + this.updateStateStatusMap(address, {}); const dbTx = await this._db.createTransactionRunner(); // Use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress) if input to address is a contract address. @@ -448,30 +453,19 @@ export class Indexer { ); } - getIPLDData (ipldBlock: IPLDBlockInterface): any { - return codec.decode(Buffer.from(ipldBlock.data)); + getStateData (state: StateInterface): any { + return codec.decode(Buffer.from(state.data)); } - async pushToIPFS (data: any): Promise { - await this._ipfsClient.push(data); - } + async getLatestStateIndexedBlock (): Promise { + // Get current stateSyncStatus. + const stateSyncStatus = await this._db.getStateSyncStatus(); + assert(stateSyncStatus, 'stateSyncStatus not found'); - isIPFSConfigured (): boolean { - const ipfsAddr = this._serverConfig.ipfsApiAddr; + // Get all the blocks at height stateSyncStatus.latestIndexedBlockNumber. + const blocksAtHeight = await this.getBlocksAtHeight(stateSyncStatus.latestIndexedBlockNumber, false); - // Return false if ipfsAddr is undefined | null | empty string. - return (ipfsAddr !== undefined && ipfsAddr !== null && ipfsAddr !== ''); - } - - async getLatestHooksProcessedBlock (): Promise { - // Get current hookStatus. - const ipldStatus = await this._db.getIPLDStatus(); - assert(ipldStatus, 'IPLD status not found'); - - // Get all the blocks at height hookStatus.latestProcessedBlockNumber. - const blocksAtHeight = await this.getBlocksAtHeight(ipldStatus.latestHooksBlockNumber, false); - - // There can exactly one block at hookStatus.latestProcessedBlockNumber height. + // There can exactly one block at stateSyncStatus.latestIndexedBlockNumber height. assert(blocksAtHeight.length === 1); return blocksAtHeight[0]; @@ -487,11 +481,11 @@ export class Indexer { // For each contract, merge the diff till now to create a checkpoint. for (const contract of contracts) { - // Get IPLD status for the contract. - const ipldStatus = this._ipldStatusMap[contract.address]; - assert(ipldStatus, `IPLD status for contract ${contract.address} not found`); + // Get State status for the contract. + const stateStatus = this._stateStatusMap[contract.address]; + assert(stateStatus, `State status for contract ${contract.address} not found`); - const initBlockNumber = ipldStatus.init; + const initBlockNumber = stateStatus.init; // Check if contract has checkpointing on. // Check if it's time for a checkpoint or the init is in current block. @@ -511,8 +505,8 @@ export class Indexer { if (blockHash) { block = await this.getBlockProgress(blockHash); } else { - // In case of empty blockHash from checkpoint CLI, get the latest processed block from hookStatus for the checkpoint. - block = await this.getLatestHooksProcessedBlock(); + // In case of empty blockHash from checkpoint CLI, get the latest indexed block from stateSyncStatus for the checkpoint. + block = await this.getLatestStateIndexedBlock(); } assert(block); @@ -520,18 +514,6 @@ export class Indexer { const checkpointBlockHash = await this.createCheckpoint(indexer, contractAddress, block); assert(checkpointBlockHash, 'Checkpoint not created'); - // Push checkpoint to IPFS if configured. - if (this.isIPFSConfigured()) { - const checkpointIPLDBlocks = await this._db.getIPLDBlocks({ block, contractAddress, kind: StateKind.Checkpoint }); - - // There can be at most one IPLDBlock for a (block, contractAddress, kind) combination. - assert(checkpointIPLDBlocks.length <= 1); - const checkpointIPLDBlock = checkpointIPLDBlocks[0]; - - const checkpointData = this.getIPLDData(checkpointIPLDBlock); - await this.pushToIPFS(checkpointData); - } - return checkpointBlockHash; } @@ -545,8 +527,8 @@ export class Indexer { } // Create a checkpoint from the hook data without being concerned about diffs. - const ipldBlock = await this.prepareIPLDBlock(block, contractAddress, data, StateKind.Checkpoint); - await this.saveOrUpdateIPLDBlock(ipldBlock); + const state = await this.prepareStateEntry(block, contractAddress, data, StateKind.Checkpoint); + await this.saveOrUpdateState(state); } async createInit ( @@ -566,18 +548,18 @@ export class Indexer { continue; } - // Get IPLD status for the contract. - const ipldStatus = this._ipldStatusMap[contract.address]; - assert(ipldStatus, `IPLD status for contract ${contract.address} not found`); + // Get State status for the contract. + const stateStatus = this._stateStatusMap[contract.address]; + assert(stateStatus, `State status for contract ${contract.address} not found`); - // Check if a 'init' IPLDBlock already exists. - // Or if a 'diff' IPLDBlock already exists. - // Or if a 'checkpoint' IPLDBlock already exists. - // (A watcher with imported state won't have an init IPLDBlock, but it will have the imported checkpoint) + // Check if a 'init' State already exists. + // Or if a 'diff' State already exists. + // Or if a 'checkpoint' State already exists. + // (A watcher with imported state won't have an init State, but it will have the imported checkpoint) if ( - ipldStatus.init || - ipldStatus.diff || - ipldStatus.checkpoint + stateStatus.init || + stateStatus.diff || + stateStatus.checkpoint ) { continue; } @@ -589,14 +571,8 @@ export class Indexer { const block = await this.getBlockProgress(blockHash); assert(block); - const ipldBlock = await this.prepareIPLDBlock(block, contract.address, stateData, StateKind.Init); - await this.saveOrUpdateIPLDBlock(ipldBlock); - - // Push initial state to IPFS if configured. - if (this.isIPFSConfigured()) { - const ipldData = this.getIPLDData(ipldBlock); - await this.pushToIPFS(ipldData); - } + const state = await this.prepareStateEntry(block, contract.address, stateData, StateKind.Init); + await this.saveOrUpdateState(state); } } } @@ -613,27 +589,27 @@ export class Indexer { return; } - // Create a staged diff block. - const ipldBlock = await this.prepareIPLDBlock(block, contractAddress, data, StateKind.DiffStaged); - await this.saveOrUpdateIPLDBlock(ipldBlock); + // Create a staged diff state. + const state = await this.prepareStateEntry(block, contractAddress, data, StateKind.DiffStaged); + await this.saveOrUpdateState(state); } async finalizeDiffStaged (blockHash: string): Promise { const block = await this.getBlockProgress(blockHash); assert(block); - // Get all the staged diff blocks for the given blockHash. - const stagedBlocks = await this._db.getIPLDBlocks({ block, kind: StateKind.DiffStaged }); + // Get all the staged diff states for the given blockHash. + const stagedStates = await this._db.getStates({ block, kind: StateKind.DiffStaged }); // For each staged block, create a diff block. - for (const stagedBlock of stagedBlocks) { - const data = codec.decode(Buffer.from(stagedBlock.data)); - await this.createDiff(stagedBlock.contractAddress, block, data); + for (const stagedState of stagedStates) { + const data = codec.decode(Buffer.from(stagedState.data)); + await this.createDiff(stagedState.contractAddress, block, data); } - // Remove all the staged diff blocks for current blockNumber. + // Remove all the staged diff states for current blockNumber. // (Including staged diff blocks associated with pruned blocks) - await this.removeIPLDBlocks(block.blockNumber, StateKind.DiffStaged); + await this.removeStates(block.blockNumber, StateKind.DiffStaged); } async createDiff (contractAddress: string, block: BlockProgressInterface, data: any): Promise { @@ -645,29 +621,29 @@ export class Indexer { return; } - // Get IPLD status for the contract. - const ipldStatus = this._ipldStatusMap[contractAddress]; - assert(ipldStatus, `IPLD status for contract ${contractAddress} not found`); + // Get State status for the contract. + const stateStatus = this._stateStatusMap[contractAddress]; + assert(stateStatus, `State status for contract ${contractAddress} not found`); // Get the latest checkpoint block number. - const checkpointBlockNumber = ipldStatus.checkpoint; + const checkpointBlockNumber = stateStatus.checkpoint; if (!checkpointBlockNumber) { // Get the initial state block number. - const initBlockNumber = ipldStatus.init; + const initBlockNumber = stateStatus.init; // There should be an initial state at least. - assert(initBlockNumber, 'No initial state found'); + assert(initBlockNumber, `No initial state found for contract ${contractAddress}`); } else if (checkpointBlockNumber === block.blockNumber) { // Check if the latest checkpoint is in the same block if block number is same. - const checkpoint = await this._db.getLatestIPLDBlock(contractAddress, StateKind.Checkpoint); + const checkpoint = await this._db.getLatestState(contractAddress, StateKind.Checkpoint); assert(checkpoint); assert(checkpoint.block.blockHash !== block.blockHash, 'Checkpoint already created for the block hash'); } - const ipldBlock = await this.prepareIPLDBlock(block, contractAddress, data, StateKind.Diff); - await this.saveOrUpdateIPLDBlock(ipldBlock); + const state = await this.prepareStateEntry(block, contractAddress, data, StateKind.Diff); + await this.saveOrUpdateState(state); } async createCheckpoint (indexer: IndexerInterface, contractAddress: string, currentBlock: BlockProgressInterface): Promise { @@ -682,12 +658,12 @@ export class Indexer { // Make sure the block is marked complete. assert(currentBlock.isComplete, 'Block for a checkpoint should be marked as complete'); - // Get current hookStatus. - const ipldStatus = await this._db.getIPLDStatus(); - assert(ipldStatus); + // Get current stateSyncStatus. + const stateSyncStatus = await this._db.getStateSyncStatus(); + assert(stateSyncStatus); - // Make sure the hooks have been processed for the block. - assert(currentBlock.blockNumber <= ipldStatus.latestHooksBlockNumber, 'Block for a checkpoint should have hooks processed'); + // Make sure state for the block has been indexed. + assert(currentBlock.blockNumber <= stateSyncStatus.latestIndexedBlockNumber, 'State should be indexed for checkpoint at a block'); // Call state checkpoint hook and check if default checkpoint is disabled. assert(indexer.processStateCheckpoint); @@ -700,90 +676,90 @@ export class Indexer { } // Fetch the latest 'checkpoint' | 'init' for the contract to fetch diffs after it. - let prevNonDiffBlock: IPLDBlockInterface; + let prevNonDiffState: StateInterface; let diffStartBlockNumber: number; - const checkpointBlock = await this._db.getLatestIPLDBlock(contractAddress, StateKind.Checkpoint, currentBlock.blockNumber - 1); + const checkpointState = await this._db.getLatestState(contractAddress, StateKind.Checkpoint, currentBlock.blockNumber - 1); - if (checkpointBlock) { - const checkpointBlockNumber = checkpointBlock.block.blockNumber; + if (checkpointState) { + const checkpointBlockNumber = checkpointState.block.blockNumber; - prevNonDiffBlock = checkpointBlock; + prevNonDiffState = checkpointState; diffStartBlockNumber = checkpointBlockNumber; - // Update IPLD status map with the latest checkpoint info. + // Update State status map with the latest checkpoint info. // Essential while importing state as checkpoint at the snapshot block is added by import-state CLI. - // (job-runner won't have the updated ipld status) - this.updateIPLDStatusMap(contractAddress, { checkpoint: checkpointBlockNumber }); + // (job-runner won't have the updated State status) + this.updateStateStatusMap(contractAddress, { checkpoint: checkpointBlockNumber }); } else { // There should be an initial state at least. - const initBlock = await this._db.getLatestIPLDBlock(contractAddress, StateKind.Init); - assert(initBlock, 'No initial state found'); + const initBlock = await this._db.getLatestState(contractAddress, StateKind.Init); + assert(initBlock, `No initial state found for contract ${contractAddress}`); - prevNonDiffBlock = initBlock; + prevNonDiffState = initBlock; // Take block number previous to initial state block as the checkpoint is to be created in the same block. diffStartBlockNumber = initBlock.block.blockNumber - 1; } - // Fetching all diff blocks after the latest 'checkpoint' | 'init'. - const diffBlocks = await this._db.getDiffIPLDBlocksInRange(contractAddress, diffStartBlockNumber, currentBlock.blockNumber); - - const prevNonDiffBlockData = codec.decode(Buffer.from(prevNonDiffBlock.data)) as any; - const data = { - state: prevNonDiffBlockData.state + const prevNonDiffStateData = codec.decode(Buffer.from(prevNonDiffState.data)) as any; + let data = { + state: prevNonDiffStateData.state }; - for (const diffBlock of diffBlocks) { - const diff = codec.decode(Buffer.from(diffBlock.data)) as any; - data.state = _.merge(data.state, diff.state); - } + console.time(`time:indexer#createCheckpoint-${contractAddress}`); - const ipldBlock = await this.prepareIPLDBlock(currentBlock, contractAddress, data, StateKind.Checkpoint); - await this.saveOrUpdateIPLDBlock(ipldBlock); + // Fetching and merging all diff blocks after the latest 'checkpoint' | 'init'. + data = await this._mergeDiffsInRange(data, contractAddress, diffStartBlockNumber, currentBlock.blockNumber); + const state = await this.prepareStateEntry(currentBlock, contractAddress, data, StateKind.Checkpoint); + await this.saveOrUpdateState(state); + + console.timeEnd(`time:indexer#createCheckpoint-${contractAddress}`); return currentBlock.blockHash; } - async prepareIPLDBlock (block: BlockProgressInterface, contractAddress: string, data: any, kind: StateKind):Promise { - console.time('time:ipld-indexer#prepareIPLDBlock'); - let ipldBlock: IPLDBlockInterface; + async prepareStateEntry (block: BlockProgressInterface, contractAddress: string, data: any, kind: StateKind):Promise { + console.time('time:indexer#prepareStateEntry'); + let stateEntry: StateInterface; - // Get IPLD status for the contract. - const ipldStatus = this._ipldStatusMap[contractAddress]; - assert(ipldStatus, `IPLD status for contract ${contractAddress} not found`); + // Get State status for the contract. + const stateStatus = this._stateStatusMap[contractAddress]; + assert(stateStatus, `State status for contract ${contractAddress} not found`); - // Get an existing 'init' | 'diff' | 'diff_staged' | 'checkpoint' IPLDBlock for current block, contractAddress to update. - let currentIPLDBlock: IPLDBlockInterface | undefined; - const prevIPLDBlockNumber = ipldStatus[kind]; + // Get an existing 'init' | 'diff' | 'diff_staged' | 'checkpoint' State for current block, contractAddress to update. + let currentState: StateInterface | undefined; + const prevStateBlockNumber = stateStatus[kind]; - // Fetch from DB for previous IPLD block or for checkpoint kind. - if (kind === 'checkpoint' || (prevIPLDBlockNumber && prevIPLDBlockNumber === block.blockNumber)) { - const currentIPLDBlocks = await this._db.getIPLDBlocks({ block, contractAddress, kind }); + // Fetch previous State from DB if: + // present at the same height (to update) + // or for checkpoint kind (to build upon previous checkpoint) + if (kind === 'checkpoint' || (prevStateBlockNumber && prevStateBlockNumber === block.blockNumber)) { + const currentStates = await this._db.getStates({ block, contractAddress, kind }); - // There can be at most one IPLDBlock for a (block, contractAddress, kind) combination. - assert(currentIPLDBlocks.length <= 1); - currentIPLDBlock = currentIPLDBlocks[0]; + // There can be at most one State for a (block, contractAddress, kind) combination. + assert(currentStates.length <= 1); + currentState = currentStates[0]; } - if (currentIPLDBlock) { - // Update current IPLDBlock of same kind if it exists. - ipldBlock = currentIPLDBlock; + if (currentState) { + // Update current State of same kind if it exists. + stateEntry = currentState; // Update the data field. - const oldData = codec.decode(Buffer.from(ipldBlock.data)); + const oldData = codec.decode(Buffer.from(stateEntry.data)); data = _.merge(oldData, data); } else { - // Create a new IPLDBlock instance. - ipldBlock = this._db.getNewIPLDBlock(); + // Create a new State instance. + stateEntry = this._db.getNewState(); - // Fetch the parent IPLDBlock. - const parentIPLDBlock = await this._db.getLatestIPLDBlock(contractAddress, null, block.blockNumber); + // Fetch the parent State entry. + const parentState = await this._db.getLatestState(contractAddress, null, block.blockNumber); - // Setting the meta-data for an IPLDBlock (done only once per IPLD block). + // Setting the meta-data for a State entry (done only once per State entry). data.meta = { id: contractAddress, kind, parent: { - '/': parentIPLDBlock ? parentIPLDBlock.cid : null + '/': parentState ? parentState.cid : null }, ethBlock: { cid: { @@ -803,8 +779,8 @@ export class Indexer { // Calculating the CID: v1, code: dag-cbor, hash. const cid = CID.create(1, codec.code, hash); - // Update ipldBlock with new data. - ipldBlock = Object.assign(ipldBlock, { + // Update stateEntry with new data. + stateEntry = Object.assign(stateEntry, { block, contractAddress, cid: cid.toString(), @@ -812,19 +788,19 @@ export class Indexer { data: Buffer.from(bytes) }); - console.timeEnd('time:ipld-indexer#prepareIPLDBlock'); - return ipldBlock; + console.timeEnd('time:indexer#prepareStateEntry'); + return stateEntry; } - async getIPLDBlocksByHash (blockHash: string): Promise { + async getStatesByHash (blockHash: string): Promise { const block = await this.getBlockProgress(blockHash); assert(block); - return this._db.getIPLDBlocks({ block }); + return this._db.getStates({ block }); } - async getIPLDBlockByCid (cid: string): Promise { - const ipldBlocks = await this._db.getIPLDBlocks({ cid }); + async getStateByCID (cid: string): Promise { + const ipldBlocks = await this._db.getStates({ cid }); // There can be only one IPLDBlock with a particular cid. assert(ipldBlocks.length <= 1); @@ -832,19 +808,19 @@ export class Indexer { return ipldBlocks[0]; } - async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlockInterface): Promise { + async saveOrUpdateState (state: StateInterface): Promise { const dbTx = await this._db.createTransactionRunner(); let res; try { - res = await this._db.saveOrUpdateIPLDBlock(dbTx, ipldBlock); + res = await this._db.saveOrUpdateState(dbTx, state); - // Get IPLD status for the contract. - const ipldStatus = this._ipldStatusMap[res.contractAddress]; - assert(ipldStatus, `IPLD status for contract ${res.contractAddress} not found`); + // Get State status for the contract. + const stateStatus = this._stateStatusMap[res.contractAddress]; + assert(stateStatus, `State status for contract ${res.contractAddress} not found`); - // Update the IPLD status for the kind. - ipldStatus[res.kind] = res.block.blockNumber; + // Update the State status for the kind. + stateStatus[res.kind] = res.block.blockNumber; await dbTx.commitTransaction(); } catch (error) { @@ -857,11 +833,11 @@ export class Indexer { return res; } - async removeIPLDBlocks (blockNumber: number, kind: StateKind): Promise { + async removeStates (blockNumber: number, kind: StateKind): Promise { const dbTx = await this._db.createTransactionRunner(); try { - await this._db.removeIPLDBlocks(dbTx, blockNumber, kind); + await this._db.removeStates(dbTx, blockNumber, kind); await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -871,28 +847,29 @@ export class Indexer { } } - async fetchIPLDStatus (): Promise { + async fetchStateStatus (): Promise { const contracts = Object.values(this._watchedContracts); + // TODO: Fire a single query for all contracts. for (const contract of contracts) { - const initIPLDBlock = await this._db.getLatestIPLDBlock(contract.address, StateKind.Init); - const diffIPLDBlock = await this._db.getLatestIPLDBlock(contract.address, StateKind.Diff); - const diffStagedIPLDBlock = await this._db.getLatestIPLDBlock(contract.address, StateKind.DiffStaged); - const checkpointIPLDBlock = await this._db.getLatestIPLDBlock(contract.address, StateKind.Checkpoint); + const initState = await this._db.getLatestState(contract.address, StateKind.Init); + const diffState = await this._db.getLatestState(contract.address, StateKind.Diff); + const diffStagedState = await this._db.getLatestState(contract.address, StateKind.DiffStaged); + const checkpointState = await this._db.getLatestState(contract.address, StateKind.Checkpoint); - this._ipldStatusMap[contract.address] = { - init: initIPLDBlock?.block.blockNumber, - diff: diffIPLDBlock?.block.blockNumber, - diff_staged: diffStagedIPLDBlock?.block.blockNumber, - checkpoint: checkpointIPLDBlock?.block.blockNumber + this._stateStatusMap[contract.address] = { + init: initState?.block.blockNumber, + diff: diffState?.block.blockNumber, + diff_staged: diffStagedState?.block.blockNumber, + checkpoint: checkpointState?.block.blockNumber }; } } - updateIPLDStatusMap (address: string, ipldStatus: IpldStatus): void { - // Get and update IPLD status for the contract. - const ipldStatusOld = this._ipldStatusMap[address]; - this._ipldStatusMap[address] = _.merge(ipldStatusOld, ipldStatus); + updateStateStatusMap (address: string, stateStatus: StateStatus): void { + // Get and update State status for the contract. + const oldStateStatus = this._stateStatusMap[address]; + this._stateStatusMap[address] = _.merge(oldStateStatus, stateStatus); } parseEvent (logDescription: ethers.utils.LogDescription): { eventName: string, eventInfo: any } { @@ -933,4 +910,25 @@ export class Indexer { return arg; } + + async _mergeDiffsInRange (data: { state: any }, contractAddress: string, startBlock: number, endBlock: number): Promise<{ state: any }> { + // Merge all diff blocks in the given range in batches. + for (let i = startBlock; i < endBlock;) { + const endBlockHeight = Math.min(i + DIFF_MERGE_BATCH_SIZE, endBlock); + + console.time(`time:indexer#_mergeDiffsInRange-${i}-${endBlockHeight}-${contractAddress}`); + const diffBlocks = await this._db.getDiffStatesInRange(contractAddress, i, endBlockHeight); + + // Merge all diff blocks in the current batch. + for (const diffBlock of diffBlocks) { + const diff = codec.decode(Buffer.from(diffBlock.data)) as any; + data.state = _.merge(data.state, diff.state); + } + + i = endBlockHeight; + console.timeEnd(`time:indexer#_mergeDiffsInRange-${i}-${endBlockHeight}-${contractAddress}`); + } + + return data; + } } diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index 464c4492..ee01e26e 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -19,7 +19,12 @@ import { import { JobQueue } from './job-queue'; import { EventInterface, IndexerInterface, SyncStatusInterface } from './types'; import { wait } from './misc'; -import { createPruningJob, processBatchEvents } from './common'; +import { + createPruningJob, + createHooksJob, + createCheckpointJob, + processBatchEvents +} from './common'; import { lastBlockNumEvents, lastBlockProcessDuration, lastProcessedBlockNumber } from './metrics'; const log = debug('vulcanize:job-runner'); @@ -50,14 +55,21 @@ export class JobRunner { await this._indexBlock(job, syncStatus); break; - case JOB_KIND_PRUNE: + case JOB_KIND_PRUNE: { await this._pruneChain(job, syncStatus); + + // Create a hooks job for parent block of latestCanonicalBlock pruning for first block is skipped as it is assumed to be a canonical block. + const latestCanonicalBlock = await this._indexer.getLatestCanonicalBlock(); + await createHooksJob(this._jobQueue, latestCanonicalBlock.parentHash, latestCanonicalBlock.blockNumber - 1); break; + } default: log(`Invalid Job kind ${kind} in QUEUE_BLOCK_PROCESSING.`); break; } + + await this._jobQueue.markComplete(job); } async processEvent (job: any): Promise { @@ -69,7 +81,7 @@ export class JobRunner { break; case JOB_KIND_CONTRACT: - await this._updateWatchedContracts(job); + this._updateWatchedContracts(job); break; default: @@ -80,12 +92,84 @@ export class JobRunner { await this._jobQueue.markComplete(job); } - handleShutdown () { + async processHooks (job: any): Promise { + const { data: { blockHash, blockNumber } } = job; + + // Get the current stateSyncStatus. + const stateSyncStatus = await this._indexer.getStateSyncStatus(); + + if (stateSyncStatus) { + if (stateSyncStatus.latestIndexedBlockNumber < (blockNumber - 1)) { + // Create hooks job for parent block. + const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); + await createHooksJob(this._jobQueue, parentBlock.blockHash, parentBlock.blockNumber); + + const message = `State for blockNumber ${blockNumber - 1} not indexed yet, aborting`; + log(message); + + throw new Error(message); + } + + if (stateSyncStatus.latestIndexedBlockNumber > (blockNumber - 1)) { + log(`State for blockNumber ${blockNumber} already indexed`); + + return; + } + } + + // Process the hooks for the given block number. + await this._indexer.processCanonicalBlock(blockHash, blockNumber); + + // Update the stateSyncStatus. + await this._indexer.updateStateSyncStatusIndexedBlock(blockNumber); + + // Create a checkpoint job after completion of a hooks job. + await createCheckpointJob(this._jobQueue, blockHash, blockNumber); + + await this._jobQueue.markComplete(job); + } + + async processCheckpoint (job: any): Promise { + const { data: { blockHash, blockNumber } } = job; + + // Get the current stateSyncStatus. + const stateSyncStatus = await this._indexer.getStateSyncStatus(); + assert(stateSyncStatus); + + if (stateSyncStatus.latestCheckpointBlockNumber >= 0) { + if (stateSyncStatus.latestCheckpointBlockNumber < (blockNumber - 1)) { + // Create a checkpoint job for parent block. + const [parentBlock] = await this._indexer.getBlocksAtHeight(blockNumber - 1, false); + await createCheckpointJob(this._jobQueue, parentBlock.blockHash, parentBlock.blockNumber); + + const message = `Checkpoints for blockNumber ${blockNumber - 1} not processed yet, aborting`; + log(message); + + throw new Error(message); + } + + if (stateSyncStatus.latestCheckpointBlockNumber > (blockNumber - 1)) { + log(`Checkpoints for blockNumber ${blockNumber} already processed`); + + return; + } + } + + // Process checkpoints for the given block. + await this._indexer.processCheckpoint(blockHash); + + // Update the stateSyncStatus. + await this._indexer.updateStateSyncStatusCheckpointBlock(blockNumber); + + await this._jobQueue.markComplete(job); + } + + handleShutdown (): void { process.on('SIGINT', this._processShutdown.bind(this)); process.on('SIGTERM', this._processShutdown.bind(this)); } - async _processShutdown () { + async _processShutdown (): Promise { this._shutDown = true; this._signalCount++; @@ -291,15 +375,12 @@ export class JobRunner { } } - async _updateWatchedContracts (job: any): Promise { + _updateWatchedContracts (job: any): void { const { data: { contract } } = job; assert(this._indexer.cacheContract); this._indexer.cacheContract(contract); - const ipldIndexer = this._indexer; - if (ipldIndexer.updateIPLDStatusMap) { - ipldIndexer.updateIPLDStatusMap(contract.address, {}); - } + this._indexer.updateStateStatusMap(contract.address, {}); } } diff --git a/packages/util/src/misc.ts b/packages/util/src/misc.ts index fd7e2a14..36d032b8 100644 --- a/packages/util/src/misc.ts +++ b/packages/util/src/misc.ts @@ -19,8 +19,6 @@ import { GraphDecimal } from './graph-decimal'; import * as EthDecoder from './eth'; import { getCachedBlockSize } from './block-size-cache'; -const log = debug('vulcanize:misc'); - /** * Method to wait for specified time. * @param time Time to wait in milliseconds @@ -243,7 +241,7 @@ export const getFullTransaction = async (ethClient: EthClient, txHash: string): }; }; -export const jsonBigIntStringReplacer = (_: string, value: any) => { +export const jsonBigIntStringReplacer = (_: string, value: any): any => { if (typeof value === 'bigint') { return value.toString(); } diff --git a/packages/util/src/ipld-helper.ts b/packages/util/src/state-helper.ts similarity index 53% rename from packages/util/src/ipld-helper.ts rename to packages/util/src/state-helper.ts index 714533f9..35ef2b46 100644 --- a/packages/util/src/ipld-helper.ts +++ b/packages/util/src/state-helper.ts @@ -1,10 +1,12 @@ import _ from 'lodash'; import debug from 'debug'; +import * as codec from '@ipld/dag-cbor'; -import { BlockProgressInterface, GraphDatabaseInterface } from './types'; +import { BlockProgressInterface, GraphDatabaseInterface, StateInterface } from './types'; import { jsonBigIntStringReplacer } from './misc'; +import { ResultState } from './indexer'; -const log = debug('vulcanize:ipld-helper'); +const log = debug('vulcanize:state-helper'); export const updateStateForElementaryType = (initialObject: any, stateVariable: string, value: any): any => { const object = _.cloneDeep(initialObject); @@ -21,17 +23,17 @@ export const updateStateForMappingType = (initialObject: any, stateVariable: str return _.setWith(object, keys, value, Object); }; -export const verifyCheckpointData = async (database: GraphDatabaseInterface, block: BlockProgressInterface, data: any) => { +export const verifyCheckpointData = async (database: GraphDatabaseInterface, block: BlockProgressInterface, data: any): Promise => { const { state } = data; for (const [entityName, idEntityMap] of Object.entries(state)) { - for (const [id, ipldEntity] of Object.entries(idEntityMap as {[key: string]: any})) { + for (const [id, stateEntity] of Object.entries(idEntityMap as {[key: string]: any})) { const entityData = await database.getEntity(entityName, id, block.blockHash) as any; // Compare entities. - const diffFound = Object.keys(ipldEntity) + const diffFound = Object.keys(stateEntity) .some(key => { - let ipldValue = ipldEntity[key]; + let stateValue = stateEntity[key]; if (key === 'blockNumber') { entityData.blockNumber = entityData._blockNumber; @@ -41,23 +43,23 @@ export const verifyCheckpointData = async (database: GraphDatabaseInterface, blo entityData.blockHash = entityData._blockHash; } - if (typeof ipldEntity[key] === 'object' && ipldEntity[key]?.id) { - ipldValue = ipldEntity[key].id; + if (typeof stateEntity[key] === 'object' && stateEntity[key]?.id) { + stateValue = stateEntity[key].id; } if ( - Array.isArray(ipldEntity[key]) && - ipldEntity[key].length && - ipldEntity[key][0].id + Array.isArray(stateEntity[key]) && + stateEntity[key].length && + stateEntity[key][0].id ) { - // Map IPLD entity 1 to N relation field array to match DB entity. - ipldValue = ipldEntity[key].map(({ id }: { id: string }) => id); + // Map State entity 1 to N relation field array to match DB entity. + stateValue = stateEntity[key].map(({ id }: { id: string }) => id); // Sort DB entity 1 to N relation field array. entityData[key] = entityData[key].sort((a: string, b: string) => a.localeCompare(b)); } - return JSON.stringify(ipldValue) !== JSON.stringify(entityData[key], jsonBigIntStringReplacer); + return JSON.stringify(stateValue) !== JSON.stringify(entityData[key], jsonBigIntStringReplacer); }); if (diffFound) { @@ -68,3 +70,23 @@ export const verifyCheckpointData = async (database: GraphDatabaseInterface, blo } } }; + +export const getResultState = (state: StateInterface): ResultState => { + const block = state.block; + + const data = codec.decode(Buffer.from(state.data)) as any; + + return { + block: { + cid: block.cid, + hash: block.blockHash, + number: block.blockNumber, + timestamp: block.blockTimestamp, + parentHash: block.parentHash + }, + contractAddress: state.contractAddress, + cid: state.cid, + kind: state.kind, + data: JSON.stringify(data) + }; +}; diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 74d85c50..e5011ad7 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -7,7 +7,7 @@ import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; import { ServerConfig } from './config'; import { Where, QueryOptions } from './database'; -import { ValueResult, IpldStatus } from './indexer'; +import { ValueResult, StateStatus } from './indexer'; export enum StateKind { Diff = 'diff', @@ -43,11 +43,10 @@ export interface SyncStatusInterface { initialIndexedBlockNumber: number; } -export interface IpldStatusInterface { +export interface StateSyncStatusInterface { id: number; - latestHooksBlockNumber: number; + latestIndexedBlockNumber: number; latestCheckpointBlockNumber: number; - latestIPFSBlockNumber: number } export interface EventInterface { @@ -70,7 +69,7 @@ export interface ContractInterface { checkpoint: boolean; } -export interface IPLDBlockInterface { +export interface StateInterface { id: number; block: BlockProgressInterface; contractAddress: string; @@ -85,9 +84,11 @@ export interface IndexerInterface { getBlockProgress (blockHash: string): Promise getBlockProgressEntities (where: FindConditions, options: FindManyOptions): Promise getEvent (id: string): Promise - getSyncStatus (): Promise; + getSyncStatus (): Promise + getStateSyncStatus (): Promise getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise - getBlocksAtHeight (height: number, isPruned: boolean): Promise; + getBlocksAtHeight (height: number, isPruned: boolean): Promise + getLatestCanonicalBlock (): Promise getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise> getAncestorAtDepth (blockHash: string, depth: number): Promise fetchBlockWithEvents (block: DeepPartial): Promise @@ -97,12 +98,14 @@ export interface IndexerInterface { updateSyncStatusChainHead (blockHash: string, blockNumber: number, force?: boolean): Promise updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number, force?: boolean): Promise updateSyncStatusCanonicalBlock (blockHash: string, blockNumber: number, force?: boolean): Promise - markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise; - saveEventEntity (dbEvent: EventInterface): Promise; - processEvent (event: EventInterface): Promise; - parseEventNameAndArgs?: (kind: string, logObj: any) => any; + updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise + updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise + markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise + saveEventEntity (dbEvent: EventInterface): Promise + processEvent (event: EventInterface): Promise + parseEventNameAndArgs?: (kind: string, logObj: any) => any isWatchedContract: (address: string) => ContractInterface | undefined; - getContractsByKind?: (kind: string) => ContractInterface[]; + getContractsByKind?: (kind: string) => ContractInterface[] cacheContract?: (contract: ContractInterface) => void; watchContract?: (address: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise getEntityTypesMap?: () => Map @@ -112,10 +115,12 @@ export interface IndexerInterface { processStateCheckpoint?: (contractAddress: string, blockHash: string) => Promise processBlock: (blockProgres: BlockProgressInterface) => Promise processBlockAfterEvents?: (blockHash: string) => Promise + processCanonicalBlock (blockHash: string, blockNumber: number): Promise + processCheckpoint (blockHash: string): Promise getStorageValue (storageLayout: StorageLayout, blockHash: string, contractAddress: string, variable: string, ...mappingKeys: MappingKey[]): Promise updateSubgraphState?: (contractAddress: string, data: any) => void - updateIPLDStatusMap (address: string, ipldStatus: IpldStatus): Promise - getIPLDData (ipldBlock: IPLDBlockInterface): any + updateStateStatusMap (address: string, stateStatus: StateStatus): void + getStateData (state: StateInterface): any } export interface EventWatcherInterface { @@ -148,13 +153,13 @@ export interface DatabaseInterface { removeEntities (queryRunner: QueryRunner, entity: new () => Entity, findConditions?: FindManyOptions | FindConditions): Promise; getContracts?: () => Promise saveContract?: (queryRunner: QueryRunner, contractAddress: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise - getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise - getIPLDBlocks (where: FindConditions): Promise - getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise - getNewIPLDBlock (): IPLDBlockInterface - removeIPLDBlocks(dbTx: QueryRunner, blockNumber: number, kind: StateKind): Promise - saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlockInterface): Promise - getIPLDStatus (): Promise + getLatestState (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise + getStates (where: FindConditions): Promise + getDiffStatesInRange (contractAddress: string, startBlock: number, endBlock: number): Promise + getNewState (): StateInterface + removeStates(dbTx: QueryRunner, blockNumber: number, kind: StateKind): Promise + saveOrUpdateState (dbTx: QueryRunner, state: StateInterface): Promise + getStateSyncStatus (): Promise } export interface GraphDatabaseInterface {