From ff471da287b86f979f31b8856f63163c3b8c55b5 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Thu, 20 Jun 2024 10:05:53 +0530 Subject: [PATCH] If subgraph block handlers exist force realtime processing in watcher (#524) * Avoid historical sync for block handlers and eth_getLogs without event handlers * Stop processing after endBlock in watcher fill CLI * Codegen changes for avoiding eth_getLogs when no subgraph event handlers exist * Upgrade package version to 0.2.100 * Check for blockHandlerExists first before block processing * Fix comments for historical block processing checks --- lerna.json | 2 +- packages/cache/package.json | 2 +- packages/cli/package.json | 12 +++++----- packages/codegen/package.json | 4 ++-- .../src/templates/config-template.handlebars | 3 +-- .../src/templates/indexer-template.handlebars | 23 ++++++++++++++++++- .../src/templates/package-template.handlebars | 10 ++++---- packages/graph-node/package.json | 10 ++++---- packages/graph-node/src/watcher.ts | 15 ++++++++++++ packages/ipld-eth-client/package.json | 6 ++--- packages/peer/package.json | 2 +- packages/rpc-eth-client/package.json | 8 +++---- packages/solidity-mapper/package.json | 2 +- packages/test/package.json | 2 +- packages/tracing-client/package.json | 2 +- packages/util/package.json | 8 +++---- packages/util/src/events.ts | 12 +++++++--- packages/util/src/fill.ts | 8 +++---- packages/util/src/types.ts | 2 ++ 19 files changed, 88 insertions(+), 45 deletions(-) diff --git a/lerna.json b/lerna.json index d76913e7..2a001136 100644 --- a/lerna.json +++ b/lerna.json @@ -2,7 +2,7 @@ "packages": [ "packages/*" ], - "version": "0.2.99", + "version": "0.2.100", "npmClient": "yarn", "useWorkspaces": true, "command": { diff --git a/packages/cache/package.json b/packages/cache/package.json index ab809b8d..0661d8db 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cache", - "version": "0.2.99", + "version": "0.2.100", "description": "Generic object cache", "main": "dist/index.js", "scripts": { diff --git a/packages/cli/package.json b/packages/cli/package.json index ab2075d9..43edb6f1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cli", - "version": "0.2.99", + "version": "0.2.100", "main": "dist/index.js", "license": "AGPL-3.0", "scripts": { @@ -15,13 +15,13 @@ }, "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.99", - "@cerc-io/ipld-eth-client": "^0.2.99", + "@cerc-io/cache": "^0.2.100", + "@cerc-io/ipld-eth-client": "^0.2.100", "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.99", - "@cerc-io/rpc-eth-client": "^0.2.99", - "@cerc-io/util": "^0.2.99", + "@cerc-io/peer": "^0.2.100", + "@cerc-io/rpc-eth-client": "^0.2.100", + "@cerc-io/util": "^0.2.100", "@ethersproject/providers": "^5.4.4", "@graphql-tools/utils": "^9.1.1", "@ipld/dag-cbor": "^8.0.0", diff --git a/packages/codegen/package.json b/packages/codegen/package.json index 8906b3c5..71657a07 100644 --- a/packages/codegen/package.json +++ b/packages/codegen/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/codegen", - "version": "0.2.99", + "version": "0.2.100", "description": "Code generator", "private": true, "main": "index.js", @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/util": "^0.2.99", + "@cerc-io/util": "^0.2.100", "@graphql-tools/load-files": "^6.5.2", "@npmcli/package-json": "^5.0.0", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index 88a0cb79..1f69980c 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -10,8 +10,7 @@ checkpointInterval = 2000 # Enable state creation - # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true + enableState = false {{#if (subgraphPath)}} subgraphPath = "./subgraph-build" diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index b8d37d59..47c5cb35 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -829,6 +829,7 @@ export class Indexer implements IndexerInterface { {{/each}} } + // eslint-disable-next-line @typescript-eslint/no-empty-function _populateRelationsMap (): void { {{#each subgraphEntities as | subgraphEntity |}} {{#if subgraphEntity.relations}} @@ -867,7 +868,27 @@ export class Indexer implements IndexerInterface { assert(blockHash); assert(blockNumber); - const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this)); + {{#if (subgraphPath)}} + let dbEvents: DeepPartial[] = []; + let transactions: EthFullTransaction[] = []; + + // Fetch events and txs only if subgraph config has any event handlers + if (this._graphWatcher.eventHandlerExists) { + ({ events: dbEvents, transactions } = await this._baseIndexer.fetchEvents( + blockHash, + blockNumber, + this.eventSignaturesMap, + this.parseEventNameAndArgs.bind(this) + )); + } + {{else~}} + const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents( + blockHash, + blockNumber, + this.eventSignaturesMap, + this.parseEventNameAndArgs.bind(this) + ); + {{/if}} const dbTx = await this._db.createTransactionRunner(); try { diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index a72f7e72..b47d88fe 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -41,12 +41,12 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.99", - "@cerc-io/ipld-eth-client": "^0.2.99", - "@cerc-io/solidity-mapper": "^0.2.99", - "@cerc-io/util": "^0.2.99", + "@cerc-io/cli": "^0.2.100", + "@cerc-io/ipld-eth-client": "^0.2.100", + "@cerc-io/solidity-mapper": "^0.2.100", + "@cerc-io/util": "^0.2.100", {{#if (subgraphPath)}} - "@cerc-io/graph-node": "^0.2.99", + "@cerc-io/graph-node": "^0.2.100", {{/if}} "@ethersproject/providers": "^5.4.4", "debug": "^4.3.1", diff --git a/packages/graph-node/package.json b/packages/graph-node/package.json index 16eb63b0..fa6d634c 100644 --- a/packages/graph-node/package.json +++ b/packages/graph-node/package.json @@ -1,10 +1,10 @@ { "name": "@cerc-io/graph-node", - "version": "0.2.99", + "version": "0.2.100", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { - "@cerc-io/solidity-mapper": "^0.2.99", + "@cerc-io/solidity-mapper": "^0.2.100", "@ethersproject/providers": "^5.4.4", "@graphprotocol/graph-ts": "^0.22.0", "@nomiclabs/hardhat-ethers": "^2.0.2", @@ -51,9 +51,9 @@ "dependencies": { "@apollo/client": "^3.3.19", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", - "@cerc-io/cache": "^0.2.99", - "@cerc-io/ipld-eth-client": "^0.2.99", - "@cerc-io/util": "^0.2.99", + "@cerc-io/cache": "^0.2.100", + "@cerc-io/ipld-eth-client": "^0.2.100", + "@cerc-io/util": "^0.2.100", "@types/json-diff": "^0.5.2", "@types/yargs": "^17.0.0", "bn.js": "^4.11.9", diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index 30d89141..26e13a9c 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -60,6 +60,9 @@ export class GraphWatcher { _context: Context; + _blockHandlerExists = false; + _eventHandlerExists = false; + constructor (database: GraphDatabase, ethClient: EthClient, ethProvider: providers.BaseProvider, serverConfig: ServerConfig) { this._database = database; this._ethClient = ethClient; @@ -110,6 +113,10 @@ export class GraphWatcher { }; }, {}); + // Check if handlers exist for deciding watcher behaviour + this._blockHandlerExists = this._dataSources.some(dataSource => Boolean(dataSource.mapping.blockHandlers)); + this._eventHandlerExists = this._dataSources.some(dataSource => Boolean(dataSource.mapping.eventHandlers)); + const data = await Promise.all(dataPromises); // Create a map from dataSource contract address to instance and contract interface. @@ -151,6 +158,14 @@ export class GraphWatcher { return this._dataSources; } + get blockHandlerExists (): boolean { + return this._blockHandlerExists; + } + + get eventHandlerExists (): boolean { + return this._eventHandlerExists; + } + async addContracts () { assert(this._indexer); assert(this._indexer.watchContract); diff --git a/packages/ipld-eth-client/package.json b/packages/ipld-eth-client/package.json index b0f22be7..1ea1ae2e 100644 --- a/packages/ipld-eth-client/package.json +++ b/packages/ipld-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/ipld-eth-client", - "version": "0.2.99", + "version": "0.2.100", "description": "IPLD ETH Client", "main": "dist/index.js", "scripts": { @@ -20,8 +20,8 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.99", - "@cerc-io/util": "^0.2.99", + "@cerc-io/cache": "^0.2.100", + "@cerc-io/util": "^0.2.100", "cross-fetch": "^3.1.4", "debug": "^4.3.1", "ethers": "^5.4.4", diff --git a/packages/peer/package.json b/packages/peer/package.json index bf053b0a..f45d2cc2 100644 --- a/packages/peer/package.json +++ b/packages/peer/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/peer", - "version": "0.2.99", + "version": "0.2.100", "description": "libp2p module", "main": "dist/index.js", "exports": "./dist/index.js", diff --git a/packages/rpc-eth-client/package.json b/packages/rpc-eth-client/package.json index 2b8a40da..39daf210 100644 --- a/packages/rpc-eth-client/package.json +++ b/packages/rpc-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/rpc-eth-client", - "version": "0.2.99", + "version": "0.2.100", "description": "RPC ETH Client", "main": "dist/index.js", "scripts": { @@ -19,9 +19,9 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/cache": "^0.2.99", - "@cerc-io/ipld-eth-client": "^0.2.99", - "@cerc-io/util": "^0.2.99", + "@cerc-io/cache": "^0.2.100", + "@cerc-io/ipld-eth-client": "^0.2.100", + "@cerc-io/util": "^0.2.100", "chai": "^4.3.4", "ethers": "^5.4.4", "left-pad": "^1.3.0", diff --git a/packages/solidity-mapper/package.json b/packages/solidity-mapper/package.json index a263cad9..afac51db 100644 --- a/packages/solidity-mapper/package.json +++ b/packages/solidity-mapper/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/solidity-mapper", - "version": "0.2.99", + "version": "0.2.100", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { diff --git a/packages/test/package.json b/packages/test/package.json index 02fa2505..3883fbe0 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/test", - "version": "0.2.99", + "version": "0.2.100", "main": "dist/index.js", "license": "AGPL-3.0", "private": true, diff --git a/packages/tracing-client/package.json b/packages/tracing-client/package.json index bf5f4855..a70f5f5d 100644 --- a/packages/tracing-client/package.json +++ b/packages/tracing-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/tracing-client", - "version": "0.2.99", + "version": "0.2.100", "description": "ETH VM tracing client", "main": "dist/index.js", "scripts": { diff --git a/packages/util/package.json b/packages/util/package.json index 9dd1cbe2..ab13eef1 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,13 +1,13 @@ { "name": "@cerc-io/util", - "version": "0.2.99", + "version": "0.2.100", "main": "dist/index.js", "license": "AGPL-3.0", "dependencies": { "@apollo/utils.keyvaluecache": "^1.0.1", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.99", - "@cerc-io/solidity-mapper": "^0.2.99", + "@cerc-io/peer": "^0.2.100", + "@cerc-io/solidity-mapper": "^0.2.100", "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1", "@ethersproject/properties": "^5.7.0", "@ethersproject/providers": "^5.4.4", @@ -54,7 +54,7 @@ "yargs": "^17.0.1" }, "devDependencies": { - "@cerc-io/cache": "^0.2.99", + "@cerc-io/cache": "^0.2.100", "@nomiclabs/hardhat-waffle": "^2.0.1", "@types/bunyan": "^1.8.8", "@types/express": "^4.17.14", diff --git a/packages/util/src/events.ts b/packages/util/src/events.ts index 78100bf6..b6b2e5dc 100644 --- a/packages/util/src/events.ts +++ b/packages/util/src/events.ts @@ -123,9 +123,15 @@ export class EventWatcher { startBlockNumber = syncStatus.chainHeadBlockNumber + 1; } - // Check if filter for logs is enabled - // Check if starting block for watcher is before latest canonical block - if (this._config.jobQueue.useBlockRanges && startBlockNumber < latestCanonicalBlockNumber) { + // Perform checks before starting historical block processing + if ( + // Skip historical block processing if any block handler exists + !this._indexer.graphWatcher?.blockHandlerExists && + // Run historical block processing if useBlockRanges is enabled + this._config.jobQueue.useBlockRanges && + // Only run historical block processing if we are below the frothy region + startBlockNumber < latestCanonicalBlockNumber + ) { await this.startHistoricalBlockProcessing(startBlockNumber, latestCanonicalBlockNumber); return; diff --git a/packages/util/src/fill.ts b/packages/util/src/fill.ts index 197db25b..3771f0af 100644 --- a/packages/util/src/fill.ts +++ b/packages/util/src/fill.ts @@ -80,12 +80,12 @@ export const fillBlocks = async ( const completePercentage = Math.round(blocksProcessed / numberOfBlocks * 100); log(`Processed ${blocksProcessed} of ${numberOfBlocks} blocks (${completePercentage}%)`); - await processBlockByNumber(jobQueue, blockNumber + 1); - - if (blockNumber + 1 >= endBlock) { - // Break the async loop when blockProgress event is for the endBlock and processing is complete. + if (blockNumber + 1 > endBlock) { + // Break the async loop when next block to be processed is more than endBlock. break; } + + await processBlockByNumber(jobQueue, blockNumber + 1); } } diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index c01d2f63..78888491 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -288,6 +288,8 @@ export interface GraphDatabaseInterface { } export interface GraphWatcherInterface { + readonly blockHandlerExists: boolean; + readonly eventHandlerExists: boolean; init (): Promise; setIndexer (indexer: IndexerInterface): void; }