From 6decf61d4cb51cb78b9507165c95698632677eae Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Fri, 24 Nov 2023 11:13:40 +0530 Subject: [PATCH] Update subgraph watcher demo readme (#480) * Update subgraph watcher demo readme * Pass extra block data required in subgraph block handler (#486) * Use codegen config directory name instead of file path * Get block data from extra event data * Add extra event data in processBlockAfterEvents --------- Co-authored-by: neeraj * Update example subgraph mapping code for graph-cli upgrade * Return 0 value for block size in ipld-eth-client --------- Co-authored-by: neeraj Co-authored-by: Prathamesh Musale --- .../src/templates/indexer-template.handlebars | 4 +-- packages/codegen/src/utils/subgraph.ts | 7 ++-- packages/codegen/subgraph-demo.md | 34 +++++++------------ packages/graph-node/src/watcher.ts | 11 +++--- .../test/subgraph/example1/src/mapping.ts | 6 ++-- packages/ipld-eth-client/src/eth-client.ts | 10 +++++- packages/util/src/common.ts | 2 +- packages/util/src/types.ts | 2 +- 8 files changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index fa32b83b..148b480c 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -499,10 +499,10 @@ export class Indexer implements IndexerInterface { } {{#if (subgraphPath)}} - async processBlockAfterEvents (blockHash: string, blockNumber: number): Promise { + async processBlockAfterEvents (blockHash: string, blockNumber: number, extraData: ExtraEventData): Promise { console.time('time:indexer#processBlockAfterEvents-mapping_code'); // Call subgraph handler for block. - await this._graphWatcher.handleBlock(blockHash, blockNumber); + await this._graphWatcher.handleBlock(blockHash, blockNumber, extraData); console.timeEnd('time:indexer#processBlockAfterEvents-mapping_code'); console.time('time:indexer#processBlockAfterEvents-dump_subgraph_state'); diff --git a/packages/codegen/src/utils/subgraph.ts b/packages/codegen/src/utils/subgraph.ts index 8e7749f1..609a4d83 100644 --- a/packages/codegen/src/utils/subgraph.ts +++ b/packages/codegen/src/utils/subgraph.ts @@ -69,7 +69,8 @@ export async function buildSubgraph ( network?: string } ): Promise { - const subgraphDirectory = path.resolve(codegenConfigPath, subgraphConfig.directory); + const codegenConfigDirName = path.dirname(codegenConfigPath); + const subgraphDirectory = path.resolve(codegenConfigDirName, subgraphConfig.directory); const codegenWorkingDir = process.cwd(); // Change directory to subgraph repo shell.cd(subgraphDirectory); @@ -102,7 +103,7 @@ export async function buildSubgraph ( const { code: installCode } = shell.exec(`${packageManager} install --force`); assert(installCode === 0, 'Installing dependencies exited with error'); - const subgraphConfigPath = path.resolve(codegenConfigPath, subgraphConfig.configFile); + const subgraphConfigPath = path.resolve(codegenConfigDirName, subgraphConfig.configFile); // Run graph-cli codegen const { code: codegenCode } = shell.exec(`${packageManager === 'npm' ? 'npx' : packageManager} graph codegen ${subgraphConfigPath}`); @@ -112,7 +113,7 @@ export async function buildSubgraph ( let buildCommand = `${packageManager === 'npm' ? 'npx' : packageManager} graph build ${subgraphConfigPath}`; if (subgraphConfig.networkFilePath) { - const subgraphNetworkFilePath = path.resolve(codegenConfigPath, subgraphConfig.networkFilePath); + const subgraphNetworkFilePath = path.resolve(codegenConfigDirName, subgraphConfig.networkFilePath); assert(subgraphConfig.network, 'Config subgraph.network should be set if using networkFilePath'); const subgraphNetwork = subgraphConfig.network; buildCommand = `${buildCommand} --network-file ${subgraphNetworkFilePath} --network ${subgraphNetwork}`; diff --git a/packages/codegen/subgraph-demo.md b/packages/codegen/subgraph-demo.md index 58144214..946eeaab 100644 --- a/packages/codegen/subgraph-demo.md +++ b/packages/codegen/subgraph-demo.md @@ -16,10 +16,6 @@ * For this demo, an [example subgraph](../graph-node/test/subgraph/example1) will be used - * In [package.json](../graph-node/test/subgraph/example1/package.json), the graph-ts and graph-cli dependencies are replaced by their respective cerc-io forked dependencies - - * This needs to be done for running any subgraph project with watcher-ts - * In watcher-ts [packages/graph-node](../graph-node/), deploy an `Example` contract: ```bash @@ -32,13 +28,7 @@ export EXAMPLE_ADDRESS= ``` -* In [packages/graph-node/test/subgraph/example1/subgraph.yaml](../graph-node/test/subgraph/example1/subgraph.yaml), set the source address for `Example1` datasource to the `EXAMPLE_ADDRESS`. Then in [packages/graph-node](../graph-node/) run: - - ```bash - yarn build:example - ``` - - * This will run `yarn && yarn codegen && yarn build` script in the example subgraph directory +* In [packages/graph-node/test/subgraph/example1/subgraph.yaml](../graph-node/test/subgraph/example1/subgraph.yaml), set the source address for `Example1` datasource to the `EXAMPLE_ADDRESS` * In [packages/codegen](./), create a `config.yaml` file: @@ -63,10 +53,16 @@ # Flatten the input contract file(s) [true | false] (default: true). flatten: true - # Path to the subgraph build (optional). - # Can set empty contracts array when using subgraphPath. - # Subgraph WASM files should be compiled using @cerc-io/graph-cli - subgraphPath: ../graph-node/test/subgraph/example1/build + # Config for subgraph + subgraph: + # Path to subgraph repo directory containing package.json + directory: ../graph-node/test/subgraph/example1 + + # Package manager that is used in subgraph repo for dependencies + packageManager: yarn + + # Path to subgraph manifest/config file + configFile: ../graph-node/test/subgraph/example1/subgraph.yaml ``` * Run codegen to generate watcher: @@ -116,13 +112,7 @@ yarn && yarn build ``` -* In `packages/test-watcher`, run fill for the subgraph start block: - - ```bash - yarn fill --start-block 10 --end-block 10 - ``` - - * Subgraph start block is the lowest `startBlock` in example [subgraph.yaml](../graph-node/test/subgraph/example1/subgraph.yaml) +* Configure the upstream GQL and RPC endpoints * Run the job-runner: diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index 7f1786c7..be497666 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -238,7 +238,7 @@ export class GraphWatcher { } } - async handleBlock (blockHash: string, blockNumber: number) { + async handleBlock (blockHash: string, blockNumber: number, extraData: ExtraEventData) { // Clear transactions map on handling new block. this._transactionsMap.clear(); @@ -258,11 +258,10 @@ export class GraphWatcher { continue; } - // TODO: Use extraData full block - // // Check if block data is already fetched in handleEvent method for the same block. - // if (!this._context.block || this._context.block.blockHash !== blockHash) { - // this._context.block = await getFullBlock(this._ethClient, this._ethProvider, blockHash, blockNumber); - // } + // Check if block data is already fetched in handleEvent method for the same block. + if (!this._context.block || this._context.block.blockHash !== blockHash) { + this._context.block = await getFullBlock(extraData.ethFullBlock); + } const blockData = this._context.block; assert(blockData); diff --git a/packages/graph-node/test/subgraph/example1/src/mapping.ts b/packages/graph-node/test/subgraph/example1/src/mapping.ts index c42c710b..daa0ccbc 100644 --- a/packages/graph-node/test/subgraph/example1/src/mapping.ts +++ b/packages/graph-node/test/subgraph/example1/src/mapping.ts @@ -50,6 +50,7 @@ export function handleTest (event: Test): void { if (!category) { category = new Category(author.blogCount.toString()); category.name = event.params.param1; + category.count = BigInt.zero(); } category.count = category.count + BigInt.fromString('1'); @@ -59,13 +60,14 @@ export function handleTest (event: Test): void { blog.kind = 'long'; blog.isActive = true; - const blogReviews = blog.reviews; + // eslint-disable-next-line @typescript-eslint/ban-types + const blogReviews: BigInt[] = new Array(0); blogReviews.push(BigInt.fromString('4')); blog.reviews = blogReviews; blog.author = author.id; - const categories = blog.categories; + const categories: string[] = new Array(0); categories.push(category.id); blog.categories = categories; diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts index 633c1bae..1595e65a 100644 --- a/packages/ipld-eth-client/src/eth-client.ts +++ b/packages/ipld-eth-client/src/eth-client.ts @@ -102,7 +102,15 @@ export class EthClient implements EthClientInterface { ); console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`); - return allEthHeaderCids.nodes; + return this.addBlockSize(allEthHeaderCids.nodes); + } + + // TODO: Get block size from ipld-eth-server + private addBlockSize (nodes: any[]): any[] { + // Populate block size with 0 value + return nodes.map(obj => { + return { ...obj, size: '0' }; + }); } async getFullTransaction (txHash: string, blockNumber?: number): Promise { diff --git a/packages/util/src/common.ts b/packages/util/src/common.ts index d8b7f244..e882b8b5 100644 --- a/packages/util/src/common.ts +++ b/packages/util/src/common.ts @@ -185,7 +185,7 @@ export const processBatchEvents = async ( if (indexer.processBlockAfterEvents) { if (!dbBlock.isComplete) { - await indexer.processBlockAfterEvents(dbBlock.blockHash, dbBlock.blockNumber); + await indexer.processBlockAfterEvents(dbBlock.blockHash, dbBlock.blockNumber, data); } } diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 654ea44d..f1ed7d7a 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -212,7 +212,7 @@ export interface IndexerInterface { processInitialState: (contractAddress: string, blockHash: string) => Promise processStateCheckpoint: (contractAddress: string, blockHash: string) => Promise processBlock: (blockProgres: BlockProgressInterface) => Promise - processBlockAfterEvents?: (blockHash: string, blockNumber: number) => Promise + processBlockAfterEvents?: (blockHash: string, blockNumber: number, data: ExtraEventData) => Promise processCanonicalBlock (blockHash: string, blockNumber: number): Promise processCheckpoint (blockHash: string): Promise processCLICheckpoint (contractAddress: string, blockHash?: string): Promise