From 16bb95521357f5068f614e4369a603cf6e40ac11 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 15 Nov 2021 12:11:56 +0530 Subject: [PATCH] Invoke subgraph block handlers (#43) * Add a test case to eden test to call the block handler * Add a block handler in example subgraph and call it in a watcher * Use an array map to call all the block handlers for a contract * Await on all the promises returned by block handlers map --- packages/graph-node/src/call-handler.test.ts | 29 +++-- packages/graph-node/src/eden.test.ts | 94 +++++++++------- packages/graph-node/src/utils.ts | 103 ++++++++++-------- packages/graph-node/src/watcher.ts | 32 +++++- .../test/subgraph/example1/src/mapping.ts | 26 ++++- .../test/subgraph/example1/subgraph.yaml | 2 + packages/graph-test-watcher/src/indexer.ts | 5 + packages/graph-test-watcher/src/job-runner.ts | 9 +- 8 files changed, 204 insertions(+), 96 deletions(-) diff --git a/packages/graph-node/src/call-handler.test.ts b/packages/graph-node/src/call-handler.test.ts index 0fe8758d..8da86bfe 100644 --- a/packages/graph-node/src/call-handler.test.ts +++ b/packages/graph-node/src/call-handler.test.ts @@ -8,7 +8,7 @@ import spies from 'chai-spies'; import { getDummyEventData, getTestDatabase } from '../test/utils'; import { instantiate } from './loader'; -import { createEvent, Block } from './utils'; +import { createEvent, createBlock, Block } from './utils'; import { Database } from './database'; chai.use(spies); @@ -19,7 +19,8 @@ describe('call handler in mapping code', () => { let exports: any; let db: Database; - const eventData = getDummyEventData(); + // Create dummy event data. + const dummyEventData = getDummyEventData(); before(async () => { db = getTestDatabase(); @@ -50,11 +51,11 @@ describe('call handler in mapping code', () => { it('should load the subgraph example wasm', async () => { const filePath = path.resolve(__dirname, '../test/subgraph/example1/build/Example1/Example1.wasm'); - const instance = await instantiate(db, { event: { block: eventData.block } }, filePath); + const instance = await instantiate(db, { event: { block: dummyEventData.block } }, filePath); exports = instance.exports; }); - it('should execute the handler function', async () => { + it('should execute the event handler function', async () => { const { _start, handleTest @@ -65,7 +66,7 @@ describe('call handler in mapping code', () => { _start(); // Create event params data. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'param1', value: 'abc', @@ -81,8 +82,8 @@ describe('call handler in mapping code', () => { // Dummy contract address string. const contractAddress = '0xCA6D29232D1435D8198E3E5302495417dD073d61'; - // Create Test event to be passed to handler. - const test = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event Test to be passed to handler. + const test = await createEvent(exports, contractAddress, dummyEventData); await handleTest(test); @@ -91,6 +92,20 @@ describe('call handler in mapping code', () => { expect(db.saveEntity).to.have.been.called(); }); + it('should execute the block handler function', async () => { + const { _start, handleBlock } = exports; + const blockData = dummyEventData.block; + + // Important to call _start for built subgraphs on instantiation! + // TODO: Check api version https://github.com/graphprotocol/graph-node/blob/6098daa8955bdfac597cec87080af5449807e874/runtime/wasm/src/module/mod.rs#L533 + _start(); + + // Create an ethereum block to be passed to the handler. + const block = await createBlock(exports, blockData); + + await handleBlock(block); + }); + after(() => { sandbox.restore(); }); diff --git a/packages/graph-node/src/eden.test.ts b/packages/graph-node/src/eden.test.ts index 539a0660..a57c166f 100644 --- a/packages/graph-node/src/eden.test.ts +++ b/packages/graph-node/src/eden.test.ts @@ -9,7 +9,7 @@ import chai from 'chai'; import spies from 'chai-spies'; import { instantiate } from './loader'; -import { createEvent, Block } from './utils'; +import { createEvent, Block, createBlock } from './utils'; import edenNetworkAbi from '../test/subgraph/eden/EdenNetwork/abis/EdenNetwork.json'; import merkleDistributorAbi from '../test/subgraph/eden/EdenNetworkDistribution/abis/MerkleDistributor.json'; import distributorGovernanceAbi from '../test/subgraph/eden/EdenNetworkGovernance/abis/DistributorGovernance.json'; @@ -24,7 +24,9 @@ const sandbox = chai.spy.sandbox(); describe('eden wasm loader tests', async () => { let db: Database; - const eventData = getDummyEventData(); + + // Create dummy event data. + const dummyEventData = getDummyEventData(); before(async () => { db = getTestDatabase(); @@ -65,7 +67,7 @@ describe('eden wasm loader tests', async () => { it('should load the subgraph network wasm', async () => { const filePath = path.resolve(__dirname, '../test/subgraph/eden/EdenNetwork/EdenNetwork.wasm'); - ({ exports } = await instantiate(db, { event: { block: eventData.block } }, filePath, data)); + ({ exports } = await instantiate(db, { event: { block: dummyEventData.block } }, filePath, data)); const { _start } = exports; _start(); }); @@ -76,7 +78,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy SlotClaimedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'slot', kind: 'uint8', @@ -114,8 +116,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy SlotClaimedEvent to be passed to handler. - const slotClaimedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event SlotClaimedEvent to be passed to handler. + const slotClaimedEvent = await createEvent(exports, contractAddress, dummyEventData); await slotClaimed(slotClaimedEvent); }); @@ -126,7 +128,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy SlotDelegateUpdatedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'slot', kind: 'uint8', @@ -149,8 +151,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy SlotDelegateUpdatedEvent to be passed to handler. - const slotClaimedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event SlotDelegateUpdatedEvent to be passed to handler. + const slotClaimedEvent = await createEvent(exports, contractAddress, dummyEventData); await slotDelegateUpdated(slotClaimedEvent); }); @@ -161,7 +163,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy StakeEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'staker', kind: 'address', @@ -174,8 +176,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy StakeEvent to be passed to handler. - const stakeEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event StakeEvent to be passed to handler. + const stakeEvent = await createEvent(exports, contractAddress, dummyEventData); await stake(stakeEvent); }); @@ -186,7 +188,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy UnstakeEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'staker', kind: 'address', @@ -199,8 +201,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy UnstakeEvent to be passed to handler. - const unstakeEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event UnstakeEvent to be passed to handler. + const unstakeEvent = await createEvent(exports, contractAddress, dummyEventData); await unstake(unstakeEvent); }); @@ -224,7 +226,7 @@ describe('eden wasm loader tests', async () => { it('should load the subgraph network distribution wasm', async () => { const filePath = path.resolve(__dirname, '../test/subgraph/eden/EdenNetworkDistribution/EdenNetworkDistribution.wasm'); - ({ exports } = await instantiate(db, { event: { block: eventData.block } }, filePath, data)); + ({ exports } = await instantiate(db, { event: { block: dummyEventData.block } }, filePath, data)); const { _start } = exports; _start(); }); @@ -235,7 +237,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy ClaimedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'index', kind: 'uint256', @@ -258,8 +260,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy ClaimedEvent to be passed to handler. - const claimedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event ClaimedEvent to be passed to handler. + const claimedEvent = await createEvent(exports, contractAddress, dummyEventData); await claimed(claimedEvent); }); @@ -270,7 +272,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy SlashedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'account', kind: 'address', @@ -283,8 +285,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy SlashedEvent to be passed to handler. - const slashedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event SlashedEvent to be passed to handler. + const slashedEvent = await createEvent(exports, contractAddress, dummyEventData); await slashed(slashedEvent); }); @@ -295,7 +297,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy MerkleRootUpdatedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'merkleRoot', kind: 'bytes32', @@ -313,8 +315,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy MerkleRootUpdatedEvent to be passed to handler. - const merkleRootUpdatedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event MerkleRootUpdatedEvent to be passed to handler. + const merkleRootUpdatedEvent = await createEvent(exports, contractAddress, dummyEventData); await merkleRootUpdated(merkleRootUpdatedEvent); }); @@ -325,7 +327,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy AccountUpdatedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'account', kind: 'address', @@ -343,8 +345,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy AccountUpdatedEvent to be passed to handler. - const accountUpdatedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event AccountUpdatedEvent to be passed to handler. + const accountUpdatedEvent = await createEvent(exports, contractAddress, dummyEventData); await accountUpdated(accountUpdatedEvent); }); @@ -368,7 +370,7 @@ describe('eden wasm loader tests', async () => { it('should load the subgraph network governance wasm', async () => { const filePath = path.resolve(__dirname, '../test/subgraph/eden/EdenNetworkGovernance/EdenNetworkGovernance.wasm'); - ({ exports } = await instantiate(db, { event: { block: eventData.block } }, filePath, data)); + ({ exports } = await instantiate(db, { event: { block: dummyEventData.block } }, filePath, data)); const { _start } = exports; _start(); }); @@ -379,7 +381,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy BlockProducerAddedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'produces', kind: 'address', @@ -387,8 +389,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy BlockProducerAddedEvent to be passed to handler. - const blockProducerAddedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event BlockProducerAddedEvent to be passed to handler. + const blockProducerAddedEvent = await createEvent(exports, contractAddress, dummyEventData); await blockProducerAdded(blockProducerAddedEvent); }); @@ -399,7 +401,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy BlockProducerRemovedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'producer', kind: 'address', @@ -407,8 +409,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy BlockProducerRemovedEvent to be passed to handler. - const blockProducerRemovedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event BlockProducerRemovedEvent to be passed to handler. + const blockProducerRemovedEvent = await createEvent(exports, contractAddress, dummyEventData); await blockProducerRemoved(blockProducerRemovedEvent); }); @@ -419,7 +421,7 @@ describe('eden wasm loader tests', async () => { } = exports; // Create dummy BlockProducerRewardCollectorChangedEvent params. - eventData.eventParams = [ + dummyEventData.eventParams = [ { name: 'producer', kind: 'address', @@ -437,8 +439,8 @@ describe('eden wasm loader tests', async () => { } ]; - // Create dummy BlockProducerRewardCollectorChangedEvent to be passed to handler. - const blockProducerRewardCollectorChangedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event BlockProducerRewardCollectorChangedEvent to be passed to handler. + const blockProducerRewardCollectorChangedEvent = await createEvent(exports, contractAddress, dummyEventData); await blockProducerRewardCollectorChanged(blockProducerRewardCollectorChangedEvent); }); @@ -448,13 +450,23 @@ describe('eden wasm loader tests', async () => { rewardScheduleChanged } = exports; - eventData.eventParams = []; + dummyEventData.eventParams = []; - // Create dummy RewardScheduleChangedEvent to be passed to handler. - const rewardScheduleChangedEvent = await createEvent(exports, contractAddress, eventData); + // Create an ethereum event RewardScheduleChangedEvent to be passed to handler. + const rewardScheduleChangedEvent = await createEvent(exports, contractAddress, dummyEventData); await rewardScheduleChanged(rewardScheduleChangedEvent); }); + + it('should call the block handler', async () => { + const { handleBlock } = exports; + const blockData = dummyEventData.block; + + // Create an ethereum block to be passed to the handler. + const block = await createBlock(exports, blockData); + + await handleBlock(block); + }); }); after(() => { diff --git a/packages/graph-node/src/utils.ts b/packages/graph-node/src/utils.ts index 3ca8b55a..40dde3ac 100644 --- a/packages/graph-node/src/utils.ts +++ b/packages/graph-node/src/utils.ts @@ -164,51 +164,7 @@ export const createEvent = async (instanceExports: any, contractAddress: string, id_of_type: idOfType } = instanceExports; - // Fill block data. - const blockHashByteArray = await ByteArray.fromHexString(await __newString(blockData.blockHash)); - const blockHash = await Bytes.fromByteArray(blockHashByteArray); - - const parentHashByteArray = await ByteArray.fromHexString(await __newString(blockData.parentHash)); - const parentHash = await Bytes.fromByteArray(parentHashByteArray); - - const blockNumber = await BigInt.fromString(await __newString(blockData.blockNumber)); - - const blockTimestamp = await BigInt.fromString(await __newString(blockData.timestamp)); - - const stateRootByteArray = await ByteArray.fromHexString(await __newString(blockData.stateRoot)); - const stateRoot = await Bytes.fromByteArray(stateRootByteArray); - - const transactionsRootByteArray = await ByteArray.fromHexString(await __newString(blockData.txRoot)); - const transactionsRoot = await Bytes.fromByteArray(transactionsRootByteArray); - - const receiptsRootByteArray = await ByteArray.fromHexString(await __newString(blockData.receiptRoot)); - const receiptsRoot = await Bytes.fromByteArray(receiptsRootByteArray); - - const totalDifficulty = await BigInt.fromString(await __newString(blockData.td)); - - // Missing fields from watcher in block data: - // unclesHash - // author - // gasUsed - // gasLimit - // difficulty - // size - const block = await ethereum.Block.__new( - blockHash, - parentHash, - await Bytes.empty(), - await Address.zero(), - stateRoot, - transactionsRoot, - receiptsRoot, - blockNumber, - await BigInt.fromI32(0), - await BigInt.fromI32(0), - blockTimestamp, - await BigInt.fromI32(0), - totalDifficulty, - null - ); + const block = await createBlock(instanceExports, blockData); // Fill transaction data. const txHashByteArray = await ByteArray.fromHexString(await __newString(tx.hash)); @@ -264,6 +220,63 @@ export const createEvent = async (instanceExports: any, contractAddress: string, ); }; +export const createBlock = async (instanceExports: any, blockData: Block): Promise => { + const { + __newString, + Address, + BigInt, + ethereum, + Bytes, + ByteArray + } = instanceExports; + + // Fill block data. + const blockHashByteArray = await ByteArray.fromHexString(await __newString(blockData.blockHash)); + const blockHash = await Bytes.fromByteArray(blockHashByteArray); + + const parentHashByteArray = await ByteArray.fromHexString(await __newString(blockData.parentHash)); + const parentHash = await Bytes.fromByteArray(parentHashByteArray); + + const blockNumber = await BigInt.fromString(await __newString(blockData.blockNumber)); + + const blockTimestamp = await BigInt.fromString(await __newString(blockData.timestamp)); + + const stateRootByteArray = await ByteArray.fromHexString(await __newString(blockData.stateRoot)); + const stateRoot = await Bytes.fromByteArray(stateRootByteArray); + + const transactionsRootByteArray = await ByteArray.fromHexString(await __newString(blockData.txRoot)); + const transactionsRoot = await Bytes.fromByteArray(transactionsRootByteArray); + + const receiptsRootByteArray = await ByteArray.fromHexString(await __newString(blockData.receiptRoot)); + const receiptsRoot = await Bytes.fromByteArray(receiptsRootByteArray); + + const totalDifficulty = await BigInt.fromString(await __newString(blockData.td)); + + // Missing fields from watcher in block data: + // unclesHash + // author + // gasUsed + // gasLimit + // difficulty + // size + return await ethereum.Block.__new( + blockHash, + parentHash, + await Bytes.empty(), + await Address.zero(), + stateRoot, + transactionsRoot, + receiptsRoot, + blockNumber, + await BigInt.fromI32(0), + await BigInt.fromI32(0), + blockTimestamp, + await BigInt.fromI32(0), + totalDifficulty, + null + ); +}; + export const getSubgraphConfig = async (subgraphPath: string): Promise => { const configFilePath = path.resolve(path.join(subgraphPath, 'subgraph.yaml')); const fileExists = await fs.pathExists(configFilePath); diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index e9238ca2..8ef7a7e0 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -13,7 +13,7 @@ import { ResultObject } from '@vulcanize/assemblyscript/lib/loader'; import { EthClient } from '@vulcanize/ipld-eth-client'; import { IndexerInterface } from '@vulcanize/util'; -import { createEvent, getSubgraphConfig } from './utils'; +import { createBlock, createEvent, getSubgraphConfig } from './utils'; import { Context, instantiate } from './loader'; import { Database } from './database'; @@ -149,6 +149,36 @@ export class GraphWatcher { await exports[eventHandler.handler](ethereumEvent); } + async handleBlock (blockHash: string) { + const { + allEthHeaderCids: { + nodes: [ + blockData + ] + } + } = await this._postgraphileClient.getBlocks({ blockHash }); + + // Call block handler(s) for each contract. + for (const dataSource of this._dataSources) { + // Check if block handler(s) are configured. + if (!dataSource.mapping.blockHandlers) { + continue; + } + + const { instance: { exports } } = this._dataSourceMap[dataSource.source.address]; + + // Create ethereum block to be passed to a wasm block handler. + const ethereumBlock = await createBlock(exports, blockData); + + // Call all the block handlers one after the another for a contract. + const blockHandlerPromises = dataSource.mapping.blockHandlers.map(async (blockHandler: any): Promise => { + await exports[blockHandler.handler](ethereumBlock); + }); + + await Promise.all(blockHandlerPromises); + } + } + setIndexer (indexer: IndexerInterface): void { this._indexer = indexer; } diff --git a/packages/graph-node/test/subgraph/example1/src/mapping.ts b/packages/graph-node/test/subgraph/example1/src/mapping.ts index df8c4afa..0d01e1fa 100644 --- a/packages/graph-node/test/subgraph/example1/src/mapping.ts +++ b/packages/graph-node/test/subgraph/example1/src/mapping.ts @@ -1,4 +1,4 @@ -import { Address, log, BigInt, BigDecimal, ByteArray, dataSource } from '@graphprotocol/graph-ts'; +import { Address, log, BigInt, BigDecimal, ByteArray, dataSource, ethereum } from '@graphprotocol/graph-ts'; import { Example1, @@ -54,6 +54,30 @@ export function handleTest (event: Test): void { // - contract.getMethod(...) } +export function handleBlock (block: ethereum.Block): void { + log.debug('block.hash: {}', [block.hash.toHexString()]); + log.debug('block.parentHash: {}', [block.parentHash.toHexString()]); + log.debug('block.unclesHash: {}', [block.unclesHash.toHexString()]); + log.debug('block.author: {}', [block.author.toHexString()]); + log.debug('block.stateRoot: {}', [block.stateRoot.toHexString()]); + log.debug('block.transactionsRoot: {}', [block.transactionsRoot.toHexString()]); + log.debug('block.receiptsRoot: {}', [block.receiptsRoot.toHexString()]); + log.debug('block.number: {}', [block.number.toString()]); + log.debug('block.gasUsed: {}', [block.gasUsed.toString()]); + log.debug('block.gasLimit: {}', [block.gasLimit.toString()]); + log.debug('block.timestamp: {}', [block.timestamp.toString()]); + log.debug('block.difficulty: {}', [block.difficulty.toString()]); + log.debug('block.totalDifficulty: {}', [block.totalDifficulty.toString()]); + + const blockSize = block.size; + + if (blockSize) { + log.debug('block.size: {}', [blockSize.toString()]); + } else { + log.debug('block.size: {}', ['null']); + } +} + export function testEthCall (): void { log.debug('In test eth call', []); diff --git a/packages/graph-node/test/subgraph/example1/subgraph.yaml b/packages/graph-node/test/subgraph/example1/subgraph.yaml index e202db1e..4b2b5dcb 100644 --- a/packages/graph-node/test/subgraph/example1/subgraph.yaml +++ b/packages/graph-node/test/subgraph/example1/subgraph.yaml @@ -20,4 +20,6 @@ dataSources: eventHandlers: - event: Test(string,uint8) handler: handleTest + blockHandlers: + - handler: handleBlock file: ./src/mapping.ts diff --git a/packages/graph-test-watcher/src/indexer.ts b/packages/graph-test-watcher/src/indexer.ts index 3658429d..05f8a6b9 100644 --- a/packages/graph-test-watcher/src/indexer.ts +++ b/packages/graph-test-watcher/src/indexer.ts @@ -181,6 +181,11 @@ export class Indexer { await handleEvent(this, resultEvent); } + async processBlock (blockHash: string): Promise { + // Call subgraph handler for block. + await this._graphWatcher.handleBlock(blockHash); + } + async processEvent (event: Event): Promise { // Trigger indexing of data based on the event. await this.triggerIndexingOnEvent(event); diff --git a/packages/graph-test-watcher/src/job-runner.ts b/packages/graph-test-watcher/src/job-runner.ts index 5b37f96e..366501b3 100644 --- a/packages/graph-test-watcher/src/job-runner.ts +++ b/packages/graph-test-watcher/src/job-runner.ts @@ -19,7 +19,8 @@ import { QUEUE_EVENT_PROCESSING, JobQueueConfig, DEFAULT_CONFIG_PATH, - getCustomProvider + getCustomProvider, + JOB_KIND_INDEX } from '@vulcanize/util'; import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node'; @@ -50,6 +51,12 @@ export class JobRunner { await this._jobQueue.subscribe(QUEUE_BLOCK_PROCESSING, async (job) => { await this._baseJobRunner.processBlock(job); + const { data: { kind, blockHash } } = job; + + if (kind === JOB_KIND_INDEX) { + await this._indexer.processBlock(blockHash); + } + await this._jobQueue.markComplete(job); }); }