cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2024-11-19 12:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support events handlers in multiple data sources for a contract address (#526)

Browse Source 
* Support processing events in multiple subgraph datasources for a single contract address

* Fix parsing event topic in graph-node watcher

* Update codegen templates

* Fix dummy indexer method in graph-node test

* Upgrade package versions to 0.2.102
This commit is contained in:
Nabarun Gogoi 2024-06-26 17:56:37 +05:30 committed by GitHub
parent b9a899aec1
commit 2217cd3ffb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 208 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [ "packages": [
"packages/*" "packages/*"
], ],
"version": "0.2.101", "version": "0.2.102",
"npmClient": "yarn", "npmClient": "yarn",
"useWorkspaces": true, "useWorkspaces": true,
"command": { "command": {

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/cache", "name": "@cerc-io/cache",
"version": "0.2.101", "version": "0.2.102",
"description": "Generic object cache", "description": "Generic object cache",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {

12
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/cli", "name": "@cerc-io/cli",
"version": "0.2.101", "version": "0.2.102",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"scripts": { "scripts": {
@ -15,13 +15,13 @@
}, },
"dependencies": { "dependencies": {
"@apollo/client": "^3.7.1", "@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.101", "@cerc-io/cache": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.101", "@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4", "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15", "@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.101", "@cerc-io/peer": "^0.2.102",
"@cerc-io/rpc-eth-client": "^0.2.101", "@cerc-io/rpc-eth-client": "^0.2.102",
"@cerc-io/util": "^0.2.101", "@cerc-io/util": "^0.2.102",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1", "@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0", "@ipld/dag-cbor": "^8.0.0",

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/codegen", "name": "@cerc-io/codegen",
"version": "0.2.101", "version": "0.2.102",
"description": "Code generator", "description": "Code generator",
"private": true, "private": true,
"main": "index.js", "main": "index.js",
@ -20,7 +20,7 @@
}, },
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@cerc-io/util": "^0.2.101", "@cerc-io/util": "^0.2.102",
"@graphql-tools/load-files": "^6.5.2", "@graphql-tools/load-files": "^6.5.2",
"@npmcli/package-json": "^5.0.0", "@npmcli/package-json": "^5.0.0",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

1
packages/codegen/src/data/entities/Contract.yaml
View File

@ -2,6 +2,7 @@ className: Contract
indexOn: indexOn:
- columns: - columns:
- address - address
- kind
unique: true unique: true
columns: columns:
- name: id - name: id

24
packages/codegen/src/templates/indexer-template.handlebars
View File

@ -525,23 +525,27 @@ export class Indexer implements IndexerInterface {
} }
{{/if}} {{/if}}
parseEventNameAndArgs (kind: string, logObj: any): { eventParsed: boolean, eventDetails: any } { parseEventNameAndArgs (watchedContracts: Contract[], logObj: any): { eventParsed: boolean, eventDetails: any } {
const { topics, data } = logObj; const { topics, data } = logObj;
let logDescription: ethers.utils.LogDescription | undefined;
const contract = this._contractMap.get(kind); for (const watchedContract of watchedContracts) {
const contract = this._contractMap.get(watchedContract.kind);
assert(contract); assert(contract);
let logDescription: ethers.utils.LogDescription;
try { try {
logDescription = contract.parseLog({ data, topics }); logDescription = contract.parseLog({ data, topics });
break;
} catch (err) { } catch (err) {
// Return if no matching event found // Continue loop only if no matching event found
if ((err as Error).message.includes('no matching event')) { if (!((err as Error).message.includes('no matching event'))) {
log(`WARNING: Skipping event for contract ${kind} as no matching event found in the ABI`); throw err;
return { eventParsed: false, eventDetails: {} }; }
}
} }
throw err; if (!logDescription) {
return { eventParsed: false, eventDetails: {} };
} }
const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription); const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription);
@ -647,8 +651,8 @@ export class Indexer implements IndexerInterface {
return this._baseIndexer.getEventsByFilter(blockHash, contract, name); return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
} }
isWatchedContract (address : string): Contract | undefined { isContractAddressWatched (address : string): Contract[] | undefined {
return this._baseIndexer.isWatchedContract(address); return this._baseIndexer.isContractAddressWatched(address);
} }
getWatchedContracts (): Contract[] { getWatchedContracts (): Contract[] {

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.101", "@cerc-io/cli": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.101", "@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/solidity-mapper": "^0.2.101", "@cerc-io/solidity-mapper": "^0.2.102",
"@cerc-io/util": "^0.2.101", "@cerc-io/util": "^0.2.102",
{{#if (subgraphPath)}} {{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.101", "@cerc-io/graph-node": "^0.2.102",
{{/if}} {{/if}}
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1", "debug": "^4.3.1",

10
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{ {
"name": "@cerc-io/graph-node", "name": "@cerc-io/graph-node",
"version": "0.2.101", "version": "0.2.102",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"devDependencies": { "devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.101", "@cerc-io/solidity-mapper": "^0.2.102",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0", "@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2", "@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.101", "@cerc-io/cache": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.101", "@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/util": "^0.2.101", "@cerc-io/util": "^0.2.102",
"@types/json-diff": "^0.5.2", "@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0", "@types/yargs": "^17.0.0",
"bn.js": "^4.11.9", "bn.js": "^4.11.9",

8
packages/graph-node/src/loader.ts
View File

@ -50,6 +50,7 @@ export interface Context {
rpcSupportsBlockHashParam: boolean; rpcSupportsBlockHashParam: boolean;
block?: Block; block?: Block;
contractAddress?: string; contractAddress?: string;
dataSourceName?: string;
} }
const log = debug('vulcanize:graph-node'); const log = debug('vulcanize:graph-node');
@ -719,13 +720,14 @@ export const instantiate = async (
}, },
'dataSource.context': async () => { 'dataSource.context': async () => {
assert(context.contractAddress); assert(context.contractAddress);
const contract = indexer.isWatchedContract(context.contractAddress); const watchedContracts = indexer.isContractAddressWatched(context.contractAddress);
const dataSourceContract = watchedContracts?.find(contract => contract.kind === context.dataSourceName);
if (!contract) { if (!dataSourceContract) {
return null; return null;
} }
return database.toGraphContext(instanceExports, contract.context); return database.toGraphContext(instanceExports, dataSourceContract.context);
}, },
'dataSource.network': async () => { 'dataSource.network': async () => {
assert(dataSource); assert(dataSource);

35
packages/graph-node/src/watcher.ts
View File

@ -169,7 +169,6 @@ export class GraphWatcher {
async addContracts () { async addContracts () {
assert(this._indexer); assert(this._indexer);
assert(this._indexer.watchContract); assert(this._indexer.watchContract);
assert(this._indexer.isWatchedContract);
// Watching the contract(s) if not watched already. // Watching the contract(s) if not watched already.
for (const dataSource of this._dataSources) { for (const dataSource of this._dataSources) {
@ -177,7 +176,7 @@ export class GraphWatcher {
// Skip for templates as they are added dynamically. // Skip for templates as they are added dynamically.
if (address) { if (address) {
const watchedContract = await this._indexer.isWatchedContract(address); const watchedContract = this._indexer.isContractAddressWatched(address);
if (!watchedContract) { if (!watchedContract) {
await this._indexer.watchContract(address, name, true, startBlock); await this._indexer.watchContract(address, name, true, startBlock);
@ -197,26 +196,40 @@ export class GraphWatcher {
const blockData = this._context.block; const blockData = this._context.block;
assert(blockData); assert(blockData);
assert(this._indexer && this._indexer.isWatchedContract); assert(this._indexer);
const watchedContract = this._indexer.isWatchedContract(contract); const watchedContracts = this._indexer.isContractAddressWatched(contract);
assert(watchedContract); assert(watchedContracts);
// Get dataSource in subgraph yaml based on contract address. // Get dataSources in subgraph yaml based on contract kind (same as dataSource.name)
const dataSource = this._dataSources.find(dataSource => dataSource.name === watchedContract.kind); const dataSources = this._dataSources
.filter(dataSource => watchedContracts.some(contract => contract.kind === dataSource.name));
if (!dataSource) { if (!dataSources.length) {
log(`Subgraph doesn't have configuration for contract ${contract}`); log(`Subgraph doesn't have configuration for contract ${contract}`);
return; return;
} }
for (const dataSource of dataSources) {
this._context.contractAddress = contract; this._context.contractAddress = contract;
this._context.dataSourceName = dataSource.name;
const { instance, contractInterface } = this._dataSourceMap[watchedContract.kind]; const { instance, contractInterface } = this._dataSourceMap[dataSource.name];
assert(instance); assert(instance);
const { exports: instanceExports } = instance; const { exports: instanceExports } = instance;
let eventTopic: string;
try {
eventTopic = contractInterface.getEventTopic(eventSignature);
} catch (err) {
// Continue loop only if no matching event found
if (!((err as Error).message.includes('no matching event'))) {
throw err;
}
continue;
}
// Get event handler based on event topic (from event signature). // Get event handler based on event topic (from event signature).
const eventTopic = contractInterface.getEventTopic(eventSignature);
const eventHandler = dataSource.mapping.eventHandlers.find((eventHandler: any) => { const eventHandler = dataSource.mapping.eventHandlers.find((eventHandler: any) => {
// The event signature we get from logDescription is different than that given in the subgraph yaml file. // The event signature we get from logDescription is different than that given in the subgraph yaml file.
// For eg. event in subgraph.yaml: Stake(indexed address,uint256); from logDescription: Stake(address,uint256) // For eg. event in subgraph.yaml: Stake(indexed address,uint256); from logDescription: Stake(address,uint256)
@ -257,6 +270,7 @@ export class GraphWatcher {
throw error; throw error;
} }
} }
}
async handleBlock (blockHash: string, blockNumber: number, extraData: ExtraEventData) { async handleBlock (blockHash: string, blockNumber: number, extraData: ExtraEventData) {
// Clear transactions map on handling new block. // Clear transactions map on handling new block.
@ -311,6 +325,7 @@ export class GraphWatcher {
for (const contractAddress of contractAddressList) { for (const contractAddress of contractAddressList) {
this._context.contractAddress = contractAddress; this._context.contractAddress = contractAddress;
this._context.dataSourceName = dataSource.name;
// Call all the block handlers one after another for a contract. // Call all the block handlers one after another for a contract.
const blockHandlerPromises = dataSource.mapping.blockHandlers.map(async (blockHandler: any): Promise<void> => { const blockHandlerPromises = dataSource.mapping.blockHandlers.map(async (blockHandler: any): Promise<void> => {

2
packages/graph-node/test/utils/indexer.ts
View File

@ -248,7 +248,7 @@ export class Indexer implements IndexerInterface {
return undefined; return undefined;
} }
isWatchedContract (address : string): ContractInterface | undefined { isContractAddressWatched (address : string): ContractInterface[] | undefined {
return undefined; return undefined;
} }

6
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/ipld-eth-client", "name": "@cerc-io/ipld-eth-client",
"version": "0.2.101", "version": "0.2.102",
"description": "IPLD ETH Client", "description": "IPLD ETH Client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.7.1", "@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.101", "@cerc-io/cache": "^0.2.102",
"@cerc-io/util": "^0.2.101", "@cerc-io/util": "^0.2.102",
"cross-fetch": "^3.1.4", "cross-fetch": "^3.1.4",
"debug": "^4.3.1", "debug": "^4.3.1",
"ethers": "^5.4.4", "ethers": "^5.4.4",

2
packages/peer/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/peer", "name": "@cerc-io/peer",
"version": "0.2.101", "version": "0.2.102",
"description": "libp2p module", "description": "libp2p module",
"main": "dist/index.js", "main": "dist/index.js",
"exports": "./dist/index.js", "exports": "./dist/index.js",

8
packages/rpc-eth-client/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/rpc-eth-client", "name": "@cerc-io/rpc-eth-client",
"version": "0.2.101", "version": "0.2.102",
"description": "RPC ETH Client", "description": "RPC ETH Client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {
@ -19,9 +19,9 @@
}, },
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@cerc-io/cache": "^0.2.101", "@cerc-io/cache": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.101", "@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/util": "^0.2.101", "@cerc-io/util": "^0.2.102",
"chai": "^4.3.4", "chai": "^4.3.4",
"ethers": "^5.4.4", "ethers": "^5.4.4",
"left-pad": "^1.3.0", "left-pad": "^1.3.0",

2
packages/solidity-mapper/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/solidity-mapper", "name": "@cerc-io/solidity-mapper",
"version": "0.2.101", "version": "0.2.102",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"devDependencies": { "devDependencies": {

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/test", "name": "@cerc-io/test",
"version": "0.2.101", "version": "0.2.102",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"private": true, "private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/tracing-client", "name": "@cerc-io/tracing-client",
"version": "0.2.101", "version": "0.2.102",
"description": "ETH VM tracing client", "description": "ETH VM tracing client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {

8
packages/util/package.json
View File

@ -1,13 +1,13 @@
{ {
"name": "@cerc-io/util", "name": "@cerc-io/util",
"version": "0.2.101", "version": "0.2.102",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"dependencies": { "dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1", "@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15", "@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.101", "@cerc-io/peer": "^0.2.102",
"@cerc-io/solidity-mapper": "^0.2.101", "@cerc-io/solidity-mapper": "^0.2.102",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1", "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0", "@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
@ -54,7 +54,7 @@
"yargs": "^17.0.1" "yargs": "^17.0.1"
}, },
"devDependencies": { "devDependencies": {
"@cerc-io/cache": "^0.2.101", "@cerc-io/cache": "^0.2.102",
"@nomiclabs/hardhat-waffle": "^2.0.1", "@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8", "@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14", "@types/express": "^4.17.14",

20
packages/util/src/common.ts
View File

@ -12,7 +12,7 @@ import {
UNKNOWN_EVENT_NAME UNKNOWN_EVENT_NAME
} from './constants'; } from './constants';
import { JobQueue } from './job-queue'; import { JobQueue } from './job-queue';
import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock } from './types'; import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock, ContractInterface } from './types';
import { wait } from './misc'; import { wait } from './misc';
import { OrderDirection } from './database'; import { OrderDirection } from './database';
import { JobQueueConfig } from './config'; import { JobQueueConfig } from './config';
@ -242,14 +242,14 @@ const _processEvents = async (
// throw new Error(`Events received out of order for block number ${block.blockNumber} hash ${block.blockHash}, got event index ${eventIndex} and lastProcessedEventIndex ${block.lastProcessedEventIndex}, aborting`); // throw new Error(`Events received out of order for block number ${block.blockNumber} hash ${block.blockHash}, got event index ${eventIndex} and lastProcessedEventIndex ${block.lastProcessedEventIndex}, aborting`);
// } // }
const watchedContract = indexer.isWatchedContract(event.contract); const watchedContracts = indexer.isContractAddressWatched(event.contract);
if (watchedContract) { if (watchedContracts) {
// We might not have parsed this event yet. This can happen if the contract was added // We might not have parsed this event yet. This can happen if the contract was added
// as a result of a previous event in the same block. // as a result of a previous event in the same block.
if (event.eventName === UNKNOWN_EVENT_NAME) { if (event.eventName === UNKNOWN_EVENT_NAME) {
// Parse the unknown event and save updated event to the db // Parse the unknown event and save updated event to the db
const { eventParsed, event: parsedEvent } = _parseUnknownEvent(indexer, event, watchedContract.kind); const { eventParsed, event: parsedEvent } = _parseUnknownEvent(indexer, event, watchedContracts);
if (eventParsed) { if (eventParsed) {
updatedDbEvents.push(parsedEvent); updatedDbEvents.push(parsedEvent);
@ -353,14 +353,14 @@ const _processEventsInSubgraphOrder = async (
// Parse events of initially unwatched contracts // Parse events of initially unwatched contracts
for (const event of unwatchedContractEvents) { for (const event of unwatchedContractEvents) {
const watchedContract = indexer.isWatchedContract(event.contract); const watchedContracts = indexer.isContractAddressWatched(event.contract);
if (watchedContract) { if (watchedContracts) {
// We might not have parsed this event yet. This can happen if the contract was added // We might not have parsed this event yet. This can happen if the contract was added
// as a result of a previous event in the same block. // as a result of a previous event in the same block.
if (event.eventName === UNKNOWN_EVENT_NAME) { if (event.eventName === UNKNOWN_EVENT_NAME) {
// Parse the unknown event and save updated event to the db // Parse the unknown event and save updated event to the db
const { eventParsed, event: parsedEvent } = _parseUnknownEvent(indexer, event, watchedContract.kind); const { eventParsed, event: parsedEvent } = _parseUnknownEvent(indexer, event, watchedContracts);
if (eventParsed) { if (eventParsed) {
updatedDbEvents.push(parsedEvent); updatedDbEvents.push(parsedEvent);
@ -397,12 +397,14 @@ const _getEventsBatch = async (indexer: IndexerInterface, blockHash: string, eve
); );
}; };
const _parseUnknownEvent = (indexer: IndexerInterface, event: EventInterface, contractKind: string): { eventParsed: boolean, event: EventInterface } => { const _parseUnknownEvent = (indexer: IndexerInterface, event: EventInterface, watchedContracts: ContractInterface[]): { eventParsed: boolean, event: EventInterface } => {
const logObj = JSONbigNative.parse(event.extraInfo); const logObj = JSONbigNative.parse(event.extraInfo);
assert(indexer.parseEventNameAndArgs); assert(indexer.parseEventNameAndArgs);
const { eventParsed, eventDetails: { eventName, eventInfo, eventSignature } } = indexer.parseEventNameAndArgs(contractKind, logObj); const { eventParsed, eventDetails: { eventName, eventInfo, eventSignature } } = indexer.parseEventNameAndArgs(watchedContracts, logObj);
if (!eventParsed) { if (!eventParsed) {
// Skip unparsable events
log(`WARNING: Skipping event for contract ${event.contract} as no matching event found in the ABI`);
return { eventParsed: false, event }; return { eventParsed: false, event };
} }

2
packages/util/src/database.ts
View File

@ -665,7 +665,7 @@ export class Database {
async saveContract (repo: Repository<ContractInterface>, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<ContractInterface> { async saveContract (repo: Repository<ContractInterface>, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<ContractInterface> {
const contract = await repo const contract = await repo
.createQueryBuilder() .createQueryBuilder()
.where('address = :address', { address }) .where('address = :address AND kind = :kind', { address, kind })
.getOne(); .getOne();
const entity = repo.create({ address, kind, checkpoint, startingBlock, context }); const entity = repo.create({ address, kind, checkpoint, startingBlock, context });

115
packages/util/src/indexer.ts
View File

@ -117,7 +117,7 @@ export class Indexer {
_ethProvider: ethers.providers.JsonRpcProvider; _ethProvider: ethers.providers.JsonRpcProvider;
_jobQueue: JobQueue; _jobQueue: JobQueue;
_watchedContracts: { [key: string]: ContractInterface } = {}; _watchedContractsByAddressMap: { [key: string]: ContractInterface[] } = {};
_stateStatusMap: { [key: string]: StateStatus } = {}; _stateStatusMap: { [key: string]: StateStatus } = {};
_currentEndpointIndex = { _currentEndpointIndex = {
@ -203,8 +203,12 @@ export class Indexer {
const contracts = await this._db.getContracts(); const contracts = await this._db.getContracts();
this._watchedContracts = contracts.reduce((acc: { [key: string]: ContractInterface }, contract) => { this._watchedContractsByAddressMap = contracts.reduce((acc: { [key: string]: ContractInterface[] }, contract) => {
acc[contract.address] = contract; if (!acc[contract.address]) {
acc[contract.address] = [];
}
acc[contract.address].push(contract);
return acc; return acc;
}, {}); }, {});
@ -441,7 +445,7 @@ export class Indexer {
toBlock: number, toBlock: number,
eventSignaturesMap: Map<string, string[]>, eventSignaturesMap: Map<string, string[]>,
parseEventNameAndArgs: ( parseEventNameAndArgs: (
kind: string, watchedContracts: ContractInterface[],
logObj: { topics: string[]; data: string } logObj: { topics: string[]; data: string }
) => { eventParsed: boolean, eventDetails: any } ) => { eventParsed: boolean, eventDetails: any }
): Promise<{ ): Promise<{
@ -553,7 +557,7 @@ export class Indexer {
async fetchEvents ( async fetchEvents (
blockHash: string, blockNumber: number, blockHash: string, blockNumber: number,
eventSignaturesMap: Map<string, string[]>, eventSignaturesMap: Map<string, string[]>,
parseEventNameAndArgs: (kind: string, logObj: any) => { eventParsed: boolean, eventDetails: any } parseEventNameAndArgs: (watchedContracts: ContractInterface[], logObj: any) => { eventParsed: boolean, eventDetails: any }
): Promise<{ events: DeepPartial<EventInterface>[], transactions: EthFullTransaction[]}> { ): Promise<{ events: DeepPartial<EventInterface>[], transactions: EthFullTransaction[]}> {
const { addresses, topics } = this._createLogsFilters(eventSignaturesMap); const { addresses, topics } = this._createLogsFilters(eventSignaturesMap);
const { logs, transactions } = await this._fetchLogsAndTransactions(blockHash, blockNumber, addresses, topics); const { logs, transactions } = await this._fetchLogsAndTransactions(blockHash, blockNumber, addresses, topics);
@ -572,7 +576,7 @@ export class Indexer {
blockHash: string, blockNumber: number, blockHash: string, blockNumber: number,
addresses: string[], addresses: string[],
eventSignaturesMap: Map<string, string[]>, eventSignaturesMap: Map<string, string[]>,
parseEventNameAndArgs: (kind: string, logObj: any) => { eventParsed: boolean, eventDetails: any } parseEventNameAndArgs: (watchedContracts: ContractInterface[], logObj: any) => { eventParsed: boolean, eventDetails: any }
): Promise<DeepPartial<EventInterface>[]> { ): Promise<DeepPartial<EventInterface>[]> {
const { topics } = this._createLogsFilters(eventSignaturesMap); const { topics } = this._createLogsFilters(eventSignaturesMap);
const { logs, transactions } = await this._fetchLogsAndTransactions(blockHash, blockNumber, addresses, topics); const { logs, transactions } = await this._fetchLogsAndTransactions(blockHash, blockNumber, addresses, topics);
@ -618,7 +622,7 @@ export class Indexer {
createDbEventsFromLogsAndTxs ( createDbEventsFromLogsAndTxs (
blockHash: string, blockHash: string,
logs: any, transactions: any, logs: any, transactions: any,
parseEventNameAndArgs: (kind: string, logObj: any) => { eventParsed: boolean, eventDetails: any } parseEventNameAndArgs: (watchedContracts: ContractInterface[], logObj: any) => { eventParsed: boolean, eventDetails: any }
): DeepPartial<EventInterface>[] { ): DeepPartial<EventInterface>[] {
const transactionMap: {[key: string]: any} = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => { const transactionMap: {[key: string]: any} = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => {
acc[transaction.txHash] = transaction; acc[transaction.txHash] = transaction;
@ -665,12 +669,13 @@ export class Indexer {
const extraInfo: { [key: string]: any } = { topics, data, tx, logIndex }; const extraInfo: { [key: string]: any } = { topics, data, tx, logIndex };
const contract = ethers.utils.getAddress(address); const contract = ethers.utils.getAddress(address);
const watchedContract = this.isWatchedContract(contract); const watchedContracts = this.isContractAddressWatched(contract);
if (watchedContract) { if (watchedContracts) {
const { eventParsed, eventDetails } = parseEventNameAndArgs(watchedContract.kind, logObj); const { eventParsed, eventDetails } = parseEventNameAndArgs(watchedContracts, logObj);
if (!eventParsed) { if (!eventParsed) {
// Skip unparsable events // Skip unparsable events
log(`WARNING: Skipping event for contract ${contract} as no matching event found in ABI`);
continue; continue;
} }
@ -856,19 +861,22 @@ export class Indexer {
return this._db.getEventsInRange(fromBlockNumber, toBlockNumber); return this._db.getEventsInRange(fromBlockNumber, toBlockNumber);
} }
isWatchedContract (address : string): ContractInterface | undefined { isContractAddressWatched (address : string): ContractInterface[] | undefined {
return this._watchedContracts[address]; return this._watchedContractsByAddressMap[address];
} }
getContractsByKind (kind: string): ContractInterface[] { getContractsByKind (kind: string): ContractInterface[] {
const watchedContracts = Object.values(this._watchedContracts) const watchedContracts = Object.values(this._watchedContractsByAddressMap)
.filter(contract => contract.kind === kind); .reduce(
(acc, contracts) => acc.concat(contracts.filter(contract => contract.kind === kind)),
[]
);
return watchedContracts; return watchedContracts;
} }
getWatchedContracts (): ContractInterface[] { getWatchedContracts (): ContractInterface[] {
return Object.values(this._watchedContracts); return Object.values(this._watchedContractsByAddressMap).flat();
} }
async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> { async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
@ -902,7 +910,19 @@ export class Indexer {
} }
cacheContract (contract: ContractInterface): void { cacheContract (contract: ContractInterface): void {
this._watchedContracts[contract.address] = contract; if (!this._watchedContractsByAddressMap[contract.address]) {
this._watchedContractsByAddressMap[contract.address] = [];
}
// Check if contract with kind is already cached and skip
const isAlreadyCached = this._watchedContractsByAddressMap[contract.address]
.some(watchedContract => contract.id === watchedContract.id);
if (isAlreadyCached) {
return;
}
this._watchedContractsByAddressMap[contract.address].push(contract);
} }
async getStorageValue (storageLayout: StorageLayout, blockHash: string, token: string, variable: string, ...mappingKeys: any[]): Promise<ValueResult> { async getStorageValue (storageLayout: StorageLayout, blockHash: string, token: string, variable: string, ...mappingKeys: any[]): Promise<ValueResult> {
@ -940,7 +960,7 @@ export class Indexer {
} }
// Get all the contracts. // Get all the contracts.
const contracts = Object.values(this._watchedContracts); const [contracts] = Object.values(this._watchedContractsByAddressMap);
// Getting the block for checkpoint. // Getting the block for checkpoint.
const block = await this.getBlockProgress(blockHash); const block = await this.getBlockProgress(blockHash);
@ -994,10 +1014,11 @@ export class Indexer {
} }
// Get the contract. // Get the contract.
const contract = this._watchedContracts[contractAddress]; const watchedContracts = this._watchedContractsByAddressMap[contractAddress];
assert(contract, `Contract ${contractAddress} not watched`); assert(watchedContracts, `Contract ${contractAddress} not watched`);
const [firstWatchedContract] = watchedContracts.sort((a, b) => a.startingBlock - b.startingBlock);
if (block.blockNumber < contract.startingBlock) { if (block.blockNumber < firstWatchedContract.startingBlock) {
return; return;
} }
@ -1016,10 +1037,13 @@ export class Indexer {
} }
// Get all the contracts. // Get all the contracts.
const contracts = Object.values(this._watchedContracts); const watchedContractsByAddress = Object.values(this._watchedContractsByAddressMap);
// Create an initial state for each contract. // Create an initial state for each contract.
for (const contract of contracts) { for (const watchedContracts of watchedContractsByAddress) {
// Get the first watched contract
const [contract] = watchedContracts.sort((a, b) => a.startingBlock - b.startingBlock);
// Check if contract has checkpointing on. // Check if contract has checkpointing on.
if (contract.checkpoint) { if (contract.checkpoint) {
// Check if starting block not reached yet. // Check if starting block not reached yet.
@ -1064,8 +1088,9 @@ export class Indexer {
assert(block); assert(block);
// Get the contract. // Get the contract.
const contract = this._watchedContracts[contractAddress]; const watchedContracts = this._watchedContractsByAddressMap[contractAddress];
assert(contract, `Contract ${contractAddress} not watched`); assert(watchedContracts, `Contract ${contractAddress} not watched`);
const [contract] = watchedContracts.sort((a, b) => a.startingBlock - b.startingBlock);
if (block.blockNumber < contract.startingBlock) { if (block.blockNumber < contract.startingBlock) {
return; return;
@ -1100,8 +1125,9 @@ export class Indexer {
} }
// Get the contract. // Get the contract.
const contract = this._watchedContracts[contractAddress]; const watchedContracts = this._watchedContractsByAddressMap[contractAddress];
assert(contract, `Contract ${contractAddress} not watched`); assert(watchedContracts, `Contract ${contractAddress} not watched`);
const [contract] = watchedContracts.sort((a, b) => a.startingBlock - b.startingBlock);
if (block.blockNumber < contract.startingBlock) { if (block.blockNumber < contract.startingBlock) {
return; return;
@ -1138,8 +1164,9 @@ export class Indexer {
} }
// Get the contract. // Get the contract.
const contract = this._watchedContracts[contractAddress]; const watchedContracts = this._watchedContractsByAddressMap[contractAddress];
assert(contract, `Contract ${contractAddress} not watched`); assert(watchedContracts, `Contract ${contractAddress} not watched`);
const [contract] = watchedContracts.sort((a, b) => a.startingBlock - b.startingBlock);
if (currentBlock.blockNumber < contract.startingBlock) { if (currentBlock.blockNumber < contract.startingBlock) {
return; return;
@ -1341,16 +1368,16 @@ export class Indexer {
return; return;
} }
const contracts = Object.values(this._watchedContracts); const contractAddresses = Object.keys(this._watchedContractsByAddressMap);
// TODO: Fire a single query for all contracts. // TODO: Fire a single query for all contracts.
for (const contract of contracts) { for (const contractAddress of contractAddresses) {
const initState = await this._db.getLatestState(contract.address, StateKind.Init); const initState = await this._db.getLatestState(contractAddress, StateKind.Init);
const diffState = await this._db.getLatestState(contract.address, StateKind.Diff); const diffState = await this._db.getLatestState(contractAddress, StateKind.Diff);
const diffStagedState = await this._db.getLatestState(contract.address, StateKind.DiffStaged); const diffStagedState = await this._db.getLatestState(contractAddress, StateKind.DiffStaged);
const checkpointState = await this._db.getLatestState(contract.address, StateKind.Checkpoint); const checkpointState = await this._db.getLatestState(contractAddress, StateKind.Checkpoint);
this._stateStatusMap[contract.address] = { this._stateStatusMap[contractAddress] = {
init: initState?.block.blockNumber, init: initState?.block.blockNumber,
diff: diffState?.block.blockNumber, diff: diffState?.block.blockNumber,
diff_staged: diffStagedState?.block.blockNumber, diff_staged: diffStagedState?.block.blockNumber,
@ -1372,7 +1399,7 @@ export class Indexer {
} }
await this._db.deleteEntitiesByConditions(dbTx, 'contract', { startingBlock: MoreThan(blockNumber) }); await this._db.deleteEntitiesByConditions(dbTx, 'contract', { startingBlock: MoreThan(blockNumber) });
this._clearWatchedContracts((watchedContracts) => watchedContracts.startingBlock > blockNumber); this._clearWatchedContracts((watchedContract) => watchedContract.startingBlock > blockNumber);
await this._db.deleteEntitiesByConditions(dbTx, 'block_progress', { blockNumber: MoreThan(blockNumber) }); await this._db.deleteEntitiesByConditions(dbTx, 'block_progress', { blockNumber: MoreThan(blockNumber) });
@ -1414,7 +1441,7 @@ export class Indexer {
} }
} }
async clearProcessedBlockData (block: BlockProgressInterface, entities: EntityTarget<{ blockNumber: number }>[]): Promise<void> { async clearProcessedBlockData (block: BlockProgressInterface, entities: EntityTarget<{ blockHash: string }>[]): Promise<void> {
const dbTx = await this._db.createTransactionRunner(); const dbTx = await this._db.createTransactionRunner();
try { try {
@ -1434,11 +1461,15 @@ export class Indexer {
} }
} }
_clearWatchedContracts (removFilter: (watchedContract: ContractInterface) => boolean): void { _clearWatchedContracts (removeFilter: (watchedContract: ContractInterface) => boolean): void {
this._watchedContracts = Object.values(this._watchedContracts) this._watchedContractsByAddressMap = Object.entries(this._watchedContractsByAddressMap)
.filter(watchedContract => !removFilter(watchedContract)) .map(([address, watchedContracts]): [string, ContractInterface[]] => [
.reduce((acc: {[key: string]: ContractInterface}, watchedContract) => { address,
acc[watchedContract.address] = watchedContract; watchedContracts.filter(watchedContract => !removeFilter(watchedContract))
])
.filter(([, watchedContracts]) => watchedContracts.length)
.reduce((acc: {[key: string]: ContractInterface[]}, [address, watchedContracts]) => {
acc[address] = watchedContracts;
return acc; return acc;
}, {}); }, {});

4
packages/util/src/types.ts
View File

@ -202,8 +202,8 @@ export interface IndexerInterface {
saveEventEntity (dbEvent: EventInterface): Promise<EventInterface> saveEventEntity (dbEvent: EventInterface): Promise<EventInterface>
saveEvents (dbEvents: DeepPartial<EventInterface>[]): Promise<void> saveEvents (dbEvents: DeepPartial<EventInterface>[]): Promise<void>
processEvent (event: EventInterface, extraData: ExtraEventData): Promise<void> processEvent (event: EventInterface, extraData: ExtraEventData): Promise<void>
parseEventNameAndArgs?: (kind: string, logObj: any) => { eventParsed: boolean, eventDetails: any } parseEventNameAndArgs?: (watchedContracts: ContractInterface[], logObj: any) => { eventParsed: boolean, eventDetails: any }
isWatchedContract: (address: string) => ContractInterface | undefined; isContractAddressWatched: (address: string) => ContractInterface[] | undefined;
getWatchedContracts: () => ContractInterface[] getWatchedContracts: () => ContractInterface[]
getContractsByKind?: (kind: string) => ContractInterface[] getContractsByKind?: (kind: string) => ContractInterface[]
addContracts?: () => Promise<void> addContracts?: () => Promise<void>