Changes for triggering subgraph event handlers in multiple datasources

This commit is contained in:
Nabarun 2024-06-25 19:36:34 +05:30
parent 6d76de44e4
commit 978ba8a71a
6 changed files with 110 additions and 79 deletions

View File

@ -12,6 +12,12 @@
yarn
```
* Run build:
```bash
yarn build
```
* Create a postgres12 database for the watcher:
```bash

View File

@ -78,6 +78,10 @@
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
# Switch clients if eth_getLogs call takes more than threshold (in secs)
# Set to 0 for disabling switching
getLogsClientSwitchThresholdInSecs = 0
[upstream.cache]
name = "requests"
enabled = false

View File

@ -39,11 +39,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.99",
"@cerc-io/ipld-eth-client": "^0.2.99",
"@cerc-io/solidity-mapper": "^0.2.99",
"@cerc-io/util": "^0.2.99",
"@cerc-io/graph-node": "^0.2.99",
"@cerc-io/cli": "^0.2.101",
"@cerc-io/ipld-eth-client": "^0.2.101",
"@cerc-io/solidity-mapper": "^0.2.101",
"@cerc-io/util": "^0.2.101",
"@cerc-io/graph-node": "^0.2.101",
"@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1",
"decimal.js": "^10.3.1",

View File

@ -5,7 +5,7 @@
import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
@Entity()
@Index(['address'], { unique: true })
@Index(['address', 'kind'], { unique: true })
export class Contract {
@PrimaryGeneratedColumn()
id!: number;

View File

@ -5,11 +5,10 @@
import assert from 'assert';
import { DeepPartial, FindConditions, FindManyOptions, ObjectLiteral } from 'typeorm';
import debug from 'debug';
import { ethers, constants } from 'ethers';
import { ethers, constants, providers } from 'ethers';
import { GraphQLResolveInfo } from 'graphql';
import { JsonFragment } from '@ethersproject/abi';
import { BaseProvider } from '@ethersproject/providers';
import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
import {
Indexer as BaseIndexer,
@ -34,8 +33,10 @@ import {
UpstreamConfig,
EthFullBlock,
EthFullTransaction,
ExtraEventData
ExtraEventData,
ContractInterface
} from '@cerc-io/util';
import { initClients } from '@cerc-io/cli';
import { GraphWatcher } from '@cerc-io/graph-node';
import LendingMarketOperationLogicArtifacts from './artifacts/LendingMarketOperationLogic.json';
@ -85,7 +86,7 @@ const KIND_ORDERBOOKLOGIC = 'OrderBookLogic';
export class Indexer implements IndexerInterface {
_db: Database;
_ethClient: EthClient;
_ethProvider: BaseProvider;
_ethProvider: providers.JsonRpcProvider;
_baseIndexer: BaseIndexer;
_serverConfig: ServerConfig;
_upstreamConfig: UpstreamConfig;
@ -108,7 +109,7 @@ export class Indexer implements IndexerInterface {
},
db: DatabaseInterface,
clients: Clients,
ethProvider: BaseProvider,
ethProvider: providers.JsonRpcProvider,
jobQueue: JobQueue,
graphWatcher?: GraphWatcherInterface
) {
@ -237,13 +238,17 @@ export class Indexer implements IndexerInterface {
await this._baseIndexer.fetchStateStatus();
}
switchClients ({ ethClient, ethProvider }: { ethClient: EthClient, ethProvider: BaseProvider }): void {
async switchClients (): Promise<void> {
const { ethClient, ethProvider } = await this._baseIndexer.switchClients(initClients);
this._ethClient = ethClient;
this._ethProvider = ethProvider;
this._baseIndexer.switchClients({ ethClient, ethProvider });
this._graphWatcher.switchClients({ ethClient, ethProvider });
}
async isGetLogsRequestsSlow (): Promise<boolean> {
return this._baseIndexer.isGetLogsRequestsSlow();
}
async getMetaData (block: BlockHeight): Promise<ResultMeta | null> {
return this._baseIndexer.getMetaData(block);
}
@ -431,23 +436,27 @@ export class Indexer implements IndexerInterface {
console.timeEnd('time:indexer#processBlockAfterEvents-dump_subgraph_state');
}
parseEventNameAndArgs (kind: string, logObj: any): { eventParsed: boolean, eventDetails: any } {
parseEventNameAndArgs (watchedContracts: Contract[], logObj: any): { eventParsed: boolean, eventDetails: any } {
const { topics, data } = logObj;
let logDescription: ethers.utils.LogDescription | undefined;
const contract = this._contractMap.get(kind);
assert(contract);
for (const watchedContract of watchedContracts) {
const contract = this._contractMap.get(watchedContract.kind);
assert(contract);
let logDescription: ethers.utils.LogDescription;
try {
logDescription = contract.parseLog({ data, topics });
} catch (err) {
// Return if no matching event found
if ((err as Error).message.includes('no matching event')) {
log(`WARNING: Skipping event for contract ${kind} as no matching event found in the ABI`);
return { eventParsed: false, eventDetails: {} };
try {
logDescription = contract.parseLog({ data, topics });
break;
} catch (err) {
// Continue loop only if no matching event found
if (!((err as Error).message.includes('no matching event'))) {
throw err;
}
}
}
throw err;
if (!logDescription) {
return { eventParsed: false, eventDetails: {} };
}
const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription);
@ -551,8 +560,8 @@ export class Indexer implements IndexerInterface {
return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
}
isWatchedContract (address : string): Contract | undefined {
return this._baseIndexer.isWatchedContract(address);
isContractAddressWatched (address : string): ContractInterface[] | undefined {
return this._baseIndexer.isContractAddressWatched(address);
}
getWatchedContracts (): Contract[] {
@ -818,6 +827,7 @@ export class Indexer implements IndexerInterface {
});
}
// eslint-disable-next-line @typescript-eslint/no-empty-function
_populateRelationsMap (): void {
this._relationsMap.set(Transaction, {
user: {
@ -957,7 +967,18 @@ export class Indexer implements IndexerInterface {
assert(blockHash);
assert(blockNumber);
const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
let dbEvents: DeepPartial<Event>[] = [];
let transactions: EthFullTransaction[] = [];
// Fetch events and txs only if subgraph config has any event handlers
if (this._graphWatcher.eventHandlerExists) {
({ events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(
blockHash,
blockNumber,
this.eventSignaturesMap,
this.parseEventNameAndArgs.bind(this)
));
}
const dbTx = await this._db.createTransactionRunner();
try {

102
yarn.lock
View File

@ -194,10 +194,10 @@
binaryen "101.0.0-nightly.20210723"
long "^4.0.0"
"@cerc-io/cache@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.99/cache-0.2.99.tgz#70487c16dc502773a850062ec9662cd331c75176"
integrity sha512-BPEn1Tb4G5UKiSPt+fWAOaQl2bob75Wkua11hgAD4GGfXDEWtJe8yGllsaQbiCzRmTCny6101ECwApdqolA7fg==
"@cerc-io/cache@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.101/cache-0.2.101.tgz#cafda5d3c60e528ff857732b4d5611009bcb5d60"
integrity sha512-tR+ewkBPVP7xGqcgSUUmm2hg6jPet9ukoFd0sBaB19wsuo/iTlZnx8sECfpbd/mwfv0GiSBqDQGImPsjszPTHA==
dependencies:
canonical-json "^0.0.4"
debug "^4.3.1"
@ -205,19 +205,19 @@
fs-extra "^10.0.0"
level "^7.0.0"
"@cerc-io/cli@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.99/cli-0.2.99.tgz#ef1d251b00d383add897d25ff746cbbbfbe0536f"
integrity sha512-vbTxQ1CghmaFXODB5dDXBc2amMS6w4Pm/jHjix2eIe2A1bGI27C7xPtlPooFx5Ug5zzUY6eQ2B8BavVjgVs8hQ==
"@cerc-io/cli@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.101/cli-0.2.101.tgz#3bd8905d984ca0a3569276ab3aba18e0ae516594"
integrity sha512-ouvYxzpvlJytOPt6HZf9tEPYj3yJimslomFlGTwfKNgyzvXmTm+ZjWpml2JxlH5k+djwTO8T+LQXaXJfMoSRsA==
dependencies:
"@apollo/client" "^3.7.1"
"@cerc-io/cache" "^0.2.99"
"@cerc-io/ipld-eth-client" "^0.2.99"
"@cerc-io/cache" "^0.2.101"
"@cerc-io/ipld-eth-client" "^0.2.101"
"@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
"@cerc-io/nitro-node" "^0.1.15"
"@cerc-io/peer" "^0.2.99"
"@cerc-io/rpc-eth-client" "^0.2.99"
"@cerc-io/util" "^0.2.99"
"@cerc-io/peer" "^0.2.101"
"@cerc-io/rpc-eth-client" "^0.2.101"
"@cerc-io/util" "^0.2.101"
"@ethersproject/providers" "^5.4.4"
"@graphql-tools/utils" "^9.1.1"
"@ipld/dag-cbor" "^8.0.0"
@ -238,16 +238,16 @@
typeorm "0.2.37"
yargs "^17.0.1"
"@cerc-io/graph-node@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fgraph-node/-/0.2.99/graph-node-0.2.99.tgz#d6c0ad0278d192d47f0f6f3319a6aaab88799562"
integrity sha512-RXCJ0CyyUguBGnDRzWjLM0xDegdoKxl10h0Cda2u58LZRkEyRON3EHAOcNbNn/NF4I9E9p0cVdsMsFdjksI8gQ==
"@cerc-io/graph-node@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fgraph-node/-/0.2.101/graph-node-0.2.101.tgz#1afb34b97dfa7ddc7c84c3650b454e39ef4f10f0"
integrity sha512-lhh7Q1e/LpiV0IZifHSGc4HOeyGVYLcllzmLDnnGwjn8xQxZOGIPgu2Y6qrd29IAdujEiCTyHIrX+IE73+5LMQ==
dependencies:
"@apollo/client" "^3.3.19"
"@cerc-io/assemblyscript" "0.19.10-watcher-ts-0.1.2"
"@cerc-io/cache" "^0.2.99"
"@cerc-io/ipld-eth-client" "^0.2.99"
"@cerc-io/util" "^0.2.99"
"@cerc-io/cache" "^0.2.101"
"@cerc-io/ipld-eth-client" "^0.2.101"
"@cerc-io/util" "^0.2.101"
"@types/json-diff" "^0.5.2"
"@types/yargs" "^17.0.0"
bn.js "^4.11.9"
@ -264,14 +264,14 @@
typeorm-naming-strategies "^2.0.0"
yargs "^17.0.1"
"@cerc-io/ipld-eth-client@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.99/ipld-eth-client-0.2.99.tgz#cda85282579fdd5081949f584056e15f28defbe9"
integrity sha512-v/Msg6EXNTokZTRAoAojGaP+BmTAC9lJ9p447vJoc6PPB+/7cETkBeALKDEMx72vVU6eN7izsfbK0veHDZ5R2g==
"@cerc-io/ipld-eth-client@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.101/ipld-eth-client-0.2.101.tgz#7ddbe5ff089134d27633f7be320d7c4e502a802a"
integrity sha512-CDC7h4+J9RlDSWrwui88MIAAASnshRU1dobOgc5/UCkxEMyk7TWUwSjDmKYhtBYCsS67NTc/X/XfAxZbxhyorA==
dependencies:
"@apollo/client" "^3.7.1"
"@cerc-io/cache" "^0.2.99"
"@cerc-io/util" "^0.2.99"
"@cerc-io/cache" "^0.2.101"
"@cerc-io/util" "^0.2.101"
cross-fetch "^3.1.4"
debug "^4.3.1"
ethers "^5.4.4"
@ -394,10 +394,10 @@
lodash "^4.17.21"
uint8arrays "^4.0.3"
"@cerc-io/peer@^0.2.65":
version "0.2.90"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.90/peer-0.2.90.tgz#4507ab3d793b7c4e35af26f26470e2770aac6319"
integrity sha512-jrmbjJBXwrSYOqNkXaaAh2J30gVKV+4vXnZAiPYbpl6Tlo72iJOwWAzJLXJej48NNCkJkdpXkuxcsQZtsW90mQ==
"@cerc-io/peer@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.101/peer-0.2.101.tgz#1cc565f8848a415127d18206f9b2d6ff13aa81b2"
integrity sha512-24OwRpWwjaYnnRgBQ6w1Pw/TIPZKv9WEzeoknZLrIxDQURSjZwOQGXj+hPwFZiw7Wym8wdq4FYyMqkDs5G+dSg==
dependencies:
"@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
"@cerc-io/prometheus-metrics" "1.1.4"
@ -424,10 +424,10 @@
unique-names-generator "^4.7.1"
yargs "^17.0.1"
"@cerc-io/peer@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.99/peer-0.2.99.tgz#27008bf4276980119697d100e6cf6ab0c08ec6fb"
integrity sha512-0AoZsMpIgAQR3p5YTmzXhIUIoxu28+8CJM0BdsOTABjfYfpfwXZvnUGAUyx/QvjTgtKinDvDTiFPPu9b/LEOuQ==
"@cerc-io/peer@^0.2.65":
version "0.2.90"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.90/peer-0.2.90.tgz#4507ab3d793b7c4e35af26f26470e2770aac6319"
integrity sha512-jrmbjJBXwrSYOqNkXaaAh2J30gVKV+4vXnZAiPYbpl6Tlo72iJOwWAzJLXJej48NNCkJkdpXkuxcsQZtsW90mQ==
dependencies:
"@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
"@cerc-io/prometheus-metrics" "1.1.4"
@ -466,23 +466,23 @@
it-stream-types "^1.0.4"
promjs "^0.4.2"
"@cerc-io/rpc-eth-client@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Frpc-eth-client/-/0.2.99/rpc-eth-client-0.2.99.tgz#58f1e82adf8b72f6e3725c5042f97ff9f35df81f"
integrity sha512-mUz/opmb6X6NCtxREtAD8m1glsU25mnntQAS6dXt6mQ15A+vwhnf/rWBunkeeAVYviGgRIamfgMWC3CAs9dP9Q==
"@cerc-io/rpc-eth-client@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Frpc-eth-client/-/0.2.101/rpc-eth-client-0.2.101.tgz#b7956a48caa8781dca3f723af0a65e9410fbe20f"
integrity sha512-cKlR0V8CsZ53OeLHIotqLdTL7fuIYIn5qI3ZrAkTBYaZL807gHXvl2Orm+Bivm+ak5fraQHdKvSpHymih7iQew==
dependencies:
"@cerc-io/cache" "^0.2.99"
"@cerc-io/ipld-eth-client" "^0.2.99"
"@cerc-io/util" "^0.2.99"
"@cerc-io/cache" "^0.2.101"
"@cerc-io/ipld-eth-client" "^0.2.101"
"@cerc-io/util" "^0.2.101"
chai "^4.3.4"
ethers "^5.4.4"
left-pad "^1.3.0"
mocha "^8.4.0"
"@cerc-io/solidity-mapper@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.99/solidity-mapper-0.2.99.tgz#d6e666e55a3ff598413e84b1da854fd6923249dd"
integrity sha512-wQrxC+JmIoxnhwjSA36dp/Ri5zxIrlp8KlhBaan3gtE7nVPsPfKn04xlrKd8r822ih/QfzgLexpC5kxrWM42Qw==
"@cerc-io/solidity-mapper@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.101/solidity-mapper-0.2.101.tgz#f50f1987a955def3e8f4bab58115daf09f2586e5"
integrity sha512-hPUSIN23X1R0tAht9o0TRP5z+wwQLx8Y57aJUBx7DBuco4zt293vZcM0faiMUbv78glwgvq0RxYGtPITNQpOFg==
dependencies:
dotenv "^10.0.0"
@ -491,15 +491,15 @@
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fts-channel/-/1.0.3-ts-nitro-0.1.1/ts-channel-1.0.3-ts-nitro-0.1.1.tgz#0768781313a167295c0bf21307f47e02dc17e936"
integrity sha512-2jFICUSyffuZ+8+qRhXuLSJq4GJ6Y02wxiXoubH0Kzv2lIKkJtWICY1ZQQhtXAvP0ncAQB85WJHqtqwH8l7J3Q==
"@cerc-io/util@^0.2.99":
version "0.2.99"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.99/util-0.2.99.tgz#363df154c240ab32270ddf5e510c45ebd887fb6d"
integrity sha512-BTBhSHd503A+7H2VWbQzHdM8/w1TyAI9alts6fp3K8Liecln//nbDkc285fo1H3M8tW7GYHx/84kqqQgzzCbHw==
"@cerc-io/util@^0.2.101":
version "0.2.101"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.101/util-0.2.101.tgz#c56179f5a347e962beab8226fb5bac501b2f40fa"
integrity sha512-iaj7UeIMV2M1TKKrjQToEsk5ulqOzzp1a/RmzxSynAqMRyzNwpqdEsaxIbGqT2hk6mHVsQdRYPkhSLos9jx6CQ==
dependencies:
"@apollo/utils.keyvaluecache" "^1.0.1"
"@cerc-io/nitro-node" "^0.1.15"
"@cerc-io/peer" "^0.2.99"
"@cerc-io/solidity-mapper" "^0.2.99"
"@cerc-io/peer" "^0.2.101"
"@cerc-io/solidity-mapper" "^0.2.101"
"@cerc-io/ts-channel" "1.0.3-ts-nitro-0.1.1"
"@ethersproject/properties" "^5.7.0"
"@ethersproject/providers" "^5.4.4"