Changes for supporting multiple data sources for a contract address (#4)

Part of [Generate secured-finance subgraph watcher with codegen](https://www.notion.so/Generate-secured-finance-subgraph-watcher-with-codegen-2923413e0af54ea787c5435d6966f3bb)

Reviewed-on: #4
Co-authored-by: Nabarun <nabarun@deepstacksoft.com>
Co-committed-by: Nabarun <nabarun@deepstacksoft.com>

README.md

@@ -2,7 +2,7 @@
 
 ## Source
 
-* Subgraph: [secured-finance-subgraph v0.1.1](https://github.com/cerc-io/secured-finance-subgraph/releases/tag/v0.1.1)
+* Subgraph: [secured-finance-subgraph v0.1.2-watcher-0.1.1](https://github.com/cerc-io/secured-finance-subgraph/releases/tag/v0.1.2-watcher-0.1.1)
 
 ## Setup
 
@@ -12,6 +12,12 @@
   yarn
   ```
 
+* Run build:
+
+  ```bash
+  yarn build
+  ```
+
 * Create a postgres12 database for the watcher:
 
   ```bash

environments/local.toml

@@ -78,6 +78,10 @@
     # Boolean flag to filter event logs by topics
     filterLogsByTopics = true
 
+    # Switch clients if eth_getLogs call takes more than threshold (in secs)
+    # Set to 0 for disabling switching
+    getLogsClientSwitchThresholdInSecs = 0
+
   [upstream.cache]
     name = "requests"
     enabled = false

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/secured-finance-watcher-ts",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "secured-finance-watcher-ts",
   "private": true,
   "main": "dist/index.js",
@@ -39,11 +39,11 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@cerc-io/cli": "^0.2.99",
+    "@cerc-io/cli": "^0.2.102",
-    "@cerc-io/ipld-eth-client": "^0.2.99",
+    "@cerc-io/ipld-eth-client": "^0.2.102",
-    "@cerc-io/solidity-mapper": "^0.2.99",
+    "@cerc-io/solidity-mapper": "^0.2.102",
-    "@cerc-io/util": "^0.2.99",
+    "@cerc-io/util": "^0.2.102",
-    "@cerc-io/graph-node": "^0.2.99",
+    "@cerc-io/graph-node": "^0.2.102",
     "@ethersproject/providers": "^5.4.4",
     "debug": "^4.3.1",
     "decimal.js": "^10.3.1",

src/entity/Contract.ts

@@ -5,7 +5,7 @@
 import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
 
 @Entity()
-@Index(['address'], { unique: true })
+@Index(['address', 'kind'], { unique: true })
 export class Contract {
   @PrimaryGeneratedColumn()
     id!: number;

src/indexer.ts

@@ -5,11 +5,10 @@
 import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions, ObjectLiteral } from 'typeorm';
 import debug from 'debug';
-import { ethers, constants } from 'ethers';
+import { ethers, constants, providers } from 'ethers';
 import { GraphQLResolveInfo } from 'graphql';
 
 import { JsonFragment } from '@ethersproject/abi';
-import { BaseProvider } from '@ethersproject/providers';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -36,6 +35,7 @@ import {
   EthFullTransaction,
   ExtraEventData
 } from '@cerc-io/util';
+import { initClients } from '@cerc-io/cli';
 import { GraphWatcher } from '@cerc-io/graph-node';
 
 import LendingMarketOperationLogicArtifacts from './artifacts/LendingMarketOperationLogic.json';
@@ -85,7 +85,7 @@ const KIND_ORDERBOOKLOGIC = 'OrderBookLogic';
 export class Indexer implements IndexerInterface {
   _db: Database;
   _ethClient: EthClient;
-  _ethProvider: BaseProvider;
+  _ethProvider: providers.JsonRpcProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
   _upstreamConfig: UpstreamConfig;
@@ -108,7 +108,7 @@ export class Indexer implements IndexerInterface {
     },
     db: DatabaseInterface,
     clients: Clients,
-    ethProvider: BaseProvider,
+    ethProvider: providers.JsonRpcProvider,
     jobQueue: JobQueue,
     graphWatcher?: GraphWatcherInterface
   ) {
@@ -237,13 +237,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.fetchStateStatus();
   }
 
-  switchClients ({ ethClient, ethProvider }: { ethClient: EthClient, ethProvider: BaseProvider }): void {
+  async switchClients (): Promise<void> {
+    const { ethClient, ethProvider } = await this._baseIndexer.switchClients(initClients);
     this._ethClient = ethClient;
     this._ethProvider = ethProvider;
-    this._baseIndexer.switchClients({ ethClient, ethProvider });
     this._graphWatcher.switchClients({ ethClient, ethProvider });
   }
 
+  async isGetLogsRequestsSlow (): Promise<boolean> {
+    return this._baseIndexer.isGetLogsRequestsSlow();
+  }
+
   async getMetaData (block: BlockHeight): Promise<ResultMeta | null> {
     return this._baseIndexer.getMetaData(block);
   }
@@ -431,23 +435,27 @@ export class Indexer implements IndexerInterface {
     console.timeEnd('time:indexer#processBlockAfterEvents-dump_subgraph_state');
   }
 
-  parseEventNameAndArgs (kind: string, logObj: any): { eventParsed: boolean, eventDetails: any } {
+  parseEventNameAndArgs (watchedContracts: Contract[], logObj: any): { eventParsed: boolean, eventDetails: any } {
     const { topics, data } = logObj;
+    let logDescription: ethers.utils.LogDescription | undefined;
 
-    const contract = this._contractMap.get(kind);
+    for (const watchedContract of watchedContracts) {
-    assert(contract);
+      const contract = this._contractMap.get(watchedContract.kind);
+      assert(contract);
 
-    let logDescription: ethers.utils.LogDescription;
+      try {
-    try {
+        logDescription = contract.parseLog({ data, topics });
-      logDescription = contract.parseLog({ data, topics });
+        break;
-    } catch (err) {
+      } catch (err) {
-      // Return if no matching event found
+        // Continue loop only if no matching event found
-      if ((err as Error).message.includes('no matching event')) {
+        if (!((err as Error).message.includes('no matching event'))) {
-        log(`WARNING: Skipping event for contract ${kind} as no matching event found in the ABI`);
+          throw err;
-        return { eventParsed: false, eventDetails: {} };
+        }
       }
+    }
 
-      throw err;
+    if (!logDescription) {
+      return { eventParsed: false, eventDetails: {} };
     }
 
     const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription);
@@ -551,8 +559,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
 
-  isWatchedContract (address : string): Contract | undefined {
+  isContractAddressWatched (address : string): Contract[] | undefined {
-    return this._baseIndexer.isWatchedContract(address);
+    return this._baseIndexer.isContractAddressWatched(address);
   }
 
   getWatchedContracts (): Contract[] {
@@ -818,6 +826,7 @@ export class Indexer implements IndexerInterface {
     });
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
   _populateRelationsMap (): void {
     this._relationsMap.set(Transaction, {
       user: {
@@ -957,7 +966,18 @@ export class Indexer implements IndexerInterface {
     assert(blockHash);
     assert(blockNumber);
 
-    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+    let dbEvents: DeepPartial<Event>[] = [];
+    let transactions: EthFullTransaction[] = [];
+
+    // Fetch events and txs only if subgraph config has any event handlers
+    if (this._graphWatcher.eventHandlerExists) {
+      ({ events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(
+        blockHash,
+        blockNumber,
+        this.eventSignaturesMap,
+        this.parseEventNameAndArgs.bind(this)
+      ));
+    }
 
     const dbTx = await this._db.createTransactionRunner();
     try {

yarn.lock

@@ -194,10 +194,10 @@
     binaryen "101.0.0-nightly.20210723"
     long "^4.0.0"
 
-"@cerc-io/cache@^0.2.99":
+"@cerc-io/cache@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.99/cache-0.2.99.tgz#70487c16dc502773a850062ec9662cd331c75176"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.102/cache-0.2.102.tgz#2a08e4fd136a6ac3e95386fc9d19aa836f0cdcc8"
-  integrity sha512-BPEn1Tb4G5UKiSPt+fWAOaQl2bob75Wkua11hgAD4GGfXDEWtJe8yGllsaQbiCzRmTCny6101ECwApdqolA7fg==
+  integrity sha512-b69fUlaMyJvJItLN8tDSKsUTxD9v37eMKQYyXtxl36bqbbD77YtNGzdf3OnWoUbi1YPPfshPBOCd9DUtf4hjJw==
   dependencies:
     canonical-json "^0.0.4"
     debug "^4.3.1"
@@ -205,19 +205,19 @@
     fs-extra "^10.0.0"
     level "^7.0.0"
 
-"@cerc-io/cli@^0.2.99":
+"@cerc-io/cli@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.99/cli-0.2.99.tgz#ef1d251b00d383add897d25ff746cbbbfbe0536f"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.102/cli-0.2.102.tgz#3c4dbd3afecd9ae3def211ac3d54d48a773a4a82"
-  integrity sha512-vbTxQ1CghmaFXODB5dDXBc2amMS6w4Pm/jHjix2eIe2A1bGI27C7xPtlPooFx5Ug5zzUY6eQ2B8BavVjgVs8hQ==
+  integrity sha512-wyHDi8gmprsJq9q9FIam2K4FxBKPsG9ilvytztc4Vr8s7Iyg5q3d+9R9IMjuU4gmi+qwcx9i/2CSlvR0E2B7Tw==
   dependencies:
     "@apollo/client" "^3.7.1"
-    "@cerc-io/cache" "^0.2.99"
+    "@cerc-io/cache" "^0.2.102"
-    "@cerc-io/ipld-eth-client" "^0.2.99"
+    "@cerc-io/ipld-eth-client" "^0.2.102"
     "@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
     "@cerc-io/nitro-node" "^0.1.15"
-    "@cerc-io/peer" "^0.2.99"
+    "@cerc-io/peer" "^0.2.102"
-    "@cerc-io/rpc-eth-client" "^0.2.99"
+    "@cerc-io/rpc-eth-client" "^0.2.102"
-    "@cerc-io/util" "^0.2.99"
+    "@cerc-io/util" "^0.2.102"
     "@ethersproject/providers" "^5.4.4"
     "@graphql-tools/utils" "^9.1.1"
     "@ipld/dag-cbor" "^8.0.0"
@@ -238,16 +238,16 @@
     typeorm "0.2.37"
     yargs "^17.0.1"
 
-"@cerc-io/graph-node@^0.2.99":
+"@cerc-io/graph-node@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fgraph-node/-/0.2.99/graph-node-0.2.99.tgz#d6c0ad0278d192d47f0f6f3319a6aaab88799562"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fgraph-node/-/0.2.102/graph-node-0.2.102.tgz#2c0f1f0d4ed375c55003122dd77ab9ae7912e742"
-  integrity sha512-RXCJ0CyyUguBGnDRzWjLM0xDegdoKxl10h0Cda2u58LZRkEyRON3EHAOcNbNn/NF4I9E9p0cVdsMsFdjksI8gQ==
+  integrity sha512-A9G0x4o2pGGQ8qID33aipXAAxpuDvTrG1vPko7uB9wjSZMSpoipDbomUQZDZcQs19qKJ9u4l1Q2S1PxlpsiW4w==
   dependencies:
     "@apollo/client" "^3.3.19"
     "@cerc-io/assemblyscript" "0.19.10-watcher-ts-0.1.2"
-    "@cerc-io/cache" "^0.2.99"
+    "@cerc-io/cache" "^0.2.102"
-    "@cerc-io/ipld-eth-client" "^0.2.99"
+    "@cerc-io/ipld-eth-client" "^0.2.102"
-    "@cerc-io/util" "^0.2.99"
+    "@cerc-io/util" "^0.2.102"
     "@types/json-diff" "^0.5.2"
     "@types/yargs" "^17.0.0"
     bn.js "^4.11.9"
@@ -264,14 +264,14 @@
     typeorm-naming-strategies "^2.0.0"
     yargs "^17.0.1"
 
-"@cerc-io/ipld-eth-client@^0.2.99":
+"@cerc-io/ipld-eth-client@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.99/ipld-eth-client-0.2.99.tgz#cda85282579fdd5081949f584056e15f28defbe9"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.102/ipld-eth-client-0.2.102.tgz#c9d25b6c292c733b6b5669909f96d2dcb92132ff"
-  integrity sha512-v/Msg6EXNTokZTRAoAojGaP+BmTAC9lJ9p447vJoc6PPB+/7cETkBeALKDEMx72vVU6eN7izsfbK0veHDZ5R2g==
+  integrity sha512-CimfNG9B4D+jf1wVvq/6f6IT6yFPpJZpOmizpp6YpjfwdP31rWR50iBJQjx+rWyk9R+8yvSmMhS90mB/vg6ujQ==
   dependencies:
     "@apollo/client" "^3.7.1"
-    "@cerc-io/cache" "^0.2.99"
+    "@cerc-io/cache" "^0.2.102"
-    "@cerc-io/util" "^0.2.99"
+    "@cerc-io/util" "^0.2.102"
     cross-fetch "^3.1.4"
     debug "^4.3.1"
     ethers "^5.4.4"
@@ -394,10 +394,10 @@
     lodash "^4.17.21"
     uint8arrays "^4.0.3"
 
-"@cerc-io/peer@^0.2.65":
+"@cerc-io/peer@^0.2.102":
-  version "0.2.90"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.90/peer-0.2.90.tgz#4507ab3d793b7c4e35af26f26470e2770aac6319"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.102/peer-0.2.102.tgz#b6ff92c97e52b940ade21e6e0b793b0a599d8704"
-  integrity sha512-jrmbjJBXwrSYOqNkXaaAh2J30gVKV+4vXnZAiPYbpl6Tlo72iJOwWAzJLXJej48NNCkJkdpXkuxcsQZtsW90mQ==
+  integrity sha512-d+hB4vB202TjiXhVr6uXQGOh4PC6kx7EuHKwxabxPOQROlWYk2/h1WO+ExydzeNaxZK1YPsDjnVfu71LkusX5g==
   dependencies:
     "@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
     "@cerc-io/prometheus-metrics" "1.1.4"
@@ -424,10 +424,10 @@
     unique-names-generator "^4.7.1"
     yargs "^17.0.1"
 
-"@cerc-io/peer@^0.2.99":
+"@cerc-io/peer@^0.2.65":
-  version "0.2.99"
+  version "0.2.90"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.99/peer-0.2.99.tgz#27008bf4276980119697d100e6cf6ab0c08ec6fb"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.90/peer-0.2.90.tgz#4507ab3d793b7c4e35af26f26470e2770aac6319"
-  integrity sha512-0AoZsMpIgAQR3p5YTmzXhIUIoxu28+8CJM0BdsOTABjfYfpfwXZvnUGAUyx/QvjTgtKinDvDTiFPPu9b/LEOuQ==
+  integrity sha512-jrmbjJBXwrSYOqNkXaaAh2J30gVKV+4vXnZAiPYbpl6Tlo72iJOwWAzJLXJej48NNCkJkdpXkuxcsQZtsW90mQ==
   dependencies:
     "@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
     "@cerc-io/prometheus-metrics" "1.1.4"
@@ -466,23 +466,23 @@
     it-stream-types "^1.0.4"
     promjs "^0.4.2"
 
-"@cerc-io/rpc-eth-client@^0.2.99":
+"@cerc-io/rpc-eth-client@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Frpc-eth-client/-/0.2.99/rpc-eth-client-0.2.99.tgz#58f1e82adf8b72f6e3725c5042f97ff9f35df81f"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Frpc-eth-client/-/0.2.102/rpc-eth-client-0.2.102.tgz#c29b5e8e8b3f73212993ac7ef4c2b34d9b76747c"
-  integrity sha512-mUz/opmb6X6NCtxREtAD8m1glsU25mnntQAS6dXt6mQ15A+vwhnf/rWBunkeeAVYviGgRIamfgMWC3CAs9dP9Q==
+  integrity sha512-IR25u6KrWZTnF2QBma+LQmmtT+kB4QG7LurDBi/yX1ywt3Z3L+w88BegAX9tbG0Y5sHmIPJqKhj8lgnY3jHuJg==
   dependencies:
-    "@cerc-io/cache" "^0.2.99"
+    "@cerc-io/cache" "^0.2.102"
-    "@cerc-io/ipld-eth-client" "^0.2.99"
+    "@cerc-io/ipld-eth-client" "^0.2.102"
-    "@cerc-io/util" "^0.2.99"
+    "@cerc-io/util" "^0.2.102"
     chai "^4.3.4"
     ethers "^5.4.4"
     left-pad "^1.3.0"
     mocha "^8.4.0"
 
-"@cerc-io/solidity-mapper@^0.2.99":
+"@cerc-io/solidity-mapper@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.99/solidity-mapper-0.2.99.tgz#d6e666e55a3ff598413e84b1da854fd6923249dd"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.102/solidity-mapper-0.2.102.tgz#87e7dd124e908a2a311e4cff90406bdf0d74f1ad"
-  integrity sha512-wQrxC+JmIoxnhwjSA36dp/Ri5zxIrlp8KlhBaan3gtE7nVPsPfKn04xlrKd8r822ih/QfzgLexpC5kxrWM42Qw==
+  integrity sha512-ovqcfuiMCR/9FGXX0qp/6piuZ+l394QLXGIidYgirXFVdWkCNjl1p3Y7VevXNke+KN8FdU1eLqmMfHE0QMpnOw==
   dependencies:
     dotenv "^10.0.0"
 
@@ -491,15 +491,15 @@
   resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fts-channel/-/1.0.3-ts-nitro-0.1.1/ts-channel-1.0.3-ts-nitro-0.1.1.tgz#0768781313a167295c0bf21307f47e02dc17e936"
   integrity sha512-2jFICUSyffuZ+8+qRhXuLSJq4GJ6Y02wxiXoubH0Kzv2lIKkJtWICY1ZQQhtXAvP0ncAQB85WJHqtqwH8l7J3Q==
 
-"@cerc-io/util@^0.2.99":
+"@cerc-io/util@^0.2.102":
-  version "0.2.99"
+  version "0.2.102"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.99/util-0.2.99.tgz#363df154c240ab32270ddf5e510c45ebd887fb6d"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.102/util-0.2.102.tgz#1e76d4db59f9beda09d3455251c725ce409b4665"
-  integrity sha512-BTBhSHd503A+7H2VWbQzHdM8/w1TyAI9alts6fp3K8Liecln//nbDkc285fo1H3M8tW7GYHx/84kqqQgzzCbHw==
+  integrity sha512-V8hnnfUgPE/zUULn3y1aJY2PKc0Pv4PcH8KY0BABPu5F5lhvCjW/XRMzJluFdwurUOT2Gyw+3vY7iNqVf15kGQ==
   dependencies:
     "@apollo/utils.keyvaluecache" "^1.0.1"
     "@cerc-io/nitro-node" "^0.1.15"
-    "@cerc-io/peer" "^0.2.99"
+    "@cerc-io/peer" "^0.2.102"
-    "@cerc-io/solidity-mapper" "^0.2.99"
+    "@cerc-io/solidity-mapper" "^0.2.102"
     "@cerc-io/ts-channel" "1.0.3-ts-nitro-0.1.1"
     "@ethersproject/properties" "^5.7.0"
     "@ethersproject/providers" "^5.4.4"