Upgrade watcher version to 0.2.102 (#18 )

Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) - Set getLogsClientSwitchThresholdInSecs to 30 seconds Reviewed-on: #18 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>
Upgrade watcher version to 0.2.101 (#17 )
2024-07-04 09:06:55 +00:00 · 2024-07-04 08:50:50 +00:00 · 2024-06-13 09:09:40 +00:00
7 changed files with 115 additions and 81 deletions
@@ -12,6 +12,12 @@
  yarn
  ```

+* Run build:
+
+  ```bash
+  yarn build
+  ```
+
 * Create a postgres12 database for the watcher:

  ```bash
@@ -10,7 +10,6 @@
  checkpointInterval = 2000

  # Enable state creation
-  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
  enableState = false

  subgraphPath = "./subgraph-build"
@@ -79,6 +78,10 @@
    # Boolean flag to filter event logs by topics
    filterLogsByTopics = true

+    # Switch clients if eth_getLogs call takes more than threshold (in secs)
+    # Set to 0 for disabling switching
+    getLogsClientSwitchThresholdInSecs = 30
+
  [upstream.cache]
    name = "requests"
    enabled = false
@@ -90,7 +93,10 @@
  jobDelayInMilliSecs = 100
  eventsInBatch = 50
  subgraphEventsOrder = true
-  blockDelayInMilliSecs = 2000
+  blockDelayInMilliSecs = 30000
+
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0

  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
@@ -1,6 +1,6 @@
 {
  "name": "@cerc-io/ajna-watcher-ts",
-  "version": "0.1.12",
+  "version": "0.1.15",
  "description": "ajna-watcher-ts",
  "private": true,
  "main": "dist/index.js",
@@ -39,11 +39,11 @@
  "homepage": "https://github.com/cerc-io/watcher-ts#readme",
  "dependencies": {
    "@apollo/client": "^3.3.19",
-    "@cerc-io/cli": "^0.2.94",
-    "@cerc-io/ipld-eth-client": "^0.2.94",
-    "@cerc-io/solidity-mapper": "^0.2.94",
-    "@cerc-io/util": "^0.2.94",
-    "@cerc-io/graph-node": "^0.2.94",
+    "@cerc-io/cli": "^0.2.102",
+    "@cerc-io/ipld-eth-client": "^0.2.102",
+    "@cerc-io/solidity-mapper": "^0.2.102",
+    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/graph-node": "^0.2.102",
    "@ethersproject/providers": "^5.4.4",
    "debug": "^4.3.1",
    "decimal.js": "^10.3.1",
@@ -5,7 +5,7 @@
 import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';

@Entity()
-@Index(['address'], { unique: true })
+@Index(['address', 'kind'], { unique: true })
 export class Contract {
  @PrimaryGeneratedColumn()
    id!: number;
@@ -5,11 +5,10 @@
 import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions, ObjectLiteral } from 'typeorm';
 import debug from 'debug';
-import { ethers, constants } from 'ethers';
+import { ethers, constants, providers } from 'ethers';
 import { GraphQLResolveInfo } from 'graphql';

 import { JsonFragment } from '@ethersproject/abi';
-import { BaseProvider } from '@ethersproject/providers';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
  Indexer as BaseIndexer,
@@ -36,6 +35,7 @@ import {
  EthFullTransaction,
  ExtraEventData
 } from '@cerc-io/util';
+import { initClients } from '@cerc-io/cli';
 import { GraphWatcher } from '@cerc-io/graph-node';

 import PositionManagerArtifacts from './artifacts/PositionManager.json';
@@ -148,7 +148,7 @@ const KIND_ERC721POOL = 'ERC721Pool';
 export class Indexer implements IndexerInterface {
  _db: Database;
  _ethClient: EthClient;
-  _ethProvider: BaseProvider;
+  _ethProvider: providers.JsonRpcProvider;
  _baseIndexer: BaseIndexer;
  _serverConfig: ServerConfig;
  _upstreamConfig: UpstreamConfig;
@@ -171,7 +171,7 @@ export class Indexer implements IndexerInterface {
    },
    db: DatabaseInterface,
    clients: Clients,
-    ethProvider: BaseProvider,
+    ethProvider: providers.JsonRpcProvider,
    jobQueue: JobQueue,
    graphWatcher?: GraphWatcherInterface
  ) {
@@ -326,13 +326,17 @@ export class Indexer implements IndexerInterface {
    await this._baseIndexer.fetchStateStatus();
  }

-  switchClients ({ ethClient, ethProvider }: { ethClient: EthClient, ethProvider: BaseProvider }): void {
+  async switchClients (): Promise<void> {
+    const { ethClient, ethProvider } = await this._baseIndexer.switchClients(initClients);
    this._ethClient = ethClient;
    this._ethProvider = ethProvider;
-    this._baseIndexer.switchClients({ ethClient, ethProvider });
    this._graphWatcher.switchClients({ ethClient, ethProvider });
  }

+  async isGetLogsRequestsSlow (): Promise<boolean> {
+    return this._baseIndexer.isGetLogsRequestsSlow();
+  }
+
  async getMetaData (block: BlockHeight): Promise<ResultMeta | null> {
    return this._baseIndexer.getMetaData(block);
  }
@@ -520,23 +524,27 @@ export class Indexer implements IndexerInterface {
    console.timeEnd('time:indexer#processBlockAfterEvents-dump_subgraph_state');
  }

-  parseEventNameAndArgs (kind: string, logObj: any): { eventParsed: boolean, eventDetails: any } {
+  parseEventNameAndArgs (watchedContracts: Contract[], logObj: any): { eventParsed: boolean, eventDetails: any } {
    const { topics, data } = logObj;
+    let logDescription: ethers.utils.LogDescription | undefined;

-    const contract = this._contractMap.get(kind);
-    assert(contract);
+    for (const watchedContract of watchedContracts) {
+      const contract = this._contractMap.get(watchedContract.kind);
+      assert(contract);

-    let logDescription: ethers.utils.LogDescription;
-    try {
-      logDescription = contract.parseLog({ data, topics });
-    } catch (err) {
-      // Return if no matching event found
-      if ((err as Error).message.includes('no matching event')) {
-        log(`WARNING: Skipping event for contract ${kind} as no matching event found in the ABI`);
-        return { eventParsed: false, eventDetails: {} };
+      try {
+        logDescription = contract.parseLog({ data, topics });
+        break;
+      } catch (err) {
+        // Continue loop only if no matching event found
+        if (!((err as Error).message.includes('no matching event'))) {
+          throw err;
+        }
      }
+    }

-      throw err;
+    if (!logDescription) {
+      return { eventParsed: false, eventDetails: {} };
    }

    const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription);
@@ -640,8 +648,8 @@ export class Indexer implements IndexerInterface {
    return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
  }

-  isWatchedContract (address : string): Contract | undefined {
-    return this._baseIndexer.isWatchedContract(address);
+  isContractAddressWatched (address : string): Contract[] | undefined {
+    return this._baseIndexer.isContractAddressWatched(address);
  }

  getWatchedContracts (): Contract[] {
@@ -1548,6 +1556,7 @@ export class Indexer implements IndexerInterface {
    });
  }

+  // eslint-disable-next-line @typescript-eslint/no-empty-function
  _populateRelationsMap (): void {
    this._relationsMap.set(Token, {
      pools: {
@@ -2278,7 +2287,18 @@ export class Indexer implements IndexerInterface {
    assert(blockHash);
    assert(blockNumber);

-    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+    let dbEvents: DeepPartial<Event>[] = [];
+    let transactions: EthFullTransaction[] = [];
+
+    // Fetch events and txs only if subgraph config has any event handlers
+    if (this._graphWatcher.eventHandlerExists) {
+      ({ events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(
+        blockHash,
+        blockNumber,
+        this.eventSignaturesMap,
+        this.parseEventNameAndArgs.bind(this)
+      ));
+    }

    const dbTx = await this._db.createTransactionRunner();
    try {
@@ -133,6 +133,8 @@ const executeAndRecordMetrics = async (
      apiKey: expressContext.req.header('x-api-key'),
      origin: expressContext.req.headers.origin
    });
+
+    throw error;
  } finally {
    endTimer();
  }
@@ -199,10 +199,10 @@
    binaryen "101.0.0-nightly.20210723"
    long "^4.0.0"

-"@cerc-io/cache@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.94/cache-0.2.94.tgz#e1557bfc3c85ebe3b83f24c79b55c7324797e7ed"
-  integrity sha512-9NBMd+19aBs3lgiie0X6TJizcn7yBCfEXXlbvMvac9SgYDPcloU+ghauQephAxayRjaWkrgjjfEWR/I0x6FwqA==
+"@cerc-io/cache@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.102/cache-0.2.102.tgz#2a08e4fd136a6ac3e95386fc9d19aa836f0cdcc8"
+  integrity sha512-b69fUlaMyJvJItLN8tDSKsUTxD9v37eMKQYyXtxl36bqbbD77YtNGzdf3OnWoUbi1YPPfshPBOCd9DUtf4hjJw==
  dependencies:
    canonical-json "^0.0.4"
    debug "^4.3.1"
@@ -210,19 +210,19 @@
    fs-extra "^10.0.0"
    level "^7.0.0"

-"@cerc-io/cli@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.94/cli-0.2.94.tgz#f193165a4b85174c48150051ea4b0e3df57fdf9e"
-  integrity sha512-aJGzH630bhYca74efVEU37sCjxRLmERDObzNpKIjbmYjKPg6urMbYI8gs33R2HIS/FUOMS39F5B9NMwgCQd/Vw==
+"@cerc-io/cli@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.102/cli-0.2.102.tgz#3c4dbd3afecd9ae3def211ac3d54d48a773a4a82"
+  integrity sha512-wyHDi8gmprsJq9q9FIam2K4FxBKPsG9ilvytztc4Vr8s7Iyg5q3d+9R9IMjuU4gmi+qwcx9i/2CSlvR0E2B7Tw==
  dependencies:
    "@apollo/client" "^3.7.1"
-    "@cerc-io/cache" "^0.2.94"
-    "@cerc-io/ipld-eth-client" "^0.2.94"
+    "@cerc-io/cache" "^0.2.102"
+    "@cerc-io/ipld-eth-client" "^0.2.102"
    "@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
    "@cerc-io/nitro-node" "^0.1.15"
-    "@cerc-io/peer" "^0.2.94"
-    "@cerc-io/rpc-eth-client" "^0.2.94"
-    "@cerc-io/util" "^0.2.94"
+    "@cerc-io/peer" "^0.2.102"
+    "@cerc-io/rpc-eth-client" "^0.2.102"
+    "@cerc-io/util" "^0.2.102"
    "@ethersproject/providers" "^5.4.4"
    "@graphql-tools/utils" "^9.1.1"
    "@ipld/dag-cbor" "^8.0.0"
@@ -243,16 +243,16 @@
    typeorm "0.2.37"
    yargs "^17.0.1"

-"@cerc-io/graph-node@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fgraph-node/-/0.2.94/graph-node-0.2.94.tgz#e8d346daa9e7733098d039d27e7d575e0f978b20"
-  integrity sha512-bGqv/5dq2saBcekA/YqDcepvRaZVIMPXt/lDEcP+8lXmavImXO1sdnut7JyMDQCy3DNKXT6IGADPM/olUwnDPQ==
+"@cerc-io/graph-node@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fgraph-node/-/0.2.102/graph-node-0.2.102.tgz#2c0f1f0d4ed375c55003122dd77ab9ae7912e742"
+  integrity sha512-A9G0x4o2pGGQ8qID33aipXAAxpuDvTrG1vPko7uB9wjSZMSpoipDbomUQZDZcQs19qKJ9u4l1Q2S1PxlpsiW4w==
  dependencies:
    "@apollo/client" "^3.3.19"
    "@cerc-io/assemblyscript" "0.19.10-watcher-ts-0.1.2"
-    "@cerc-io/cache" "^0.2.94"
-    "@cerc-io/ipld-eth-client" "^0.2.94"
-    "@cerc-io/util" "^0.2.94"
+    "@cerc-io/cache" "^0.2.102"
+    "@cerc-io/ipld-eth-client" "^0.2.102"
+    "@cerc-io/util" "^0.2.102"
    "@types/json-diff" "^0.5.2"
    "@types/yargs" "^17.0.0"
    bn.js "^4.11.9"
@@ -269,14 +269,14 @@
    typeorm-naming-strategies "^2.0.0"
    yargs "^17.0.1"

-"@cerc-io/ipld-eth-client@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.94/ipld-eth-client-0.2.94.tgz#22f79eaf82da60a25d8f89f23cf45581c012d6ff"
-  integrity sha512-IOOZrryZ3M7ndhk8N9NeRaXvzC8eU2OS0z7fVL0xZ/h3Ei4D+dz8fVJbCvgGRB6mJadrMXYyhahOtEAJYxjNHg==
+"@cerc-io/ipld-eth-client@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.102/ipld-eth-client-0.2.102.tgz#c9d25b6c292c733b6b5669909f96d2dcb92132ff"
+  integrity sha512-CimfNG9B4D+jf1wVvq/6f6IT6yFPpJZpOmizpp6YpjfwdP31rWR50iBJQjx+rWyk9R+8yvSmMhS90mB/vg6ujQ==
  dependencies:
    "@apollo/client" "^3.7.1"
-    "@cerc-io/cache" "^0.2.94"
-    "@cerc-io/util" "^0.2.94"
+    "@cerc-io/cache" "^0.2.102"
+    "@cerc-io/util" "^0.2.102"
    cross-fetch "^3.1.4"
    debug "^4.3.1"
    ethers "^5.4.4"
@@ -399,10 +399,10 @@
    lodash "^4.17.21"
    uint8arrays "^4.0.3"

-"@cerc-io/peer@^0.2.65":
-  version "0.2.82"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.82/peer-0.2.82.tgz#e27a1cafd06c25f9c4e02d49d4615f45a76ce22b"
-  integrity sha512-wKkBXO7hxuN13Av60otDj+2O7nyoogYp+n8uTKNU9cv6J87OLRUZZAuGb5FggUONgDqxS3cyaXD6vqLpkhbbIA==
+"@cerc-io/peer@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.102/peer-0.2.102.tgz#b6ff92c97e52b940ade21e6e0b793b0a599d8704"
+  integrity sha512-d+hB4vB202TjiXhVr6uXQGOh4PC6kx7EuHKwxabxPOQROlWYk2/h1WO+ExydzeNaxZK1YPsDjnVfu71LkusX5g==
  dependencies:
    "@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
    "@cerc-io/prometheus-metrics" "1.1.4"
@@ -429,10 +429,10 @@
    unique-names-generator "^4.7.1"
    yargs "^17.0.1"

-"@cerc-io/peer@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.94/peer-0.2.94.tgz#c8776a9968b32d96fb91ed2a2c6d4b62b365e4a0"
-  integrity sha512-QQANziPnT08guAD/Nwy/HoqgIIX+w1+GS2bOUE4hfS+LJmTiZ+Zquiqcj69qGQeMQxfiUHmsBPttioF+B9rp/A==
+"@cerc-io/peer@^0.2.65":
+  version "0.2.82"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.82/peer-0.2.82.tgz#e27a1cafd06c25f9c4e02d49d4615f45a76ce22b"
+  integrity sha512-wKkBXO7hxuN13Av60otDj+2O7nyoogYp+n8uTKNU9cv6J87OLRUZZAuGb5FggUONgDqxS3cyaXD6vqLpkhbbIA==
  dependencies:
    "@cerc-io/libp2p" "^0.42.2-laconic-0.1.4"
    "@cerc-io/prometheus-metrics" "1.1.4"
@@ -471,23 +471,23 @@
    it-stream-types "^1.0.4"
    promjs "^0.4.2"

-"@cerc-io/rpc-eth-client@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Frpc-eth-client/-/0.2.94/rpc-eth-client-0.2.94.tgz#0c4f679abacf0aed831cbd295d21780f2a3dd17a"
-  integrity sha512-S3UvHO7Qb4Z43+fArMbrs78CP8l81D5Ootd225MQ2//TGmu9QsIK9+oqdI1CIVAQCbxI7QAStzdz76Y0px2iFA==
+"@cerc-io/rpc-eth-client@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Frpc-eth-client/-/0.2.102/rpc-eth-client-0.2.102.tgz#c29b5e8e8b3f73212993ac7ef4c2b34d9b76747c"
+  integrity sha512-IR25u6KrWZTnF2QBma+LQmmtT+kB4QG7LurDBi/yX1ywt3Z3L+w88BegAX9tbG0Y5sHmIPJqKhj8lgnY3jHuJg==
  dependencies:
-    "@cerc-io/cache" "^0.2.94"
-    "@cerc-io/ipld-eth-client" "^0.2.94"
-    "@cerc-io/util" "^0.2.94"
+    "@cerc-io/cache" "^0.2.102"
+    "@cerc-io/ipld-eth-client" "^0.2.102"
+    "@cerc-io/util" "^0.2.102"
    chai "^4.3.4"
    ethers "^5.4.4"
    left-pad "^1.3.0"
    mocha "^8.4.0"

-"@cerc-io/solidity-mapper@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.94/solidity-mapper-0.2.94.tgz#a414653bf20fcbedd91d4a544c6a019c8a02d958"
-  integrity sha512-kO2xqyAeq2+bbxBcItsT6i8qK1F0BhaYPzV1tTNXgGbkp3j1do86mc6IeTEeV6J/8e9EaeRBsOSQOdGOHUjcDA==
+"@cerc-io/solidity-mapper@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.102/solidity-mapper-0.2.102.tgz#87e7dd124e908a2a311e4cff90406bdf0d74f1ad"
+  integrity sha512-ovqcfuiMCR/9FGXX0qp/6piuZ+l394QLXGIidYgirXFVdWkCNjl1p3Y7VevXNke+KN8FdU1eLqmMfHE0QMpnOw==
  dependencies:
    dotenv "^10.0.0"

@@ -496,15 +496,15 @@
  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fts-channel/-/1.0.3-ts-nitro-0.1.1/ts-channel-1.0.3-ts-nitro-0.1.1.tgz#0768781313a167295c0bf21307f47e02dc17e936"
  integrity sha512-2jFICUSyffuZ+8+qRhXuLSJq4GJ6Y02wxiXoubH0Kzv2lIKkJtWICY1ZQQhtXAvP0ncAQB85WJHqtqwH8l7J3Q==

-"@cerc-io/util@^0.2.94":
-  version "0.2.94"
-  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.94/util-0.2.94.tgz#5d4c8046cf44641eac39a6f6ee11f2eb8b298c1f"
-  integrity sha512-4B1RkUKZoRMVVxWK5u+5d3GEmn0IN1FoAbkfCsBXMTH+OGI+VVGg6bRozaluy7pegpwDh1KtiH0A7NFjUup4OQ==
+"@cerc-io/util@^0.2.102":
+  version "0.2.102"
+  resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.102/util-0.2.102.tgz#1e76d4db59f9beda09d3455251c725ce409b4665"
+  integrity sha512-V8hnnfUgPE/zUULn3y1aJY2PKc0Pv4PcH8KY0BABPu5F5lhvCjW/XRMzJluFdwurUOT2Gyw+3vY7iNqVf15kGQ==
  dependencies:
    "@apollo/utils.keyvaluecache" "^1.0.1"
    "@cerc-io/nitro-node" "^0.1.15"
-    "@cerc-io/peer" "^0.2.94"
-    "@cerc-io/solidity-mapper" "^0.2.94"
+    "@cerc-io/peer" "^0.2.102"
+    "@cerc-io/solidity-mapper" "^0.2.102"
    "@cerc-io/ts-channel" "1.0.3-ts-nitro-0.1.1"
    "@ethersproject/properties" "^5.7.0"
    "@ethersproject/providers" "^5.4.4"