Export metric for total ETH RPC count (#528)

* Export metric for total ETH RPC count by methods

* Fix endpoint switch on max retries of new block

* Upgrade package versions

lerna.json

@@ -2,7 +2,7 @@
   "packages": [
     "packages/*"
   ],
-  "version": "0.2.102",
+  "version": "0.2.103",
   "npmClient": "yarn",
   "useWorkspaces": true,
   "command": {

packages/cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/cache",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "description": "Generic object cache",
   "main": "dist/index.js",
   "scripts": {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/cli",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "scripts": {
@@ -15,13 +15,13 @@
   },
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@cerc-io/cache": "^0.2.102",
+    "@cerc-io/cache": "^0.2.103",
-    "@cerc-io/ipld-eth-client": "^0.2.102",
+    "@cerc-io/ipld-eth-client": "^0.2.103",
     "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
     "@cerc-io/nitro-node": "^0.1.15",
-    "@cerc-io/peer": "^0.2.102",
+    "@cerc-io/peer": "^0.2.103",
-    "@cerc-io/rpc-eth-client": "^0.2.102",
+    "@cerc-io/rpc-eth-client": "^0.2.103",
-    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/util": "^0.2.103",
     "@ethersproject/providers": "^5.4.4",
     "@graphql-tools/utils": "^9.1.1",
     "@ipld/dag-cbor": "^8.0.0",

packages/codegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/codegen",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "description": "Code generator",
   "private": true,
   "main": "index.js",
@@ -20,7 +20,7 @@
   },
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
-    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/util": "^0.2.103",
     "@graphql-tools/load-files": "^6.5.2",
     "@npmcli/package-json": "^5.0.0",
     "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

packages/codegen/src/templates/indexer-template.handlebars

@@ -249,7 +249,7 @@ export class Indexer implements IndexerInterface {
       };
     }
 
-    const { block: { number } } = await this._ethClient.getBlockByHash(blockHash);
+    const { block: { number } } = await this.getBlockByHash(blockHash);
     const blockNumber = ethers.BigNumber.from(number).toNumber();
 
     log('{{query.name}}: db miss, fetching from upstream server');
@@ -679,6 +679,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocks(blockFilter);
   }
 
+  async getBlockByHash (blockHash?: string): Promise<{ block: any }> {
+    return this._baseIndexer.getBlockByHash(blockHash);
+  }
+
   async updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
     return this._baseIndexer.updateSyncStatusIndexedBlock(blockHash, blockNumber, force);
   }

packages/codegen/src/templates/package-template.handlebars

@@ -41,12 +41,12 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@cerc-io/cli": "^0.2.102",
+    "@cerc-io/cli": "^0.2.103",
-    "@cerc-io/ipld-eth-client": "^0.2.102",
+    "@cerc-io/ipld-eth-client": "^0.2.103",
-    "@cerc-io/solidity-mapper": "^0.2.102",
+    "@cerc-io/solidity-mapper": "^0.2.103",
-    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/util": "^0.2.103",
     {{#if (subgraphPath)}}
-    "@cerc-io/graph-node": "^0.2.102",
+    "@cerc-io/graph-node": "^0.2.103",
     {{/if}}
     "@ethersproject/providers": "^5.4.4",
     "debug": "^4.3.1",

packages/graph-node/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@cerc-io/graph-node",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "devDependencies": {
-    "@cerc-io/solidity-mapper": "^0.2.102",
+    "@cerc-io/solidity-mapper": "^0.2.103",
     "@ethersproject/providers": "^5.4.4",
     "@graphprotocol/graph-ts": "^0.22.0",
     "@nomiclabs/hardhat-ethers": "^2.0.2",
@@ -51,9 +51,9 @@
   "dependencies": {
     "@apollo/client": "^3.3.19",
     "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
-    "@cerc-io/cache": "^0.2.102",
+    "@cerc-io/cache": "^0.2.103",
-    "@cerc-io/ipld-eth-client": "^0.2.102",
+    "@cerc-io/ipld-eth-client": "^0.2.103",
-    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/util": "^0.2.103",
     "@types/json-diff": "^0.5.2",
     "@types/yargs": "^17.0.0",
     "bn.js": "^4.11.9",

packages/graph-node/test/utils/indexer.ts

@@ -95,6 +95,10 @@ export class Indexer implements IndexerInterface {
     return undefined;
   }
 
+  async getBlockByHash (blockHash?: string): Promise<{ block: any }> {
+    return { block: undefined };
+  }
+
   async getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]> {
     assert(height);
     assert(isPruned);

packages/ipld-eth-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/ipld-eth-client",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "description": "IPLD ETH Client",
   "main": "dist/index.js",
   "scripts": {
@@ -20,8 +20,8 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@cerc-io/cache": "^0.2.102",
+    "@cerc-io/cache": "^0.2.103",
-    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/util": "^0.2.103",
     "cross-fetch": "^3.1.4",
     "debug": "^4.3.1",
     "ethers": "^5.4.4",

packages/peer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/peer",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "description": "libp2p module",
   "main": "dist/index.js",
   "exports": "./dist/index.js",

packages/rpc-eth-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/rpc-eth-client",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "description": "RPC ETH Client",
   "main": "dist/index.js",
   "scripts": {
@@ -19,9 +19,9 @@
   },
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
-    "@cerc-io/cache": "^0.2.102",
+    "@cerc-io/cache": "^0.2.103",
-    "@cerc-io/ipld-eth-client": "^0.2.102",
+    "@cerc-io/ipld-eth-client": "^0.2.103",
-    "@cerc-io/util": "^0.2.102",
+    "@cerc-io/util": "^0.2.103",
     "chai": "^4.3.4",
     "ethers": "^5.4.4",
     "left-pad": "^1.3.0",

packages/solidity-mapper/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/solidity-mapper",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "devDependencies": {

packages/test/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/test",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "private": true,

packages/tracing-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/tracing-client",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "description": "ETH VM tracing client",
   "main": "dist/index.js",
   "scripts": {

packages/util/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@cerc-io/util",
-  "version": "0.2.102",
+  "version": "0.2.103",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "dependencies": {
     "@apollo/utils.keyvaluecache": "^1.0.1",
     "@cerc-io/nitro-node": "^0.1.15",
-    "@cerc-io/peer": "^0.2.102",
+    "@cerc-io/peer": "^0.2.103",
-    "@cerc-io/solidity-mapper": "^0.2.102",
+    "@cerc-io/solidity-mapper": "^0.2.103",
     "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
     "@ethersproject/properties": "^5.7.0",
     "@ethersproject/providers": "^5.4.4",
@@ -54,7 +54,7 @@
     "yargs": "^17.0.1"
   },
   "devDependencies": {
-    "@cerc-io/cache": "^0.2.102",
+    "@cerc-io/cache": "^0.2.103",
     "@nomiclabs/hardhat-waffle": "^2.0.1",
     "@types/bunyan": "^1.8.8",
     "@types/express": "^4.17.14",

packages/util/src/common.ts

@@ -71,6 +71,22 @@ export const fetchBlocksAtHeight = async (
 
   // Try fetching blocks from eth-server until found.
   while (!blocks.length) {
+    const { block: latestBlock } = await indexer.getBlockByHash();
+    const blockProcessingOffset = jobQueueConfig.blockProcessingOffset ?? 0;
+
+    // Process block if it is blockProcessingOffset blocks behind latest block
+    if (latestBlock.number < blockNumber + blockProcessingOffset) {
+      // Check number of retries for fetching new block
+      if (jobQueueConfig.maxNewBlockRetries && newBlockRetries > jobQueueConfig.maxNewBlockRetries) {
+        throw new Error(NEW_BLOCK_MAX_RETRIES_ERROR);
+      }
+
+      newBlockRetries++;
+      log(`Latest block: ${latestBlock.number}, blockProcessingOffset: ${blockProcessingOffset}; retry block to process: ${blockNumber} after ${jobQueueConfig.blockDelayInMilliSecs}ms`);
+      await wait(jobQueueConfig.blockDelayInMilliSecs);
+      continue;
+    }
+
     console.time(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
     const ethFullBlocks = await indexer.getBlocks({ blockNumber });
     console.timeEnd(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
@@ -84,18 +100,8 @@ export const fetchBlocksAtHeight = async (
 
     // Fitler null blocks
     blocks = ethFullBlocks.filter(block => Boolean(block)) as EthFullBlock[];
+    assert(blocks.length, `Blocks at ${blockNumber} should exist as latest block is ${latestBlock}`);
 
-    if (!blocks.length) {
-      log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`);
-
-      // Check number of retries for fetching new block
-      if (jobQueueConfig.maxNewBlockRetries && newBlockRetries > jobQueueConfig.maxNewBlockRetries) {
-        throw new Error(NEW_BLOCK_MAX_RETRIES_ERROR);
-      }
-
-      newBlockRetries++;
-      await wait(jobQueueConfig.blockDelayInMilliSecs);
-    } else {
     blocks.forEach(block => {
       blockAndEventsMap.set(
         block.blockHash,
@@ -110,7 +116,6 @@ export const fetchBlocksAtHeight = async (
       );
     });
   }
-  }
 
   assert(blocks.length, 'Blocks not fetched');
 

packages/util/src/events.ts

@@ -14,7 +14,6 @@ import { createPruningJob, processBlockByNumber } from './common';
 import { OrderDirection } from './database';
 import { HistoricalJobData, HistoricalJobResponseData } from './job-runner';
 import { JobQueueConfig, ServerConfig } from './config';
-import { wait } from './misc';
 
 const EVENT = 'event';
 const BLOCK_PROGRESS_EVENT = 'block-progress-event';
@@ -105,7 +104,7 @@ export class EventWatcher {
     // Get latest block in chain and sync status from DB
     // Also get historical-processing queue size
     const [{ block: latestBlock }, syncStatus, historicalProcessingQueueSize] = await Promise.all([
-      this._ethClient.getBlockByHash(),
+      this._indexer.getBlockByHash(),
       this._indexer.getSyncStatus(),
       this._jobQueue.getQueueSize(QUEUE_HISTORICAL_PROCESSING, 'completed')
     ]);
@@ -196,18 +195,7 @@ export class EventWatcher {
       }
 
       if (isComplete) {
-        while (true) {
-          const { block: latestBlock } = await this._ethClient.getBlockByHash();
-
-          // Process block if it is blockProcessingOffset blocks behind latest block
-          if (latestBlock.number >= blockNumber + (this._config.jobQueue.blockProcessingOffset ?? 0)) {
         await processBlockByNumber(this._jobQueue, blockNumber + 1);
-            break;
-          }
-
-          log(`Latest block: ${latestBlock.number}; retry next block to process: ${blockNumber + 1} after ${this._config.jobQueue.blockDelayInMilliSecs}ms`);
-          await wait(this._config.jobQueue.blockDelayInMilliSecs);
-        }
       }
     }
   }

packages/util/src/indexer.ts

@@ -394,6 +394,10 @@ export class Indexer {
     return blocks;
   }
 
+  async getBlockByHash (blockHash?: string): Promise<{ block: any }> {
+    return this._ethClient.getBlockByHash(blockHash);
+  }
+
   async getBlockProgress (blockHash: string): Promise<BlockProgressInterface | undefined> {
     return this._db.getBlockProgress(blockHash);
   }

packages/util/src/metrics.ts

@@ -82,6 +82,12 @@ export const isSyncingHistoricalBlocks = new client.Gauge({
 });
 isSyncingHistoricalBlocks.set(Number(undefined));
 
+export const ethRpcCount = new client.Counter({
+  name: 'watcher_eth_rpc_total',
+  help: 'Total number of ETH RPC requests',
+  labelNames: ['method', 'provider']
+});
+
 export const ethRpcErrors = new client.Counter({
   name: 'watcher_eth_rpc_errors',
   help: 'Number of ETH RPC request errors',

packages/util/src/misc.ts

@@ -22,7 +22,7 @@ import { ResultEvent } from './indexer';
 import { EventInterface, EthFullBlock, EthFullTransaction } from './types';
 import { BlockHeight } from './database';
 import { Transaction } from './graph/utils';
-import { ethRpcErrors, ethRpcRequestDuration } from './metrics';
+import { ethRpcCount, ethRpcErrors, ethRpcRequestDuration } from './metrics';
 
 const JSONbigNative = JSONbig({ useNativeBigInt: true });
 
@@ -379,6 +379,7 @@ export class MonitoredStaticJsonRpcProvider extends providers.StaticJsonRpcProvi
       // Rethrow the error
       throw err;
     } finally {
+      ethRpcCount.inc({ method, provider: this.connection.url }, 1);
       endTimer();
     }
   }

packages/util/src/types.ts

@@ -171,6 +171,7 @@ export interface IndexerInterface {
   getSyncStatus (): Promise<SyncStatusInterface | undefined>
   getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
   getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<Array<EthFullBlock | null>>
+  getBlockByHash (blockHash?: string): Promise<{ block: any }>
   getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]>
   getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined>
   getLatestStateIndexedBlock (): Promise<BlockProgressInterface>