Add metrics to monitor errors and duration for ETH RPC requests (#507)

* Add metrics to monitor errors and duration for ETH RPC requests

* Check for server error

* Add a metric with configured upstream ETH RPC endpoints

* Use Gauge for RPC requests duration metric

* Filter out unknown events while loading event count on start

* Update package versions

* Rethrow errors in overridden send provider method

lerna.json

@@ -2,7 +2,7 @@
   "packages": [
     "packages/*"
   ],
-  "version": "0.2.86",
+  "version": "0.2.87",
   "npmClient": "yarn",
   "useWorkspaces": true,
   "command": {

packages/cache/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/cache",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "description": "Generic object cache",
   "main": "dist/index.js",
   "scripts": {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/cli",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "scripts": {
@@ -15,13 +15,13 @@
   },
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@cerc-io/cache": "^0.2.86",
+    "@cerc-io/cache": "^0.2.87",
-    "@cerc-io/ipld-eth-client": "^0.2.86",
+    "@cerc-io/ipld-eth-client": "^0.2.87",
     "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
     "@cerc-io/nitro-node": "^0.1.15",
-    "@cerc-io/peer": "^0.2.86",
+    "@cerc-io/peer": "^0.2.87",
-    "@cerc-io/rpc-eth-client": "^0.2.86",
+    "@cerc-io/rpc-eth-client": "^0.2.87",
-    "@cerc-io/util": "^0.2.86",
+    "@cerc-io/util": "^0.2.87",
     "@ethersproject/providers": "^5.4.4",
     "@graphql-tools/utils": "^9.1.1",
     "@ipld/dag-cbor": "^8.0.0",

packages/cli/src/job-runner.ts

@@ -23,7 +23,8 @@ import {
   startMetricsServer,
   Config,
   UpstreamConfig,
-  NEW_BLOCK_MAX_RETRIES_ERROR
+  NEW_BLOCK_MAX_RETRIES_ERROR,
+  setActiveUpstreamEndpointMetric
 } from '@cerc-io/util';
 
 import { BaseCmd } from './base';
@@ -138,6 +139,8 @@ export class JobRunnerCmd {
 
           const { ethClient, ethProvider } = await initClients(config, this._currentEndpointIndex);
           indexer.switchClients({ ethClient, ethProvider });
+          setActiveUpstreamEndpointMetric(config, this._currentEndpointIndex.rpcProviderEndpoint);
+
           log(`RPC endpoint ${oldRpcEndpoint} is not working; failing over to new RPC endpoint ${ethProvider.connection.url}`);
         }
       });

packages/codegen/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/codegen",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "description": "Code generator",
   "private": true,
   "main": "index.js",
@@ -20,7 +20,7 @@
   },
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
-    "@cerc-io/util": "^0.2.86",
+    "@cerc-io/util": "^0.2.87",
     "@graphql-tools/load-files": "^6.5.2",
     "@npmcli/package-json": "^5.0.0",
     "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

packages/codegen/src/templates/package-template.handlebars

@@ -41,12 +41,12 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@cerc-io/cli": "^0.2.86",
+    "@cerc-io/cli": "^0.2.87",
-    "@cerc-io/ipld-eth-client": "^0.2.86",
+    "@cerc-io/ipld-eth-client": "^0.2.87",
-    "@cerc-io/solidity-mapper": "^0.2.86",
+    "@cerc-io/solidity-mapper": "^0.2.87",
-    "@cerc-io/util": "^0.2.86",
+    "@cerc-io/util": "^0.2.87",
     {{#if (subgraphPath)}}
-    "@cerc-io/graph-node": "^0.2.86",
+    "@cerc-io/graph-node": "^0.2.87",
     {{/if}}
     "@ethersproject/providers": "^5.4.4",
     "debug": "^4.3.1",

packages/graph-node/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@cerc-io/graph-node",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "devDependencies": {
-    "@cerc-io/solidity-mapper": "^0.2.86",
+    "@cerc-io/solidity-mapper": "^0.2.87",
     "@ethersproject/providers": "^5.4.4",
     "@graphprotocol/graph-ts": "^0.22.0",
     "@nomiclabs/hardhat-ethers": "^2.0.2",
@@ -51,9 +51,9 @@
   "dependencies": {
     "@apollo/client": "^3.3.19",
     "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
-    "@cerc-io/cache": "^0.2.86",
+    "@cerc-io/cache": "^0.2.87",
-    "@cerc-io/ipld-eth-client": "^0.2.86",
+    "@cerc-io/ipld-eth-client": "^0.2.87",
-    "@cerc-io/util": "^0.2.86",
+    "@cerc-io/util": "^0.2.87",
     "@types/json-diff": "^0.5.2",
     "@types/yargs": "^17.0.0",
     "bn.js": "^4.11.9",

packages/ipld-eth-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/ipld-eth-client",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "description": "IPLD ETH Client",
   "main": "dist/index.js",
   "scripts": {
@@ -20,8 +20,8 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.7.1",
-    "@cerc-io/cache": "^0.2.86",
+    "@cerc-io/cache": "^0.2.87",
-    "@cerc-io/util": "^0.2.86",
+    "@cerc-io/util": "^0.2.87",
     "cross-fetch": "^3.1.4",
     "debug": "^4.3.1",
     "ethers": "^5.4.4",

packages/peer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/peer",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "description": "libp2p module",
   "main": "dist/index.js",
   "exports": "./dist/index.js",

packages/rpc-eth-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/rpc-eth-client",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "description": "RPC ETH Client",
   "main": "dist/index.js",
   "scripts": {
@@ -19,9 +19,9 @@
   },
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
-    "@cerc-io/cache": "^0.2.86",
+    "@cerc-io/cache": "^0.2.87",
-    "@cerc-io/ipld-eth-client": "^0.2.86",
+    "@cerc-io/ipld-eth-client": "^0.2.87",
-    "@cerc-io/util": "^0.2.86",
+    "@cerc-io/util": "^0.2.87",
     "chai": "^4.3.4",
     "ethers": "^5.4.4",
     "left-pad": "^1.3.0",

packages/rpc-eth-client/src/eth-client.ts

@@ -6,12 +6,13 @@ import assert from 'assert';
 import { errors, providers, utils } from 'ethers';
 
 import { Cache } from '@cerc-io/cache';
-import { encodeHeader, escapeHexString, EthClient as EthClientInterface, EthFullBlock, EthFullTransaction } from '@cerc-io/util';
+import {
+  encodeHeader, escapeHexString,
+  EthClient as EthClientInterface, EthFullBlock, EthFullTransaction,
+  MonitoredStaticJsonRpcProvider, FUTURE_BLOCK_ERROR, NULL_BLOCK_ERROR
+} from '@cerc-io/util';
 import { padKey } from '@cerc-io/ipld-eth-client';
 
-const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
-const NULL_BLOCK_ERROR = 'requested epoch was a null round';
-
 export interface Config {
   cache: Cache | undefined;
   rpcEndpoint: string;
@@ -35,7 +36,7 @@ export class EthClient implements EthClientInterface {
   constructor (config: Config) {
     const { rpcEndpoint, cache } = config;
     assert(rpcEndpoint, 'Missing RPC endpoint');
-    this._provider = new providers.StaticJsonRpcProvider({
+    this._provider = new MonitoredStaticJsonRpcProvider({
       url: rpcEndpoint,
       allowGzip: true
     });

packages/solidity-mapper/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/solidity-mapper",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "devDependencies": {

packages/test/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/test",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "private": true,

packages/tracing-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/tracing-client",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "description": "ETH VM tracing client",
   "main": "dist/index.js",
   "scripts": {

packages/util/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@cerc-io/util",
-  "version": "0.2.86",
+  "version": "0.2.87",
   "main": "dist/index.js",
   "license": "AGPL-3.0",
   "dependencies": {
     "@apollo/utils.keyvaluecache": "^1.0.1",
     "@cerc-io/nitro-node": "^0.1.15",
-    "@cerc-io/peer": "^0.2.86",
+    "@cerc-io/peer": "^0.2.87",
-    "@cerc-io/solidity-mapper": "^0.2.86",
+    "@cerc-io/solidity-mapper": "^0.2.87",
     "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
     "@ethersproject/properties": "^5.7.0",
     "@ethersproject/providers": "^5.4.4",
@@ -52,7 +52,7 @@
     "yargs": "^17.0.1"
   },
   "devDependencies": {
-    "@cerc-io/cache": "^0.2.86",
+    "@cerc-io/cache": "^0.2.87",
     "@nomiclabs/hardhat-waffle": "^2.0.1",
     "@types/bunyan": "^1.8.8",
     "@types/express": "^4.17.14",

packages/util/src/database.ts

@@ -18,7 +18,8 @@ import {
   QueryRunner,
   Repository,
   SelectQueryBuilder,
-  WhereExpressionBuilder
+  WhereExpressionBuilder,
+  Not
 } from 'typeorm';
 import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
 import _ from 'lodash';
@@ -1318,7 +1319,7 @@ export class Database {
 
   async _fetchEventCount (): Promise<void> {
     const res = await this._conn.getRepository('event')
-      .count();
+      .count({ where: { eventName: Not(UNKNOWN_EVENT_NAME) } });
 
     eventCount.set(res);
   }

packages/util/src/metrics.ts

@@ -95,6 +95,24 @@ export const isSyncingHistoricalBlocks = new client.Gauge({
 });
 isSyncingHistoricalBlocks.set(Number(undefined));
 
+export const ethRpcErrors = new client.Counter({
+  name: 'watcher_eth_rpc_errors',
+  help: 'Number of ETH RPC request errors',
+  labelNames: ['method', 'provider']
+});
+
+export const ethRpcRequestDuration = new client.Gauge({
+  name: 'watcher_eth_rpc_request_duration',
+  help: 'ETH RPC request duration (in seconds)',
+  labelNames: ['method', 'provider']
+});
+
+const upstreamEndpointsMetric = new client.Gauge({
+  name: 'watcher_config_upstream_endpoints',
+  help: 'Configured upstream ETH RPC endpoints',
+  labelNames: ['provider']
+});
+
 // Export metrics on a server
 const app: Application = express();
 
@@ -126,6 +144,8 @@ export const startMetricsServer = async (config: Config, indexer: IndexerInterfa
 
   await registerWatcherConfigMetrics(config);
 
+  setActiveUpstreamEndpointMetric(config, endpointIndexes.rpcProviderEndpoint);
+
   await registerDBSizeMetrics(config);
 
   await registerUpstreamChainHeadMetrics(config, endpointIndexes.rpcProviderEndpoint);
@@ -144,6 +164,13 @@ export const startMetricsServer = async (config: Config, indexer: IndexerInterfa
   });
 };
 
+export const setActiveUpstreamEndpointMetric = ({ upstream }: Config, currentEndpointIndex: number): void => {
+  const endpoints = upstream.ethServer.rpcProviderEndpoints;
+  endpoints.forEach((endpoint, index) => {
+    upstreamEndpointsMetric.set({ provider: endpoint }, Number(index === currentEndpointIndex));
+  });
+};
+
 const registerDBSizeMetrics = async ({ database, jobQueue }: Config): Promise<void> => {
   const [watcherConn, jobQueueConn] = await Promise.all([
     createConnection({

packages/util/src/misc.ts

@@ -6,7 +6,7 @@ import assert from 'assert';
 import { ValueTransformer } from 'typeorm';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import { utils, providers } from 'ethers';
+import { utils, providers, errors as ethersErrors } from 'ethers';
 import JSONbig from 'json-bigint';
 import Decimal from 'decimal.js';
 import ApolloBigInt from 'apollo-type-bigint';
@@ -22,9 +22,13 @@ import { ResultEvent } from './indexer';
 import { EventInterface, EthFullBlock, EthFullTransaction } from './types';
 import { BlockHeight } from './database';
 import { Transaction } from './graph/utils';
+import { ethRpcErrors, ethRpcRequestDuration } from './metrics';
 
 const JSONbigNative = JSONbig({ useNativeBigInt: true });
 
+export const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
+export const NULL_BLOCK_ERROR = 'requested epoch was a null round';
+
 /**
  * Method to wait for specified time.
  * @param time Time to wait in milliseconds
@@ -154,7 +158,7 @@ export const getResetYargs = (): yargs.Argv => {
 };
 
 export const getCustomProvider = (url?: utils.ConnectionInfo | string, network?: providers.Networkish): providers.JsonRpcProvider => {
-  const provider = new providers.StaticJsonRpcProvider(url, network);
+  const provider = new MonitoredStaticJsonRpcProvider(url, network);
   provider.formatter = new CustomFormatter();
   return provider;
 };
@@ -351,3 +355,31 @@ export const GraphQLBigDecimal = new GraphQLScalarType({
     return value.toFixed();
   }
 });
+
+export class MonitoredStaticJsonRpcProvider extends providers.StaticJsonRpcProvider {
+  // Override the send method
+  async send (method: string, params: Array<any>): Promise<any> {
+    // Register time taken for this request in the metrics
+    const endTimer = ethRpcRequestDuration.startTimer({ method, provider: this.connection.url });
+
+    try {
+      const result = await super.send(method, params);
+      return result;
+    } catch (err: any) {
+      // Ignore errors on fetching future blocks and if block is null (in case of filecoin)
+      if (err.code === ethersErrors.SERVER_ERROR && err.error) {
+        if (err.error.message === FUTURE_BLOCK_ERROR || err.error.message === NULL_BLOCK_ERROR) {
+          throw err;
+        }
+      }
+
+      // Register the error in metrics
+      ethRpcErrors.inc({ method, provider: this.connection.url }, 1);
+
+      // Rethrow the error
+      throw err;
+    } finally {
+      endTimer();
+    }
+  }
+}