cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2024-11-19 12:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add metrics to monitor errors and duration for ETH RPC requests (#507)

Browse Source 
* Add metrics to monitor errors and duration for ETH RPC requests

* Check for server error

* Add a metric with configured upstream ETH RPC endpoints

* Use Gauge for RPC requests duration metric

* Filter out unknown events while loading event count on start

* Update package versions

* Rethrow errors in overridden send provider method
This commit is contained in:
prathamesh0 2024-05-15 19:11:22 +05:30 committed by GitHub
parent c9696c3d9f
commit c7e6baa263
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 109 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "0.2.86",
"version": "0.2.87",
"npmClient": "yarn",
"useWorkspaces": true,
"command": {

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cache",
"version": "0.2.86",
"version": "0.2.87",
"description": "Generic object cache",
"main": "dist/index.js",
"scripts": {

12
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cli",
"version": "0.2.86",
"version": "0.2.87",
"main": "dist/index.js",
"license": "AGPL-3.0",
"scripts": {
@ -15,13 +15,13 @@
},
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.86",
"@cerc-io/ipld-eth-client": "^0.2.86",
"@cerc-io/cache": "^0.2.87",
"@cerc-io/ipld-eth-client": "^0.2.87",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.86",
"@cerc-io/rpc-eth-client": "^0.2.86",
"@cerc-io/util": "^0.2.86",
"@cerc-io/peer": "^0.2.87",
"@cerc-io/rpc-eth-client": "^0.2.87",
"@cerc-io/util": "^0.2.87",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0",

5
packages/cli/src/job-runner.ts
View File

@ -23,7 +23,8 @@ import {
startMetricsServer,
Config,
UpstreamConfig,
NEW_BLOCK_MAX_RETRIES_ERROR
NEW_BLOCK_MAX_RETRIES_ERROR,
setActiveUpstreamEndpointMetric
} from '@cerc-io/util';
import { BaseCmd } from './base';
@ -138,6 +139,8 @@ export class JobRunnerCmd {
const { ethClient, ethProvider } = await initClients(config, this._currentEndpointIndex);
indexer.switchClients({ ethClient, ethProvider });
setActiveUpstreamEndpointMetric(config, this._currentEndpointIndex.rpcProviderEndpoint);
log(`RPC endpoint ${oldRpcEndpoint} is not working; failing over to new RPC endpoint ${ethProvider.connection.url}`);
}
});

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/codegen",
"version": "0.2.86",
"version": "0.2.87",
"description": "Code generator",
"private": true,
"main": "index.js",
@ -20,7 +20,7 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/util": "^0.2.86",
"@cerc-io/util": "^0.2.87",
"@graphql-tools/load-files": "^6.5.2",
"@npmcli/package-json": "^5.0.0",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.86",
"@cerc-io/ipld-eth-client": "^0.2.86",
"@cerc-io/solidity-mapper": "^0.2.86",
"@cerc-io/util": "^0.2.86",
"@cerc-io/cli": "^0.2.87",
"@cerc-io/ipld-eth-client": "^0.2.87",
"@cerc-io/solidity-mapper": "^0.2.87",
"@cerc-io/util": "^0.2.87",
{{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.86",
"@cerc-io/graph-node": "^0.2.87",
{{/if}}
"@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1",

10
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/graph-node",
"version": "0.2.86",
"version": "0.2.87",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.86",
"@cerc-io/solidity-mapper": "^0.2.87",
"@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.86",
"@cerc-io/ipld-eth-client": "^0.2.86",
"@cerc-io/util": "^0.2.86",
"@cerc-io/cache": "^0.2.87",
"@cerc-io/ipld-eth-client": "^0.2.87",
"@cerc-io/util": "^0.2.87",
"@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0",
"bn.js": "^4.11.9",

6
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/ipld-eth-client",
"version": "0.2.86",
"version": "0.2.87",
"description": "IPLD ETH Client",
"main": "dist/index.js",
"scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.86",
"@cerc-io/util": "^0.2.86",
"@cerc-io/cache": "^0.2.87",
"@cerc-io/util": "^0.2.87",
"cross-fetch": "^3.1.4",
"debug": "^4.3.1",
"ethers": "^5.4.4",

2
packages/peer/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/peer",
"version": "0.2.86",
"version": "0.2.87",
"description": "libp2p module",
"main": "dist/index.js",
"exports": "./dist/index.js",

8
packages/rpc-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/rpc-eth-client",
"version": "0.2.86",
"version": "0.2.87",
"description": "RPC ETH Client",
"main": "dist/index.js",
"scripts": {
@ -19,9 +19,9 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/cache": "^0.2.86",
"@cerc-io/ipld-eth-client": "^0.2.86",
"@cerc-io/util": "^0.2.86",
"@cerc-io/cache": "^0.2.87",
"@cerc-io/ipld-eth-client": "^0.2.87",
"@cerc-io/util": "^0.2.87",
"chai": "^4.3.4",
"ethers": "^5.4.4",
"left-pad": "^1.3.0",

11
packages/rpc-eth-client/src/eth-client.ts
View File

@ -6,12 +6,13 @@ import assert from 'assert';
import { errors, providers, utils } from 'ethers';
import { Cache } from '@cerc-io/cache';
import { encodeHeader, escapeHexString, EthClient as EthClientInterface, EthFullBlock, EthFullTransaction } from '@cerc-io/util';
import {
encodeHeader, escapeHexString,
EthClient as EthClientInterface, EthFullBlock, EthFullTransaction,
MonitoredStaticJsonRpcProvider, FUTURE_BLOCK_ERROR, NULL_BLOCK_ERROR
} from '@cerc-io/util';
import { padKey } from '@cerc-io/ipld-eth-client';
const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
const NULL_BLOCK_ERROR = 'requested epoch was a null round';
export interface Config {
cache: Cache | undefined;
rpcEndpoint: string;
@ -35,7 +36,7 @@ export class EthClient implements EthClientInterface {
constructor (config: Config) {
const { rpcEndpoint, cache } = config;
assert(rpcEndpoint, 'Missing RPC endpoint');
this._provider = new providers.StaticJsonRpcProvider({
this._provider = new MonitoredStaticJsonRpcProvider({
url: rpcEndpoint,
allowGzip: true
});

2
packages/solidity-mapper/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/solidity-mapper",
"version": "0.2.86",
"version": "0.2.87",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/test",
"version": "0.2.86",
"version": "0.2.87",
"main": "dist/index.js",
"license": "AGPL-3.0",
"private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/tracing-client",
"version": "0.2.86",
"version": "0.2.87",
"description": "ETH VM tracing client",
"main": "dist/index.js",
"scripts": {

8
packages/util/package.json
View File

@ -1,13 +1,13 @@
{
"name": "@cerc-io/util",
"version": "0.2.86",
"version": "0.2.87",
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.86",
"@cerc-io/solidity-mapper": "^0.2.86",
"@cerc-io/peer": "^0.2.87",
"@cerc-io/solidity-mapper": "^0.2.87",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4",
@ -52,7 +52,7 @@
"yargs": "^17.0.1"
},
"devDependencies": {
"@cerc-io/cache": "^0.2.86",
"@cerc-io/cache": "^0.2.87",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14",

5
packages/util/src/database.ts
View File

@ -18,7 +18,8 @@ import {
QueryRunner,
Repository,
SelectQueryBuilder,
WhereExpressionBuilder
WhereExpressionBuilder,
Not
} from 'typeorm';
import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
import _ from 'lodash';
@ -1318,7 +1319,7 @@ export class Database {
async _fetchEventCount (): Promise<void> {
const res = await this._conn.getRepository('event')
.count();
.count({ where: { eventName: Not(UNKNOWN_EVENT_NAME) } });
eventCount.set(res);
}

27
packages/util/src/metrics.ts
View File

@ -95,6 +95,24 @@ export const isSyncingHistoricalBlocks = new client.Gauge({
});
isSyncingHistoricalBlocks.set(Number(undefined));
export const ethRpcErrors = new client.Counter({
name: 'watcher_eth_rpc_errors',
help: 'Number of ETH RPC request errors',
labelNames: ['method', 'provider']
});
export const ethRpcRequestDuration = new client.Gauge({
name: 'watcher_eth_rpc_request_duration',
help: 'ETH RPC request duration (in seconds)',
labelNames: ['method', 'provider']
});
const upstreamEndpointsMetric = new client.Gauge({
name: 'watcher_config_upstream_endpoints',
help: 'Configured upstream ETH RPC endpoints',
labelNames: ['provider']
});
// Export metrics on a server
const app: Application = express();
@ -126,6 +144,8 @@ export const startMetricsServer = async (config: Config, indexer: IndexerInterfa
await registerWatcherConfigMetrics(config);
setActiveUpstreamEndpointMetric(config, endpointIndexes.rpcProviderEndpoint);
await registerDBSizeMetrics(config);
await registerUpstreamChainHeadMetrics(config, endpointIndexes.rpcProviderEndpoint);
@ -144,6 +164,13 @@ export const startMetricsServer = async (config: Config, indexer: IndexerInterfa
});
};
export const setActiveUpstreamEndpointMetric = ({ upstream }: Config, currentEndpointIndex: number): void => {
const endpoints = upstream.ethServer.rpcProviderEndpoints;
endpoints.forEach((endpoint, index) => {
upstreamEndpointsMetric.set({ provider: endpoint }, Number(index === currentEndpointIndex));
});
};
const registerDBSizeMetrics = async ({ database, jobQueue }: Config): Promise<void> => {
const [watcherConn, jobQueueConn] = await Promise.all([
createConnection({

36
packages/util/src/misc.ts
View File

@ -6,7 +6,7 @@ import assert from 'assert';
import { ValueTransformer } from 'typeorm';
import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import { utils, providers } from 'ethers';
import { utils, providers, errors as ethersErrors } from 'ethers';
import JSONbig from 'json-bigint';
import Decimal from 'decimal.js';
import ApolloBigInt from 'apollo-type-bigint';
@ -22,9 +22,13 @@ import { ResultEvent } from './indexer';
import { EventInterface, EthFullBlock, EthFullTransaction } from './types';
import { BlockHeight } from './database';
import { Transaction } from './graph/utils';
import { ethRpcErrors, ethRpcRequestDuration } from './metrics';
const JSONbigNative = JSONbig({ useNativeBigInt: true });
export const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
export const NULL_BLOCK_ERROR = 'requested epoch was a null round';
/**
* Method to wait for specified time.
* @param time Time to wait in milliseconds
@ -154,7 +158,7 @@ export const getResetYargs = (): yargs.Argv => {
};
export const getCustomProvider = (url?: utils.ConnectionInfo | string, network?: providers.Networkish): providers.JsonRpcProvider => {
const provider = new providers.StaticJsonRpcProvider(url, network);
const provider = new MonitoredStaticJsonRpcProvider(url, network);
provider.formatter = new CustomFormatter();
return provider;
};
@ -351,3 +355,31 @@ export const GraphQLBigDecimal = new GraphQLScalarType({
return value.toFixed();
}
});
export class MonitoredStaticJsonRpcProvider extends providers.StaticJsonRpcProvider {
// Override the send method
async send (method: string, params: Array<any>): Promise<any> {
// Register time taken for this request in the metrics
const endTimer = ethRpcRequestDuration.startTimer({ method, provider: this.connection.url });
try {
const result = await super.send(method, params);
return result;
} catch (err: any) {
// Ignore errors on fetching future blocks and if block is null (in case of filecoin)
if (err.code === ethersErrors.SERVER_ERROR && err.error) {
if (err.error.message === FUTURE_BLOCK_ERROR || err.error.message === NULL_BLOCK_ERROR) {
throw err;
}
}
// Register the error in metrics
ethRpcErrors.inc({ method, provider: this.connection.url }, 1);
// Rethrow the error
throw err;
} finally {
endTimer();
}
}
}