cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2024-11-19 12:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Export metric for total ETH RPC count (#528)

Browse Source 
* Export metric for total ETH RPC count by methods

* Fix endpoint switch on max retries of new block

* Upgrade package versions
This commit is contained in:
Nabarun Gogoi 2024-07-12 16:59:23 +05:30 committed by GitHub
parent 2217cd3ffb
commit 42cb688921
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 89 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "0.2.102",
"version": "0.2.103",
"npmClient": "yarn",
"useWorkspaces": true,
"command": {

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cache",
"version": "0.2.102",
"version": "0.2.103",
"description": "Generic object cache",
"main": "dist/index.js",
"scripts": {

12
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cli",
"version": "0.2.102",
"version": "0.2.103",
"main": "dist/index.js",
"license": "AGPL-3.0",
"scripts": {
@ -15,13 +15,13 @@
},
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/cache": "^0.2.103",
"@cerc-io/ipld-eth-client": "^0.2.103",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.102",
"@cerc-io/rpc-eth-client": "^0.2.102",
"@cerc-io/util": "^0.2.102",
"@cerc-io/peer": "^0.2.103",
"@cerc-io/rpc-eth-client": "^0.2.103",
"@cerc-io/util": "^0.2.103",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0",

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/codegen",
"version": "0.2.102",
"version": "0.2.103",
"description": "Code generator",
"private": true,
"main": "index.js",
@ -20,7 +20,7 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/util": "^0.2.102",
"@cerc-io/util": "^0.2.103",
"@graphql-tools/load-files": "^6.5.2",
"@npmcli/package-json": "^5.0.0",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

6
packages/codegen/src/templates/indexer-template.handlebars
View File

@ -249,7 +249,7 @@ export class Indexer implements IndexerInterface {
};
}
const { block: { number } } = await this._ethClient.getBlockByHash(blockHash);
const { block: { number } } = await this.getBlockByHash(blockHash);
const blockNumber = ethers.BigNumber.from(number).toNumber();
log('{{query.name}}: db miss, fetching from upstream server');
@ -679,6 +679,10 @@ export class Indexer implements IndexerInterface {
return this._baseIndexer.getBlocks(blockFilter);
}
async getBlockByHash (blockHash?: string): Promise<{ block: any }> {
return this._baseIndexer.getBlockByHash(blockHash);
}
async updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
return this._baseIndexer.updateSyncStatusIndexedBlock(blockHash, blockNumber, force);
}

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/solidity-mapper": "^0.2.102",
"@cerc-io/util": "^0.2.102",
"@cerc-io/cli": "^0.2.103",
"@cerc-io/ipld-eth-client": "^0.2.103",
"@cerc-io/solidity-mapper": "^0.2.103",
"@cerc-io/util": "^0.2.103",
{{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.102",
"@cerc-io/graph-node": "^0.2.103",
{{/if}}
"@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1",

10
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/graph-node",
"version": "0.2.102",
"version": "0.2.103",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.102",
"@cerc-io/solidity-mapper": "^0.2.103",
"@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/util": "^0.2.102",
"@cerc-io/cache": "^0.2.103",
"@cerc-io/ipld-eth-client": "^0.2.103",
"@cerc-io/util": "^0.2.103",
"@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0",
"bn.js": "^4.11.9",

4
packages/graph-node/test/utils/indexer.ts
View File

@ -95,6 +95,10 @@ export class Indexer implements IndexerInterface {
return undefined;
}
async getBlockByHash (blockHash?: string): Promise<{ block: any }> {
return { block: undefined };
}
async getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]> {
assert(height);
assert(isPruned);

6
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/ipld-eth-client",
"version": "0.2.102",
"version": "0.2.103",
"description": "IPLD ETH Client",
"main": "dist/index.js",
"scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.102",
"@cerc-io/util": "^0.2.102",
"@cerc-io/cache": "^0.2.103",
"@cerc-io/util": "^0.2.103",
"cross-fetch": "^3.1.4",
"debug": "^4.3.1",
"ethers": "^5.4.4",

2
packages/peer/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/peer",
"version": "0.2.102",
"version": "0.2.103",
"description": "libp2p module",
"main": "dist/index.js",
"exports": "./dist/index.js",

8
packages/rpc-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/rpc-eth-client",
"version": "0.2.102",
"version": "0.2.103",
"description": "RPC ETH Client",
"main": "dist/index.js",
"scripts": {
@ -19,9 +19,9 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/cache": "^0.2.102",
"@cerc-io/ipld-eth-client": "^0.2.102",
"@cerc-io/util": "^0.2.102",
"@cerc-io/cache": "^0.2.103",
"@cerc-io/ipld-eth-client": "^0.2.103",
"@cerc-io/util": "^0.2.103",
"chai": "^4.3.4",
"ethers": "^5.4.4",
"left-pad": "^1.3.0",

2
packages/solidity-mapper/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/solidity-mapper",
"version": "0.2.102",
"version": "0.2.103",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/test",
"version": "0.2.102",
"version": "0.2.103",
"main": "dist/index.js",
"license": "AGPL-3.0",
"private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/tracing-client",
"version": "0.2.102",
"version": "0.2.103",
"description": "ETH VM tracing client",
"main": "dist/index.js",
"scripts": {

8
packages/util/package.json
View File

@ -1,13 +1,13 @@
{
"name": "@cerc-io/util",
"version": "0.2.102",
"version": "0.2.103",
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.102",
"@cerc-io/solidity-mapper": "^0.2.102",
"@cerc-io/peer": "^0.2.103",
"@cerc-io/solidity-mapper": "^0.2.103",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4",
@ -54,7 +54,7 @@
"yargs": "^17.0.1"
},
"devDependencies": {
"@cerc-io/cache": "^0.2.102",
"@cerc-io/cache": "^0.2.103",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14",

55
packages/util/src/common.ts
View File

@ -71,6 +71,22 @@ export const fetchBlocksAtHeight = async (
// Try fetching blocks from eth-server until found.
while (!blocks.length) {
const { block: latestBlock } = await indexer.getBlockByHash();
const blockProcessingOffset = jobQueueConfig.blockProcessingOffset ?? 0;
// Process block if it is blockProcessingOffset blocks behind latest block
if (latestBlock.number < blockNumber + blockProcessingOffset) {
// Check number of retries for fetching new block
if (jobQueueConfig.maxNewBlockRetries && newBlockRetries > jobQueueConfig.maxNewBlockRetries) {
throw new Error(NEW_BLOCK_MAX_RETRIES_ERROR);
}
newBlockRetries++;
log(`Latest block: ${latestBlock.number}, blockProcessingOffset: ${blockProcessingOffset}; retry block to process: ${blockNumber} after ${jobQueueConfig.blockDelayInMilliSecs}ms`);
await wait(jobQueueConfig.blockDelayInMilliSecs);
continue;
}
console.time(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
const ethFullBlocks = await indexer.getBlocks({ blockNumber });
console.timeEnd(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
@ -84,32 +100,21 @@ export const fetchBlocksAtHeight = async (
// Fitler null blocks
blocks = ethFullBlocks.filter(block => Boolean(block)) as EthFullBlock[];
assert(blocks.length, `Blocks at ${blockNumber} should exist as latest block is ${latestBlock}`);
if (!blocks.length) {
log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`);
// Check number of retries for fetching new block
if (jobQueueConfig.maxNewBlockRetries && newBlockRetries > jobQueueConfig.maxNewBlockRetries) {
throw new Error(NEW_BLOCK_MAX_RETRIES_ERROR);
}
newBlockRetries++;
await wait(jobQueueConfig.blockDelayInMilliSecs);
} else {
blocks.forEach(block => {
blockAndEventsMap.set(
block.blockHash,
{
// Block is set later in job-runner when saving to database
block: {} as BlockProgressInterface,
events: [],
ethFullBlock: block,
// Transactions are set later in job-runner when fetching events
ethFullTransactions: []
}
);
});
}
blocks.forEach(block => {
blockAndEventsMap.set(
block.blockHash,
{
// Block is set later in job-runner when saving to database
block: {} as BlockProgressInterface,
events: [],
ethFullBlock: block,
// Transactions are set later in job-runner when fetching events
ethFullTransactions: []
}
);
});
}
assert(blocks.length, 'Blocks not fetched');

16
packages/util/src/events.ts
View File

@ -14,7 +14,6 @@ import { createPruningJob, processBlockByNumber } from './common';
import { OrderDirection } from './database';
import { HistoricalJobData, HistoricalJobResponseData } from './job-runner';
import { JobQueueConfig, ServerConfig } from './config';
import { wait } from './misc';
const EVENT = 'event';
const BLOCK_PROGRESS_EVENT = 'block-progress-event';
@ -105,7 +104,7 @@ export class EventWatcher {
// Get latest block in chain and sync status from DB
// Also get historical-processing queue size
const [{ block: latestBlock }, syncStatus, historicalProcessingQueueSize] = await Promise.all([
this._ethClient.getBlockByHash(),
this._indexer.getBlockByHash(),
this._indexer.getSyncStatus(),
this._jobQueue.getQueueSize(QUEUE_HISTORICAL_PROCESSING, 'completed')
]);
@ -196,18 +195,7 @@ export class EventWatcher {
}
if (isComplete) {
while (true) {
const { block: latestBlock } = await this._ethClient.getBlockByHash();
// Process block if it is blockProcessingOffset blocks behind latest block
if (latestBlock.number >= blockNumber + (this._config.jobQueue.blockProcessingOffset ?? 0)) {
await processBlockByNumber(this._jobQueue, blockNumber + 1);
break;
}
log(`Latest block: ${latestBlock.number}; retry next block to process: ${blockNumber + 1} after ${this._config.jobQueue.blockDelayInMilliSecs}ms`);
await wait(this._config.jobQueue.blockDelayInMilliSecs);
}
await processBlockByNumber(this._jobQueue, blockNumber + 1);
}
}
}

4
packages/util/src/indexer.ts
View File

@ -394,6 +394,10 @@ export class Indexer {
return blocks;
}
async getBlockByHash (blockHash?: string): Promise<{ block: any }> {
return this._ethClient.getBlockByHash(blockHash);
}
async getBlockProgress (blockHash: string): Promise<BlockProgressInterface | undefined> {
return this._db.getBlockProgress(blockHash);
}

6
packages/util/src/metrics.ts
View File

@ -82,6 +82,12 @@ export const isSyncingHistoricalBlocks = new client.Gauge({
});
isSyncingHistoricalBlocks.set(Number(undefined));
export const ethRpcCount = new client.Counter({
name: 'watcher_eth_rpc_total',
help: 'Total number of ETH RPC requests',
labelNames: ['method', 'provider']
});
export const ethRpcErrors = new client.Counter({
name: 'watcher_eth_rpc_errors',
help: 'Number of ETH RPC request errors',

3
packages/util/src/misc.ts
View File

@ -22,7 +22,7 @@ import { ResultEvent } from './indexer';
import { EventInterface, EthFullBlock, EthFullTransaction } from './types';
import { BlockHeight } from './database';
import { Transaction } from './graph/utils';
import { ethRpcErrors, ethRpcRequestDuration } from './metrics';
import { ethRpcCount, ethRpcErrors, ethRpcRequestDuration } from './metrics';
const JSONbigNative = JSONbig({ useNativeBigInt: true });
@ -379,6 +379,7 @@ export class MonitoredStaticJsonRpcProvider extends providers.StaticJsonRpcProvi
// Rethrow the error
throw err;
} finally {
ethRpcCount.inc({ method, provider: this.connection.url }, 1);
endTimer();
}
}

1
packages/util/src/types.ts
View File

@ -171,6 +171,7 @@ export interface IndexerInterface {
getSyncStatus (): Promise<SyncStatusInterface | undefined>
getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<Array<EthFullBlock | null>>
getBlockByHash (blockHash?: string): Promise<{ block: any }>
getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]>
getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined>
getLatestStateIndexedBlock (): Promise<BlockProgressInterface>