cerc-io/watcher-ts
10
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2025-07-27 02:32:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle null block error in rpc-eth-client (#477)

Browse Source 
* Remove block-size-cache util methods as full block with size fetched already

* Handle null block error in rpc-eth-client

* Upgrade package versions
This commit is contained in:
Nabarun Gogoi 2023-11-17 11:58:43 +05:30 committed by GitHub
parent e54f4c9276
commit c2070a80cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 119 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "0.2.72",
"version": "0.2.73",
"npmClient": "yarn",
"useWorkspaces": true,
"command": {

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cache",
"version": "0.2.72",
"version": "0.2.73",
"description": "Generic object cache",
"main": "dist/index.js",
"scripts": {

12
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cli",
"version": "0.2.72",
"version": "0.2.73",
"main": "dist/index.js",
"license": "AGPL-3.0",
"scripts": {
@ -12,13 +12,13 @@
},
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.72",
"@cerc-io/ipld-eth-client": "^0.2.72",
"@cerc-io/cache": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.72",
"@cerc-io/rpc-eth-client": "^0.2.72",
"@cerc-io/util": "^0.2.72",
"@cerc-io/peer": "^0.2.73",
"@cerc-io/rpc-eth-client": "^0.2.73",
"@cerc-io/util": "^0.2.73",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0",

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/codegen",
"version": "0.2.72",
"version": "0.2.73",
"description": "Code generator",
"private": true,
"main": "index.js",
@ -20,7 +20,7 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/util": "^0.2.72",
"@cerc-io/util": "^0.2.73",
"@graphql-tools/load-files": "^6.5.2",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
"@solidity-parser/parser": "^0.13.2",

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.72",
"@cerc-io/ipld-eth-client": "^0.2.72",
"@cerc-io/solidity-mapper": "^0.2.72",
"@cerc-io/util": "^0.2.72",
"@cerc-io/cli": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/solidity-mapper": "^0.2.73",
"@cerc-io/util": "^0.2.73",
{{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.72",
"@cerc-io/graph-node": "^0.2.73",
{{/if}}
"@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1",

10
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/graph-node",
"version": "0.2.72",
"version": "0.2.73",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.72",
"@cerc-io/solidity-mapper": "^0.2.73",
"@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.72",
"@cerc-io/ipld-eth-client": "^0.2.72",
"@cerc-io/util": "^0.2.72",
"@cerc-io/cache": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/util": "^0.2.73",
"@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0",
"bn.js": "^4.11.9",

6
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/ipld-eth-client",
"version": "0.2.72",
"version": "0.2.73",
"description": "IPLD ETH Client",
"main": "dist/index.js",
"scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.72",
"@cerc-io/util": "^0.2.72",
"@cerc-io/cache": "^0.2.73",
"@cerc-io/util": "^0.2.73",
"cross-fetch": "^3.1.4",
"debug": "^4.3.1",
"ethers": "^5.4.4",

2
packages/peer/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/peer",
"version": "0.2.72",
"version": "0.2.73",
"description": "libp2p module",
"main": "dist/index.js",
"exports": "./dist/index.js",

8
packages/rpc-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/rpc-eth-client",
"version": "0.2.72",
"version": "0.2.73",
"description": "RPC ETH Client",
"main": "dist/index.js",
"scripts": {
@ -19,9 +19,9 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/cache": "^0.2.72",
"@cerc-io/ipld-eth-client": "^0.2.72",
"@cerc-io/util": "^0.2.72",
"@cerc-io/cache": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/util": "^0.2.73",
"chai": "^4.3.4",
"ethers": "^5.4.4",
"left-pad": "^1.3.0",

34
packages/rpc-eth-client/src/eth-client.ts
View File

@ -10,6 +10,7 @@ import { encodeHeader, escapeHexString, EthClient as EthClientInterface, EthFull
import { padKey } from '@cerc-io/ipld-eth-client';
const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
const NULL_BLOCK_ERROR = 'requested epoch was a null round';
export interface Config {
cache: Cache | undefined;
@ -93,10 +94,7 @@ export class EthClient implements EthClientInterface {
}
];
} catch (err: any) {
// Check and ignore future block error
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
throw err;
}
return this._handleGetBlockErrors(err);
} finally {
console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
}
@ -138,10 +136,7 @@ export class EthClient implements EthClientInterface {
];
}
} catch (err: any) {
// Check and ignore future block error
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
throw err;
}
return this._handleGetBlockErrors(err);
} finally {
console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
}
@ -153,7 +148,7 @@ export class EthClient implements EthClientInterface {
};
}
async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<EthFullBlock[]> {
async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
const blockNumberHex = blockNumber ? utils.hexValue(blockNumber) : undefined;
const blockHashOrBlockNumber = blockHash ?? blockNumberHex;
assert(blockHashOrBlockNumber);
@ -207,10 +202,7 @@ export class EthClient implements EthClientInterface {
}];
}
} catch (err: any) {
// Check and ignore future block error
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
throw err;
}
return this._handleGetBlockErrors(err);
} finally {
console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
}
@ -379,4 +371,20 @@ export class EthClient implements EthClientInterface {
return result;
}
_handleGetBlockErrors (err: any): Array<null> {
if (err.code === errors.SERVER_ERROR && err.error) {
// Check null block error and return null array
if (err.error.message === NULL_BLOCK_ERROR) {
return [null];
}
// Check and ignore future block error
if (err.error.message === FUTURE_BLOCK_ERROR) {
return [];
}
}
throw err;
}
}

2
packages/solidity-mapper/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/solidity-mapper",
"version": "0.2.72",
"version": "0.2.73",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/test",
"version": "0.2.72",
"version": "0.2.73",
"main": "dist/index.js",
"license": "AGPL-3.0",
"private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/tracing-client",
"version": "0.2.72",
"version": "0.2.73",
"description": "ETH VM tracing client",
"main": "dist/index.js",
"scripts": {

8
packages/util/package.json
View File

@ -1,13 +1,13 @@
{
"name": "@cerc-io/util",
"version": "0.2.72",
"version": "0.2.73",
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.72",
"@cerc-io/solidity-mapper": "^0.2.72",
"@cerc-io/peer": "^0.2.73",
"@cerc-io/solidity-mapper": "^0.2.73",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4",
@ -52,7 +52,7 @@
"yargs": "^17.0.1"
},
"devDependencies": {
"@cerc-io/cache": "^0.2.72",
"@cerc-io/cache": "^0.2.73",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14",

81
packages/util/src/block-size-cache.ts
View File

@ -1,81 +0,0 @@
//
// Copyright 2022 Vulcanize, Inc.
//
import { utils, providers, errors } from 'ethers';
import debug from 'debug';
import { NULL_BLOCK_ERROR } from './constants';
const log = debug('vulcanize:block-size-cache');
// Number of blocks to cache after current block being processed.
const BLOCK_SIZE_CACHE_BUFFER = 10;
// Block height interval at which blockSizeMap is cleared.
// If the block being processed is divisible by BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL then blocks below that height are removed from the map.
const BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL = 50;
const blockSizeMap: Map<string, { size: string, blockNumber: number }> = new Map();
let blockSizeMapLatestHeight = -1;
export const getCachedBlockSize = async (provider: providers.JsonRpcProvider, blockHash: string, blockNumber: number): Promise<string> => {
const block = blockSizeMap.get(blockHash);
cacheBlockSizesAsync(provider, blockNumber);
if (!block) {
console.time(`time:misc#getCachedBlockSize-eth_getBlockByHash-${blockNumber}`);
const { size } = await provider.send('eth_getBlockByHash', [blockHash, false]);
console.timeEnd(`time:misc#getCachedBlockSize-eth_getBlockByHash-${blockNumber}`);
return size;
}
return block.size;
};
const cacheBlockSizesAsync = async (provider: providers.JsonRpcProvider, blockNumber: number): Promise<void> => {
const endBlockHeight = blockNumber + BLOCK_SIZE_CACHE_BUFFER;
if (blockSizeMapLatestHeight < 0) {
blockSizeMapLatestHeight = blockNumber;
}
if (endBlockHeight > blockSizeMapLatestHeight) {
const startBlockHeight = Math.max(blockNumber, blockSizeMapLatestHeight + 1);
blockSizeMapLatestHeight = endBlockHeight;
// Start prefetching blocks after latest height in blockSizeMap.
for (let i = startBlockHeight; i <= endBlockHeight; i++) {
try {
console.time(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
const block = await provider.send('eth_getBlockByNumber', [utils.hexStripZeros(utils.hexlify(i)), false]);
if (block) {
const { size, hash } = block;
blockSizeMap.set(hash, { size, blockNumber: i });
} else {
log(`No block found at height ${i}`);
}
} catch (err: any) {
// Handle null block error in case of Lotus EVM
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === NULL_BLOCK_ERROR)) {
throw err;
}
log(`Block ${i} requested was null (FEVM); Fetching next block`);
} finally {
console.timeEnd(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
}
}
}
// At interval clear previous blocks below height blockNumber from map.
if (blockNumber % BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL === 0) {
log(`cacheBlockSizesAsync-clear-map-below-${blockNumber}`);
const previousBlockHashes = Array.from(blockSizeMap.entries())
.filter(([, value]) => value.blockNumber <= blockNumber)
.map(([blockHash]) => blockHash);
previousBlockHashes.forEach(blockHash => blockSizeMap.delete(blockHash));
}
};

31
packages/util/src/common.ts
View File

@ -1,7 +1,6 @@
import debug from 'debug';
import assert from 'assert';
import { DeepPartial } from 'typeorm';
import { errors } from 'ethers';
import JSONbig from 'json-bigint';
import {
@ -10,8 +9,7 @@ import {
QUEUE_BLOCK_CHECKPOINT,
JOB_KIND_PRUNE,
JOB_KIND_INDEX,
UNKNOWN_EVENT_NAME,
NULL_BLOCK_ERROR
UNKNOWN_EVENT_NAME
} from './constants';
import { JobQueue } from './job-queue';
import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock } from './types';
@ -78,9 +76,19 @@ export const fetchBlocksAtHeight = async (
// Try fetching blocks from eth-server until found.
while (!blocks.length) {
try {
console.time('time:common#_fetchBlocks-eth-server');
blocks = await indexer.getBlocks({ blockNumber });
console.time(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
const ethFullBlocks = await indexer.getBlocks({ blockNumber });
console.timeEnd(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
// Check if all blocks are null and increment blockNumber to index next block number
if (ethFullBlocks.every(block => block === null)) {
blockNumber++;
log(`Block ${blockNumber} requested was null (FEVM); Fetching next block`);
continue;
}
// Fitler null blocks
blocks = ethFullBlocks.filter(block => Boolean(block)) as EthFullBlock[];
if (!blocks.length) {
log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`);
@ -100,17 +108,6 @@ export const fetchBlocksAtHeight = async (
);
});
}
} catch (err: any) {
// Handle null block error in case of Lotus EVM
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === NULL_BLOCK_ERROR)) {
throw err;
}
log(`Block ${blockNumber} requested was null (FEVM); Fetching next block`);
blockNumber++;
} finally {
console.timeEnd('time:common#_fetchBlocks-eth-server');
}
}
assert(blocks.length, 'Blocks not fetched');

2
packages/util/src/constants.ts
View File

@ -30,5 +30,3 @@ export const DEFAULT_PREFETCH_BATCH_SIZE = 10;
export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB
export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber'];
export const NULL_BLOCK_ERROR = 'requested epoch was a null round';

7
packages/util/src/fill.ts
View File

@ -5,7 +5,7 @@
import debug from 'debug';
import { JobQueue } from './job-queue';
import { IndexerInterface } from './types';
import { EthFullBlock, IndexerInterface } from './types';
import { wait } from './misc';
import { processBlockByNumber } from './common';
import { DEFAULT_PREFETCH_BATCH_SIZE } from './constants';
@ -107,7 +107,7 @@ const prefetchBlocks = async (
let blockNumbers = [...Array(batchEndBlock - i).keys()].map(n => n + i);
log('Fetching blockNumbers:', blockNumbers);
let blocks = [];
let blocks: EthFullBlock[] = [];
// Fetch blocks again if there are missing blocks.
while (true) {
@ -117,7 +117,8 @@ const prefetchBlocks = async (
const missingIndex = res.findIndex(blocks => blocks.length === 0);
if (missingIndex < 0) {
blocks = res.flat();
// Filter null blocks
blocks = res.flat().filter(block => Boolean(block)) as EthFullBlock[];
break;
}

4
packages/util/src/index-block.ts
View File

@ -22,7 +22,9 @@ export const indexBlock = async (
console.time('time:index-block#getBlocks-ipld-eth-server');
const blocks = await indexer.getBlocks({ blockNumber: argv.block });
blockProgressEntities = blocks.map((block: any): Partial<BlockProgressInterface> => {
// Filter null blocks and transform to BlockProgress type
blockProgressEntities = blocks.filter(block => Boolean(block))
.map((block: any): Partial<BlockProgressInterface> => {
block.blockTimestamp = Number(block.timestamp);
block.blockNumber = Number(block.blockNumber);

8
packages/util/src/indexer.ts
View File

@ -291,7 +291,7 @@ export class Indexer {
return res;
}
async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<EthFullBlock[]> {
async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
assert(blockFilter.blockHash || blockFilter.blockNumber);
const blocks = await this._ethClient.getFullBlocks(blockFilter);
@ -394,6 +394,7 @@ export class Indexer {
console.time(`time:indexer#fetchAndSaveFilteredEventsAndBlocks-fetch-blocks-txs-${fromBlock}-${toBlock}`);
const blocksPromises = Array.from(blockLogsMap.keys()).map(async (blockHash) => {
const [fullBlock] = await this._ethClient.getFullBlocks({ blockHash });
assert(fullBlock);
const block = {
...fullBlock,
@ -401,7 +402,10 @@ export class Indexer {
blockNumber: Number(fullBlock.blockNumber)
};
return { block, fullBlock } as { block: DeepPartial<BlockProgressInterface>; fullBlock: EthFullBlock };
return {
block: block as DeepPartial<BlockProgressInterface>,
fullBlock
};
});
const ethFullTxPromises = txHashes.map(async txHash => {

8
packages/util/src/job-runner.ts
View File

@ -521,16 +521,15 @@ export class JobRunner {
const newPriority = (priority || 0) + 1;
if (!parentBlock || parentBlock.blockHash !== parentHash) {
const blocks = await this._indexer.getBlocks({ blockHash: parentHash });
const [block] = await this._indexer.getBlocks({ blockHash: parentHash });
if (!blocks.length) {
if (!block) {
const message = `No blocks at parentHash ${parentHash}, aborting`;
log(message);
throw new Error(message);
}
blocks.forEach(block => {
this._blockAndEventsMap.set(
block.blockHash,
{
@ -542,9 +541,8 @@ export class JobRunner {
ethFullTransactions: []
}
);
});
const [{ cid: parentCid, blockNumber: parentBlockNumber, parentHash: grandparentHash, timestamp: parentTimestamp }] = blocks;
const { cid: parentCid, blockNumber: parentBlockNumber, parentHash: grandparentHash, timestamp: parentTimestamp } = block;
await this.jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, {
kind: JOB_KIND_INDEX,

4
packages/util/src/types.ts
View File

@ -138,7 +138,7 @@ export interface EthClient {
getFullBlocks({ blockNumber, blockHash }: {
blockNumber?: number;
blockHash?: string;
}): Promise<EthFullBlock[]>;
}): Promise<Array<EthFullBlock | null>>;
getFullTransaction(txHash: string, blockNumber?: number): Promise<EthFullTransaction>;
getBlockByHash(blockHash?: string): Promise<any>;
getLogs(vars: {
@ -167,7 +167,7 @@ export interface IndexerInterface {
getEvent (id: string): Promise<EventInterface | undefined>
getSyncStatus (): Promise<SyncStatusInterface | undefined>
getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<EthFullBlock[]>
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<Array<EthFullBlock | null>>
getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]>
getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined>
getLatestStateIndexedBlock (): Promise<BlockProgressInterface>