mirror of
https://github.com/cerc-io/watcher-ts
synced 2025-07-27 02:32:07 +00:00
Handle null block error in rpc-eth-client (#477)
* Remove block-size-cache util methods as full block with size fetched already * Handle null block error in rpc-eth-client * Upgrade package versions
This commit is contained in:
parent
e54f4c9276
commit
c2070a80cb
@ -2,7 +2,7 @@
|
|||||||
"packages": [
|
"packages": [
|
||||||
"packages/*"
|
"packages/*"
|
||||||
],
|
],
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"npmClient": "yarn",
|
"npmClient": "yarn",
|
||||||
"useWorkspaces": true,
|
"useWorkspaces": true,
|
||||||
"command": {
|
"command": {
|
||||||
|
2
packages/cache/package.json
vendored
2
packages/cache/package.json
vendored
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/cache",
|
"name": "@cerc-io/cache",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"description": "Generic object cache",
|
"description": "Generic object cache",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/cli",
|
"name": "@cerc-io/cli",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"license": "AGPL-3.0",
|
"license": "AGPL-3.0",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@ -12,13 +12,13 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@apollo/client": "^3.7.1",
|
"@apollo/client": "^3.7.1",
|
||||||
"@cerc-io/cache": "^0.2.72",
|
"@cerc-io/cache": "^0.2.73",
|
||||||
"@cerc-io/ipld-eth-client": "^0.2.72",
|
"@cerc-io/ipld-eth-client": "^0.2.73",
|
||||||
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
|
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
|
||||||
"@cerc-io/nitro-node": "^0.1.15",
|
"@cerc-io/nitro-node": "^0.1.15",
|
||||||
"@cerc-io/peer": "^0.2.72",
|
"@cerc-io/peer": "^0.2.73",
|
||||||
"@cerc-io/rpc-eth-client": "^0.2.72",
|
"@cerc-io/rpc-eth-client": "^0.2.73",
|
||||||
"@cerc-io/util": "^0.2.72",
|
"@cerc-io/util": "^0.2.73",
|
||||||
"@ethersproject/providers": "^5.4.4",
|
"@ethersproject/providers": "^5.4.4",
|
||||||
"@graphql-tools/utils": "^9.1.1",
|
"@graphql-tools/utils": "^9.1.1",
|
||||||
"@ipld/dag-cbor": "^8.0.0",
|
"@ipld/dag-cbor": "^8.0.0",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/codegen",
|
"name": "@cerc-io/codegen",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"description": "Code generator",
|
"description": "Code generator",
|
||||||
"private": true,
|
"private": true,
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
@ -20,7 +20,7 @@
|
|||||||
},
|
},
|
||||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@cerc-io/util": "^0.2.72",
|
"@cerc-io/util": "^0.2.73",
|
||||||
"@graphql-tools/load-files": "^6.5.2",
|
"@graphql-tools/load-files": "^6.5.2",
|
||||||
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
|
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
|
||||||
"@solidity-parser/parser": "^0.13.2",
|
"@solidity-parser/parser": "^0.13.2",
|
||||||
|
@ -41,12 +41,12 @@
|
|||||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@apollo/client": "^3.3.19",
|
"@apollo/client": "^3.3.19",
|
||||||
"@cerc-io/cli": "^0.2.72",
|
"@cerc-io/cli": "^0.2.73",
|
||||||
"@cerc-io/ipld-eth-client": "^0.2.72",
|
"@cerc-io/ipld-eth-client": "^0.2.73",
|
||||||
"@cerc-io/solidity-mapper": "^0.2.72",
|
"@cerc-io/solidity-mapper": "^0.2.73",
|
||||||
"@cerc-io/util": "^0.2.72",
|
"@cerc-io/util": "^0.2.73",
|
||||||
{{#if (subgraphPath)}}
|
{{#if (subgraphPath)}}
|
||||||
"@cerc-io/graph-node": "^0.2.72",
|
"@cerc-io/graph-node": "^0.2.73",
|
||||||
{{/if}}
|
{{/if}}
|
||||||
"@ethersproject/providers": "^5.4.4",
|
"@ethersproject/providers": "^5.4.4",
|
||||||
"debug": "^4.3.1",
|
"debug": "^4.3.1",
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/graph-node",
|
"name": "@cerc-io/graph-node",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"license": "AGPL-3.0",
|
"license": "AGPL-3.0",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@cerc-io/solidity-mapper": "^0.2.72",
|
"@cerc-io/solidity-mapper": "^0.2.73",
|
||||||
"@ethersproject/providers": "^5.4.4",
|
"@ethersproject/providers": "^5.4.4",
|
||||||
"@graphprotocol/graph-ts": "^0.22.0",
|
"@graphprotocol/graph-ts": "^0.22.0",
|
||||||
"@nomiclabs/hardhat-ethers": "^2.0.2",
|
"@nomiclabs/hardhat-ethers": "^2.0.2",
|
||||||
@ -51,9 +51,9 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@apollo/client": "^3.3.19",
|
"@apollo/client": "^3.3.19",
|
||||||
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
|
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
|
||||||
"@cerc-io/cache": "^0.2.72",
|
"@cerc-io/cache": "^0.2.73",
|
||||||
"@cerc-io/ipld-eth-client": "^0.2.72",
|
"@cerc-io/ipld-eth-client": "^0.2.73",
|
||||||
"@cerc-io/util": "^0.2.72",
|
"@cerc-io/util": "^0.2.73",
|
||||||
"@types/json-diff": "^0.5.2",
|
"@types/json-diff": "^0.5.2",
|
||||||
"@types/yargs": "^17.0.0",
|
"@types/yargs": "^17.0.0",
|
||||||
"bn.js": "^4.11.9",
|
"bn.js": "^4.11.9",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/ipld-eth-client",
|
"name": "@cerc-io/ipld-eth-client",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"description": "IPLD ETH Client",
|
"description": "IPLD ETH Client",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@ -20,8 +20,8 @@
|
|||||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@apollo/client": "^3.7.1",
|
"@apollo/client": "^3.7.1",
|
||||||
"@cerc-io/cache": "^0.2.72",
|
"@cerc-io/cache": "^0.2.73",
|
||||||
"@cerc-io/util": "^0.2.72",
|
"@cerc-io/util": "^0.2.73",
|
||||||
"cross-fetch": "^3.1.4",
|
"cross-fetch": "^3.1.4",
|
||||||
"debug": "^4.3.1",
|
"debug": "^4.3.1",
|
||||||
"ethers": "^5.4.4",
|
"ethers": "^5.4.4",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/peer",
|
"name": "@cerc-io/peer",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"description": "libp2p module",
|
"description": "libp2p module",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": "./dist/index.js",
|
"exports": "./dist/index.js",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/rpc-eth-client",
|
"name": "@cerc-io/rpc-eth-client",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"description": "RPC ETH Client",
|
"description": "RPC ETH Client",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@ -19,9 +19,9 @@
|
|||||||
},
|
},
|
||||||
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@cerc-io/cache": "^0.2.72",
|
"@cerc-io/cache": "^0.2.73",
|
||||||
"@cerc-io/ipld-eth-client": "^0.2.72",
|
"@cerc-io/ipld-eth-client": "^0.2.73",
|
||||||
"@cerc-io/util": "^0.2.72",
|
"@cerc-io/util": "^0.2.73",
|
||||||
"chai": "^4.3.4",
|
"chai": "^4.3.4",
|
||||||
"ethers": "^5.4.4",
|
"ethers": "^5.4.4",
|
||||||
"left-pad": "^1.3.0",
|
"left-pad": "^1.3.0",
|
||||||
|
@ -10,6 +10,7 @@ import { encodeHeader, escapeHexString, EthClient as EthClientInterface, EthFull
|
|||||||
import { padKey } from '@cerc-io/ipld-eth-client';
|
import { padKey } from '@cerc-io/ipld-eth-client';
|
||||||
|
|
||||||
const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
|
const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
|
||||||
|
const NULL_BLOCK_ERROR = 'requested epoch was a null round';
|
||||||
|
|
||||||
export interface Config {
|
export interface Config {
|
||||||
cache: Cache | undefined;
|
cache: Cache | undefined;
|
||||||
@ -93,10 +94,7 @@ export class EthClient implements EthClientInterface {
|
|||||||
}
|
}
|
||||||
];
|
];
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
// Check and ignore future block error
|
return this._handleGetBlockErrors(err);
|
||||||
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
|
console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
|
||||||
}
|
}
|
||||||
@ -138,10 +136,7 @@ export class EthClient implements EthClientInterface {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
// Check and ignore future block error
|
return this._handleGetBlockErrors(err);
|
||||||
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
|
console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
|
||||||
}
|
}
|
||||||
@ -153,7 +148,7 @@ export class EthClient implements EthClientInterface {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<EthFullBlock[]> {
|
async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
|
||||||
const blockNumberHex = blockNumber ? utils.hexValue(blockNumber) : undefined;
|
const blockNumberHex = blockNumber ? utils.hexValue(blockNumber) : undefined;
|
||||||
const blockHashOrBlockNumber = blockHash ?? blockNumberHex;
|
const blockHashOrBlockNumber = blockHash ?? blockNumberHex;
|
||||||
assert(blockHashOrBlockNumber);
|
assert(blockHashOrBlockNumber);
|
||||||
@ -207,10 +202,7 @@ export class EthClient implements EthClientInterface {
|
|||||||
}];
|
}];
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
// Check and ignore future block error
|
return this._handleGetBlockErrors(err);
|
||||||
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
|
console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
|
||||||
}
|
}
|
||||||
@ -379,4 +371,20 @@ export class EthClient implements EthClientInterface {
|
|||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_handleGetBlockErrors (err: any): Array<null> {
|
||||||
|
if (err.code === errors.SERVER_ERROR && err.error) {
|
||||||
|
// Check null block error and return null array
|
||||||
|
if (err.error.message === NULL_BLOCK_ERROR) {
|
||||||
|
return [null];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check and ignore future block error
|
||||||
|
if (err.error.message === FUTURE_BLOCK_ERROR) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/solidity-mapper",
|
"name": "@cerc-io/solidity-mapper",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"license": "AGPL-3.0",
|
"license": "AGPL-3.0",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/test",
|
"name": "@cerc-io/test",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"license": "AGPL-3.0",
|
"license": "AGPL-3.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/tracing-client",
|
"name": "@cerc-io/tracing-client",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"description": "ETH VM tracing client",
|
"description": "ETH VM tracing client",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
{
|
{
|
||||||
"name": "@cerc-io/util",
|
"name": "@cerc-io/util",
|
||||||
"version": "0.2.72",
|
"version": "0.2.73",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"license": "AGPL-3.0",
|
"license": "AGPL-3.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@apollo/utils.keyvaluecache": "^1.0.1",
|
"@apollo/utils.keyvaluecache": "^1.0.1",
|
||||||
"@cerc-io/nitro-node": "^0.1.15",
|
"@cerc-io/nitro-node": "^0.1.15",
|
||||||
"@cerc-io/peer": "^0.2.72",
|
"@cerc-io/peer": "^0.2.73",
|
||||||
"@cerc-io/solidity-mapper": "^0.2.72",
|
"@cerc-io/solidity-mapper": "^0.2.73",
|
||||||
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
|
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
|
||||||
"@ethersproject/properties": "^5.7.0",
|
"@ethersproject/properties": "^5.7.0",
|
||||||
"@ethersproject/providers": "^5.4.4",
|
"@ethersproject/providers": "^5.4.4",
|
||||||
@ -52,7 +52,7 @@
|
|||||||
"yargs": "^17.0.1"
|
"yargs": "^17.0.1"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@cerc-io/cache": "^0.2.72",
|
"@cerc-io/cache": "^0.2.73",
|
||||||
"@nomiclabs/hardhat-waffle": "^2.0.1",
|
"@nomiclabs/hardhat-waffle": "^2.0.1",
|
||||||
"@types/bunyan": "^1.8.8",
|
"@types/bunyan": "^1.8.8",
|
||||||
"@types/express": "^4.17.14",
|
"@types/express": "^4.17.14",
|
||||||
|
@ -1,81 +0,0 @@
|
|||||||
//
|
|
||||||
// Copyright 2022 Vulcanize, Inc.
|
|
||||||
//
|
|
||||||
|
|
||||||
import { utils, providers, errors } from 'ethers';
|
|
||||||
import debug from 'debug';
|
|
||||||
|
|
||||||
import { NULL_BLOCK_ERROR } from './constants';
|
|
||||||
|
|
||||||
const log = debug('vulcanize:block-size-cache');
|
|
||||||
|
|
||||||
// Number of blocks to cache after current block being processed.
|
|
||||||
const BLOCK_SIZE_CACHE_BUFFER = 10;
|
|
||||||
// Block height interval at which blockSizeMap is cleared.
|
|
||||||
// If the block being processed is divisible by BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL then blocks below that height are removed from the map.
|
|
||||||
const BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL = 50;
|
|
||||||
|
|
||||||
const blockSizeMap: Map<string, { size: string, blockNumber: number }> = new Map();
|
|
||||||
let blockSizeMapLatestHeight = -1;
|
|
||||||
|
|
||||||
export const getCachedBlockSize = async (provider: providers.JsonRpcProvider, blockHash: string, blockNumber: number): Promise<string> => {
|
|
||||||
const block = blockSizeMap.get(blockHash);
|
|
||||||
cacheBlockSizesAsync(provider, blockNumber);
|
|
||||||
|
|
||||||
if (!block) {
|
|
||||||
console.time(`time:misc#getCachedBlockSize-eth_getBlockByHash-${blockNumber}`);
|
|
||||||
const { size } = await provider.send('eth_getBlockByHash', [blockHash, false]);
|
|
||||||
console.timeEnd(`time:misc#getCachedBlockSize-eth_getBlockByHash-${blockNumber}`);
|
|
||||||
|
|
||||||
return size;
|
|
||||||
}
|
|
||||||
|
|
||||||
return block.size;
|
|
||||||
};
|
|
||||||
|
|
||||||
const cacheBlockSizesAsync = async (provider: providers.JsonRpcProvider, blockNumber: number): Promise<void> => {
|
|
||||||
const endBlockHeight = blockNumber + BLOCK_SIZE_CACHE_BUFFER;
|
|
||||||
|
|
||||||
if (blockSizeMapLatestHeight < 0) {
|
|
||||||
blockSizeMapLatestHeight = blockNumber;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (endBlockHeight > blockSizeMapLatestHeight) {
|
|
||||||
const startBlockHeight = Math.max(blockNumber, blockSizeMapLatestHeight + 1);
|
|
||||||
blockSizeMapLatestHeight = endBlockHeight;
|
|
||||||
|
|
||||||
// Start prefetching blocks after latest height in blockSizeMap.
|
|
||||||
for (let i = startBlockHeight; i <= endBlockHeight; i++) {
|
|
||||||
try {
|
|
||||||
console.time(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
|
|
||||||
const block = await provider.send('eth_getBlockByNumber', [utils.hexStripZeros(utils.hexlify(i)), false]);
|
|
||||||
|
|
||||||
if (block) {
|
|
||||||
const { size, hash } = block;
|
|
||||||
blockSizeMap.set(hash, { size, blockNumber: i });
|
|
||||||
} else {
|
|
||||||
log(`No block found at height ${i}`);
|
|
||||||
}
|
|
||||||
} catch (err: any) {
|
|
||||||
// Handle null block error in case of Lotus EVM
|
|
||||||
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === NULL_BLOCK_ERROR)) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
|
|
||||||
log(`Block ${i} requested was null (FEVM); Fetching next block`);
|
|
||||||
} finally {
|
|
||||||
console.timeEnd(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// At interval clear previous blocks below height blockNumber from map.
|
|
||||||
if (blockNumber % BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL === 0) {
|
|
||||||
log(`cacheBlockSizesAsync-clear-map-below-${blockNumber}`);
|
|
||||||
const previousBlockHashes = Array.from(blockSizeMap.entries())
|
|
||||||
.filter(([, value]) => value.blockNumber <= blockNumber)
|
|
||||||
.map(([blockHash]) => blockHash);
|
|
||||||
|
|
||||||
previousBlockHashes.forEach(blockHash => blockSizeMap.delete(blockHash));
|
|
||||||
}
|
|
||||||
};
|
|
@ -1,7 +1,6 @@
|
|||||||
import debug from 'debug';
|
import debug from 'debug';
|
||||||
import assert from 'assert';
|
import assert from 'assert';
|
||||||
import { DeepPartial } from 'typeorm';
|
import { DeepPartial } from 'typeorm';
|
||||||
import { errors } from 'ethers';
|
|
||||||
import JSONbig from 'json-bigint';
|
import JSONbig from 'json-bigint';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@ -10,8 +9,7 @@ import {
|
|||||||
QUEUE_BLOCK_CHECKPOINT,
|
QUEUE_BLOCK_CHECKPOINT,
|
||||||
JOB_KIND_PRUNE,
|
JOB_KIND_PRUNE,
|
||||||
JOB_KIND_INDEX,
|
JOB_KIND_INDEX,
|
||||||
UNKNOWN_EVENT_NAME,
|
UNKNOWN_EVENT_NAME
|
||||||
NULL_BLOCK_ERROR
|
|
||||||
} from './constants';
|
} from './constants';
|
||||||
import { JobQueue } from './job-queue';
|
import { JobQueue } from './job-queue';
|
||||||
import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock } from './types';
|
import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock } from './types';
|
||||||
@ -78,38 +76,37 @@ export const fetchBlocksAtHeight = async (
|
|||||||
|
|
||||||
// Try fetching blocks from eth-server until found.
|
// Try fetching blocks from eth-server until found.
|
||||||
while (!blocks.length) {
|
while (!blocks.length) {
|
||||||
try {
|
console.time(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
|
||||||
console.time('time:common#_fetchBlocks-eth-server');
|
const ethFullBlocks = await indexer.getBlocks({ blockNumber });
|
||||||
blocks = await indexer.getBlocks({ blockNumber });
|
console.timeEnd(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
|
||||||
|
|
||||||
if (!blocks.length) {
|
// Check if all blocks are null and increment blockNumber to index next block number
|
||||||
log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`);
|
if (ethFullBlocks.every(block => block === null)) {
|
||||||
await wait(jobQueueConfig.blockDelayInMilliSecs);
|
|
||||||
} else {
|
|
||||||
blocks.forEach(block => {
|
|
||||||
blockAndEventsMap.set(
|
|
||||||
block.blockHash,
|
|
||||||
{
|
|
||||||
// Block is set later in job-runner when saving to database
|
|
||||||
block: {} as BlockProgressInterface,
|
|
||||||
events: [],
|
|
||||||
ethFullBlock: block,
|
|
||||||
// Transactions are set later in job-runner when fetching events
|
|
||||||
ethFullTransactions: []
|
|
||||||
}
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (err: any) {
|
|
||||||
// Handle null block error in case of Lotus EVM
|
|
||||||
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === NULL_BLOCK_ERROR)) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
|
|
||||||
log(`Block ${blockNumber} requested was null (FEVM); Fetching next block`);
|
|
||||||
blockNumber++;
|
blockNumber++;
|
||||||
} finally {
|
log(`Block ${blockNumber} requested was null (FEVM); Fetching next block`);
|
||||||
console.timeEnd('time:common#_fetchBlocks-eth-server');
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fitler null blocks
|
||||||
|
blocks = ethFullBlocks.filter(block => Boolean(block)) as EthFullBlock[];
|
||||||
|
|
||||||
|
if (!blocks.length) {
|
||||||
|
log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`);
|
||||||
|
await wait(jobQueueConfig.blockDelayInMilliSecs);
|
||||||
|
} else {
|
||||||
|
blocks.forEach(block => {
|
||||||
|
blockAndEventsMap.set(
|
||||||
|
block.blockHash,
|
||||||
|
{
|
||||||
|
// Block is set later in job-runner when saving to database
|
||||||
|
block: {} as BlockProgressInterface,
|
||||||
|
events: [],
|
||||||
|
ethFullBlock: block,
|
||||||
|
// Transactions are set later in job-runner when fetching events
|
||||||
|
ethFullTransactions: []
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -30,5 +30,3 @@ export const DEFAULT_PREFETCH_BATCH_SIZE = 10;
|
|||||||
export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB
|
export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB
|
||||||
|
|
||||||
export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber'];
|
export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber'];
|
||||||
|
|
||||||
export const NULL_BLOCK_ERROR = 'requested epoch was a null round';
|
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
import debug from 'debug';
|
import debug from 'debug';
|
||||||
|
|
||||||
import { JobQueue } from './job-queue';
|
import { JobQueue } from './job-queue';
|
||||||
import { IndexerInterface } from './types';
|
import { EthFullBlock, IndexerInterface } from './types';
|
||||||
import { wait } from './misc';
|
import { wait } from './misc';
|
||||||
import { processBlockByNumber } from './common';
|
import { processBlockByNumber } from './common';
|
||||||
import { DEFAULT_PREFETCH_BATCH_SIZE } from './constants';
|
import { DEFAULT_PREFETCH_BATCH_SIZE } from './constants';
|
||||||
@ -107,7 +107,7 @@ const prefetchBlocks = async (
|
|||||||
let blockNumbers = [...Array(batchEndBlock - i).keys()].map(n => n + i);
|
let blockNumbers = [...Array(batchEndBlock - i).keys()].map(n => n + i);
|
||||||
log('Fetching blockNumbers:', blockNumbers);
|
log('Fetching blockNumbers:', blockNumbers);
|
||||||
|
|
||||||
let blocks = [];
|
let blocks: EthFullBlock[] = [];
|
||||||
|
|
||||||
// Fetch blocks again if there are missing blocks.
|
// Fetch blocks again if there are missing blocks.
|
||||||
while (true) {
|
while (true) {
|
||||||
@ -117,7 +117,8 @@ const prefetchBlocks = async (
|
|||||||
const missingIndex = res.findIndex(blocks => blocks.length === 0);
|
const missingIndex = res.findIndex(blocks => blocks.length === 0);
|
||||||
|
|
||||||
if (missingIndex < 0) {
|
if (missingIndex < 0) {
|
||||||
blocks = res.flat();
|
// Filter null blocks
|
||||||
|
blocks = res.flat().filter(block => Boolean(block)) as EthFullBlock[];
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,12 +22,14 @@ export const indexBlock = async (
|
|||||||
console.time('time:index-block#getBlocks-ipld-eth-server');
|
console.time('time:index-block#getBlocks-ipld-eth-server');
|
||||||
const blocks = await indexer.getBlocks({ blockNumber: argv.block });
|
const blocks = await indexer.getBlocks({ blockNumber: argv.block });
|
||||||
|
|
||||||
blockProgressEntities = blocks.map((block: any): Partial<BlockProgressInterface> => {
|
// Filter null blocks and transform to BlockProgress type
|
||||||
block.blockTimestamp = Number(block.timestamp);
|
blockProgressEntities = blocks.filter(block => Boolean(block))
|
||||||
block.blockNumber = Number(block.blockNumber);
|
.map((block: any): Partial<BlockProgressInterface> => {
|
||||||
|
block.blockTimestamp = Number(block.timestamp);
|
||||||
|
block.blockNumber = Number(block.blockNumber);
|
||||||
|
|
||||||
return block;
|
return block;
|
||||||
});
|
});
|
||||||
|
|
||||||
console.timeEnd('time:index-block#getBlocks-ipld-eth-server');
|
console.timeEnd('time:index-block#getBlocks-ipld-eth-server');
|
||||||
}
|
}
|
||||||
|
@ -291,7 +291,7 @@ export class Indexer {
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<EthFullBlock[]> {
|
async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
|
||||||
assert(blockFilter.blockHash || blockFilter.blockNumber);
|
assert(blockFilter.blockHash || blockFilter.blockNumber);
|
||||||
const blocks = await this._ethClient.getFullBlocks(blockFilter);
|
const blocks = await this._ethClient.getFullBlocks(blockFilter);
|
||||||
|
|
||||||
@ -394,6 +394,7 @@ export class Indexer {
|
|||||||
console.time(`time:indexer#fetchAndSaveFilteredEventsAndBlocks-fetch-blocks-txs-${fromBlock}-${toBlock}`);
|
console.time(`time:indexer#fetchAndSaveFilteredEventsAndBlocks-fetch-blocks-txs-${fromBlock}-${toBlock}`);
|
||||||
const blocksPromises = Array.from(blockLogsMap.keys()).map(async (blockHash) => {
|
const blocksPromises = Array.from(blockLogsMap.keys()).map(async (blockHash) => {
|
||||||
const [fullBlock] = await this._ethClient.getFullBlocks({ blockHash });
|
const [fullBlock] = await this._ethClient.getFullBlocks({ blockHash });
|
||||||
|
assert(fullBlock);
|
||||||
|
|
||||||
const block = {
|
const block = {
|
||||||
...fullBlock,
|
...fullBlock,
|
||||||
@ -401,7 +402,10 @@ export class Indexer {
|
|||||||
blockNumber: Number(fullBlock.blockNumber)
|
blockNumber: Number(fullBlock.blockNumber)
|
||||||
};
|
};
|
||||||
|
|
||||||
return { block, fullBlock } as { block: DeepPartial<BlockProgressInterface>; fullBlock: EthFullBlock };
|
return {
|
||||||
|
block: block as DeepPartial<BlockProgressInterface>,
|
||||||
|
fullBlock
|
||||||
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
const ethFullTxPromises = txHashes.map(async txHash => {
|
const ethFullTxPromises = txHashes.map(async txHash => {
|
||||||
|
@ -521,30 +521,28 @@ export class JobRunner {
|
|||||||
const newPriority = (priority || 0) + 1;
|
const newPriority = (priority || 0) + 1;
|
||||||
|
|
||||||
if (!parentBlock || parentBlock.blockHash !== parentHash) {
|
if (!parentBlock || parentBlock.blockHash !== parentHash) {
|
||||||
const blocks = await this._indexer.getBlocks({ blockHash: parentHash });
|
const [block] = await this._indexer.getBlocks({ blockHash: parentHash });
|
||||||
|
|
||||||
if (!blocks.length) {
|
if (!block) {
|
||||||
const message = `No blocks at parentHash ${parentHash}, aborting`;
|
const message = `No blocks at parentHash ${parentHash}, aborting`;
|
||||||
log(message);
|
log(message);
|
||||||
|
|
||||||
throw new Error(message);
|
throw new Error(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
blocks.forEach(block => {
|
this._blockAndEventsMap.set(
|
||||||
this._blockAndEventsMap.set(
|
block.blockHash,
|
||||||
block.blockHash,
|
{
|
||||||
{
|
// block is set later in job when saving to database
|
||||||
// block is set later in job when saving to database
|
block: {} as BlockProgressInterface,
|
||||||
block: {} as BlockProgressInterface,
|
events: [],
|
||||||
events: [],
|
ethFullBlock: block,
|
||||||
ethFullBlock: block,
|
// Transactions are set later in job when fetching events
|
||||||
// Transactions are set later in job when fetching events
|
ethFullTransactions: []
|
||||||
ethFullTransactions: []
|
}
|
||||||
}
|
);
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
const [{ cid: parentCid, blockNumber: parentBlockNumber, parentHash: grandparentHash, timestamp: parentTimestamp }] = blocks;
|
const { cid: parentCid, blockNumber: parentBlockNumber, parentHash: grandparentHash, timestamp: parentTimestamp } = block;
|
||||||
|
|
||||||
await this.jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, {
|
await this.jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, {
|
||||||
kind: JOB_KIND_INDEX,
|
kind: JOB_KIND_INDEX,
|
||||||
|
@ -138,7 +138,7 @@ export interface EthClient {
|
|||||||
getFullBlocks({ blockNumber, blockHash }: {
|
getFullBlocks({ blockNumber, blockHash }: {
|
||||||
blockNumber?: number;
|
blockNumber?: number;
|
||||||
blockHash?: string;
|
blockHash?: string;
|
||||||
}): Promise<EthFullBlock[]>;
|
}): Promise<Array<EthFullBlock | null>>;
|
||||||
getFullTransaction(txHash: string, blockNumber?: number): Promise<EthFullTransaction>;
|
getFullTransaction(txHash: string, blockNumber?: number): Promise<EthFullTransaction>;
|
||||||
getBlockByHash(blockHash?: string): Promise<any>;
|
getBlockByHash(blockHash?: string): Promise<any>;
|
||||||
getLogs(vars: {
|
getLogs(vars: {
|
||||||
@ -167,7 +167,7 @@ export interface IndexerInterface {
|
|||||||
getEvent (id: string): Promise<EventInterface | undefined>
|
getEvent (id: string): Promise<EventInterface | undefined>
|
||||||
getSyncStatus (): Promise<SyncStatusInterface | undefined>
|
getSyncStatus (): Promise<SyncStatusInterface | undefined>
|
||||||
getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
|
getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
|
||||||
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<EthFullBlock[]>
|
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<Array<EthFullBlock | null>>
|
||||||
getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]>
|
getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]>
|
||||||
getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined>
|
getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined>
|
||||||
getLatestStateIndexedBlock (): Promise<BlockProgressInterface>
|
getLatestStateIndexedBlock (): Promise<BlockProgressInterface>
|
||||||
|
Loading…
Reference in New Issue
Block a user