Handle null block error in rpc-eth-client (#477)

* Remove block-size-cache util methods as full block with size fetched already

* Handle null block error in rpc-eth-client

* Upgrade package versions
This commit is contained in:
Nabarun Gogoi 2023-11-17 11:58:43 +05:30 committed by GitHub
parent e54f4c9276
commit c2070a80cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 119 additions and 192 deletions

View File

@ -2,7 +2,7 @@
"packages": [ "packages": [
"packages/*" "packages/*"
], ],
"version": "0.2.72", "version": "0.2.73",
"npmClient": "yarn", "npmClient": "yarn",
"useWorkspaces": true, "useWorkspaces": true,
"command": { "command": {

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/cache", "name": "@cerc-io/cache",
"version": "0.2.72", "version": "0.2.73",
"description": "Generic object cache", "description": "Generic object cache",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/cli", "name": "@cerc-io/cli",
"version": "0.2.72", "version": "0.2.73",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"scripts": { "scripts": {
@ -12,13 +12,13 @@
}, },
"dependencies": { "dependencies": {
"@apollo/client": "^3.7.1", "@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.72", "@cerc-io/cache": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.72", "@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4", "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15", "@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.72", "@cerc-io/peer": "^0.2.73",
"@cerc-io/rpc-eth-client": "^0.2.72", "@cerc-io/rpc-eth-client": "^0.2.73",
"@cerc-io/util": "^0.2.72", "@cerc-io/util": "^0.2.73",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1", "@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0", "@ipld/dag-cbor": "^8.0.0",

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/codegen", "name": "@cerc-io/codegen",
"version": "0.2.72", "version": "0.2.73",
"description": "Code generator", "description": "Code generator",
"private": true, "private": true,
"main": "index.js", "main": "index.js",
@ -20,7 +20,7 @@
}, },
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@cerc-io/util": "^0.2.72", "@cerc-io/util": "^0.2.73",
"@graphql-tools/load-files": "^6.5.2", "@graphql-tools/load-files": "^6.5.2",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
"@solidity-parser/parser": "^0.13.2", "@solidity-parser/parser": "^0.13.2",

View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.72", "@cerc-io/cli": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.72", "@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/solidity-mapper": "^0.2.72", "@cerc-io/solidity-mapper": "^0.2.73",
"@cerc-io/util": "^0.2.72", "@cerc-io/util": "^0.2.73",
{{#if (subgraphPath)}} {{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.72", "@cerc-io/graph-node": "^0.2.73",
{{/if}} {{/if}}
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1", "debug": "^4.3.1",

View File

@ -1,10 +1,10 @@
{ {
"name": "@cerc-io/graph-node", "name": "@cerc-io/graph-node",
"version": "0.2.72", "version": "0.2.73",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"devDependencies": { "devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.72", "@cerc-io/solidity-mapper": "^0.2.73",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0", "@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2", "@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.72", "@cerc-io/cache": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.72", "@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/util": "^0.2.72", "@cerc-io/util": "^0.2.73",
"@types/json-diff": "^0.5.2", "@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0", "@types/yargs": "^17.0.0",
"bn.js": "^4.11.9", "bn.js": "^4.11.9",

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/ipld-eth-client", "name": "@cerc-io/ipld-eth-client",
"version": "0.2.72", "version": "0.2.73",
"description": "IPLD ETH Client", "description": "IPLD ETH Client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.7.1", "@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.72", "@cerc-io/cache": "^0.2.73",
"@cerc-io/util": "^0.2.72", "@cerc-io/util": "^0.2.73",
"cross-fetch": "^3.1.4", "cross-fetch": "^3.1.4",
"debug": "^4.3.1", "debug": "^4.3.1",
"ethers": "^5.4.4", "ethers": "^5.4.4",

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/peer", "name": "@cerc-io/peer",
"version": "0.2.72", "version": "0.2.73",
"description": "libp2p module", "description": "libp2p module",
"main": "dist/index.js", "main": "dist/index.js",
"exports": "./dist/index.js", "exports": "./dist/index.js",

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/rpc-eth-client", "name": "@cerc-io/rpc-eth-client",
"version": "0.2.72", "version": "0.2.73",
"description": "RPC ETH Client", "description": "RPC ETH Client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {
@ -19,9 +19,9 @@
}, },
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@cerc-io/cache": "^0.2.72", "@cerc-io/cache": "^0.2.73",
"@cerc-io/ipld-eth-client": "^0.2.72", "@cerc-io/ipld-eth-client": "^0.2.73",
"@cerc-io/util": "^0.2.72", "@cerc-io/util": "^0.2.73",
"chai": "^4.3.4", "chai": "^4.3.4",
"ethers": "^5.4.4", "ethers": "^5.4.4",
"left-pad": "^1.3.0", "left-pad": "^1.3.0",

View File

@ -10,6 +10,7 @@ import { encodeHeader, escapeHexString, EthClient as EthClientInterface, EthFull
import { padKey } from '@cerc-io/ipld-eth-client'; import { padKey } from '@cerc-io/ipld-eth-client';
const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')"; const FUTURE_BLOCK_ERROR = "requested a future epoch (beyond 'latest')";
const NULL_BLOCK_ERROR = 'requested epoch was a null round';
export interface Config { export interface Config {
cache: Cache | undefined; cache: Cache | undefined;
@ -93,10 +94,7 @@ export class EthClient implements EthClientInterface {
} }
]; ];
} catch (err: any) { } catch (err: any) {
// Check and ignore future block error return this._handleGetBlockErrors(err);
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
throw err;
}
} finally { } finally {
console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`); console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
} }
@ -138,10 +136,7 @@ export class EthClient implements EthClientInterface {
]; ];
} }
} catch (err: any) { } catch (err: any) {
// Check and ignore future block error return this._handleGetBlockErrors(err);
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
throw err;
}
} finally { } finally {
console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`); console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
} }
@ -153,7 +148,7 @@ export class EthClient implements EthClientInterface {
}; };
} }
async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<EthFullBlock[]> { async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
const blockNumberHex = blockNumber ? utils.hexValue(blockNumber) : undefined; const blockNumberHex = blockNumber ? utils.hexValue(blockNumber) : undefined;
const blockHashOrBlockNumber = blockHash ?? blockNumberHex; const blockHashOrBlockNumber = blockHash ?? blockNumberHex;
assert(blockHashOrBlockNumber); assert(blockHashOrBlockNumber);
@ -207,10 +202,7 @@ export class EthClient implements EthClientInterface {
}]; }];
} }
} catch (err: any) { } catch (err: any) {
// Check and ignore future block error return this._handleGetBlockErrors(err);
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === FUTURE_BLOCK_ERROR)) {
throw err;
}
} finally { } finally {
console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`); console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
} }
@ -379,4 +371,20 @@ export class EthClient implements EthClientInterface {
return result; return result;
} }
_handleGetBlockErrors (err: any): Array<null> {
if (err.code === errors.SERVER_ERROR && err.error) {
// Check null block error and return null array
if (err.error.message === NULL_BLOCK_ERROR) {
return [null];
}
// Check and ignore future block error
if (err.error.message === FUTURE_BLOCK_ERROR) {
return [];
}
}
throw err;
}
} }

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/solidity-mapper", "name": "@cerc-io/solidity-mapper",
"version": "0.2.72", "version": "0.2.73",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"devDependencies": { "devDependencies": {

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/test", "name": "@cerc-io/test",
"version": "0.2.72", "version": "0.2.73",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"private": true, "private": true,

View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/tracing-client", "name": "@cerc-io/tracing-client",
"version": "0.2.72", "version": "0.2.73",
"description": "ETH VM tracing client", "description": "ETH VM tracing client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {

View File

@ -1,13 +1,13 @@
{ {
"name": "@cerc-io/util", "name": "@cerc-io/util",
"version": "0.2.72", "version": "0.2.73",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"dependencies": { "dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1", "@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15", "@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.72", "@cerc-io/peer": "^0.2.73",
"@cerc-io/solidity-mapper": "^0.2.72", "@cerc-io/solidity-mapper": "^0.2.73",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1", "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0", "@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
@ -52,7 +52,7 @@
"yargs": "^17.0.1" "yargs": "^17.0.1"
}, },
"devDependencies": { "devDependencies": {
"@cerc-io/cache": "^0.2.72", "@cerc-io/cache": "^0.2.73",
"@nomiclabs/hardhat-waffle": "^2.0.1", "@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8", "@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14", "@types/express": "^4.17.14",

View File

@ -1,81 +0,0 @@
//
// Copyright 2022 Vulcanize, Inc.
//
import { utils, providers, errors } from 'ethers';
import debug from 'debug';
import { NULL_BLOCK_ERROR } from './constants';
const log = debug('vulcanize:block-size-cache');
// Number of blocks to cache after current block being processed.
const BLOCK_SIZE_CACHE_BUFFER = 10;
// Block height interval at which blockSizeMap is cleared.
// If the block being processed is divisible by BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL then blocks below that height are removed from the map.
const BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL = 50;
const blockSizeMap: Map<string, { size: string, blockNumber: number }> = new Map();
let blockSizeMapLatestHeight = -1;
export const getCachedBlockSize = async (provider: providers.JsonRpcProvider, blockHash: string, blockNumber: number): Promise<string> => {
const block = blockSizeMap.get(blockHash);
cacheBlockSizesAsync(provider, blockNumber);
if (!block) {
console.time(`time:misc#getCachedBlockSize-eth_getBlockByHash-${blockNumber}`);
const { size } = await provider.send('eth_getBlockByHash', [blockHash, false]);
console.timeEnd(`time:misc#getCachedBlockSize-eth_getBlockByHash-${blockNumber}`);
return size;
}
return block.size;
};
const cacheBlockSizesAsync = async (provider: providers.JsonRpcProvider, blockNumber: number): Promise<void> => {
const endBlockHeight = blockNumber + BLOCK_SIZE_CACHE_BUFFER;
if (blockSizeMapLatestHeight < 0) {
blockSizeMapLatestHeight = blockNumber;
}
if (endBlockHeight > blockSizeMapLatestHeight) {
const startBlockHeight = Math.max(blockNumber, blockSizeMapLatestHeight + 1);
blockSizeMapLatestHeight = endBlockHeight;
// Start prefetching blocks after latest height in blockSizeMap.
for (let i = startBlockHeight; i <= endBlockHeight; i++) {
try {
console.time(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
const block = await provider.send('eth_getBlockByNumber', [utils.hexStripZeros(utils.hexlify(i)), false]);
if (block) {
const { size, hash } = block;
blockSizeMap.set(hash, { size, blockNumber: i });
} else {
log(`No block found at height ${i}`);
}
} catch (err: any) {
// Handle null block error in case of Lotus EVM
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === NULL_BLOCK_ERROR)) {
throw err;
}
log(`Block ${i} requested was null (FEVM); Fetching next block`);
} finally {
console.timeEnd(`time:misc#cacheBlockSizesAsync-eth_getBlockByNumber-${i}`);
}
}
}
// At interval clear previous blocks below height blockNumber from map.
if (blockNumber % BLOCK_SIZE_MAP_CLEAR_HEIGHT_INTERVAL === 0) {
log(`cacheBlockSizesAsync-clear-map-below-${blockNumber}`);
const previousBlockHashes = Array.from(blockSizeMap.entries())
.filter(([, value]) => value.blockNumber <= blockNumber)
.map(([blockHash]) => blockHash);
previousBlockHashes.forEach(blockHash => blockSizeMap.delete(blockHash));
}
};

View File

@ -1,7 +1,6 @@
import debug from 'debug'; import debug from 'debug';
import assert from 'assert'; import assert from 'assert';
import { DeepPartial } from 'typeorm'; import { DeepPartial } from 'typeorm';
import { errors } from 'ethers';
import JSONbig from 'json-bigint'; import JSONbig from 'json-bigint';
import { import {
@ -10,8 +9,7 @@ import {
QUEUE_BLOCK_CHECKPOINT, QUEUE_BLOCK_CHECKPOINT,
JOB_KIND_PRUNE, JOB_KIND_PRUNE,
JOB_KIND_INDEX, JOB_KIND_INDEX,
UNKNOWN_EVENT_NAME, UNKNOWN_EVENT_NAME
NULL_BLOCK_ERROR
} from './constants'; } from './constants';
import { JobQueue } from './job-queue'; import { JobQueue } from './job-queue';
import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock } from './types'; import { BlockProgressInterface, IndexerInterface, EventInterface, EthFullTransaction, EthFullBlock } from './types';
@ -78,38 +76,37 @@ export const fetchBlocksAtHeight = async (
// Try fetching blocks from eth-server until found. // Try fetching blocks from eth-server until found.
while (!blocks.length) { while (!blocks.length) {
try { console.time(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
console.time('time:common#_fetchBlocks-eth-server'); const ethFullBlocks = await indexer.getBlocks({ blockNumber });
blocks = await indexer.getBlocks({ blockNumber }); console.timeEnd(`time:common#_fetchBlocks-eth-server-${blockNumber}`);
if (!blocks.length) { // Check if all blocks are null and increment blockNumber to index next block number
log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`); if (ethFullBlocks.every(block => block === null)) {
await wait(jobQueueConfig.blockDelayInMilliSecs);
} else {
blocks.forEach(block => {
blockAndEventsMap.set(
block.blockHash,
{
// Block is set later in job-runner when saving to database
block: {} as BlockProgressInterface,
events: [],
ethFullBlock: block,
// Transactions are set later in job-runner when fetching events
ethFullTransactions: []
}
);
});
}
} catch (err: any) {
// Handle null block error in case of Lotus EVM
if (!(err.code === errors.SERVER_ERROR && err.error && err.error.message === NULL_BLOCK_ERROR)) {
throw err;
}
log(`Block ${blockNumber} requested was null (FEVM); Fetching next block`);
blockNumber++; blockNumber++;
} finally { log(`Block ${blockNumber} requested was null (FEVM); Fetching next block`);
console.timeEnd('time:common#_fetchBlocks-eth-server'); continue;
}
// Fitler null blocks
blocks = ethFullBlocks.filter(block => Boolean(block)) as EthFullBlock[];
if (!blocks.length) {
log(`No blocks fetched for block number ${blockNumber}, retrying after ${jobQueueConfig.blockDelayInMilliSecs} ms delay.`);
await wait(jobQueueConfig.blockDelayInMilliSecs);
} else {
blocks.forEach(block => {
blockAndEventsMap.set(
block.blockHash,
{
// Block is set later in job-runner when saving to database
block: {} as BlockProgressInterface,
events: [],
ethFullBlock: block,
// Transactions are set later in job-runner when fetching events
ethFullTransactions: []
}
);
});
} }
} }

View File

@ -30,5 +30,3 @@ export const DEFAULT_PREFETCH_BATCH_SIZE = 10;
export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB
export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber']; export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber'];
export const NULL_BLOCK_ERROR = 'requested epoch was a null round';

View File

@ -5,7 +5,7 @@
import debug from 'debug'; import debug from 'debug';
import { JobQueue } from './job-queue'; import { JobQueue } from './job-queue';
import { IndexerInterface } from './types'; import { EthFullBlock, IndexerInterface } from './types';
import { wait } from './misc'; import { wait } from './misc';
import { processBlockByNumber } from './common'; import { processBlockByNumber } from './common';
import { DEFAULT_PREFETCH_BATCH_SIZE } from './constants'; import { DEFAULT_PREFETCH_BATCH_SIZE } from './constants';
@ -107,7 +107,7 @@ const prefetchBlocks = async (
let blockNumbers = [...Array(batchEndBlock - i).keys()].map(n => n + i); let blockNumbers = [...Array(batchEndBlock - i).keys()].map(n => n + i);
log('Fetching blockNumbers:', blockNumbers); log('Fetching blockNumbers:', blockNumbers);
let blocks = []; let blocks: EthFullBlock[] = [];
// Fetch blocks again if there are missing blocks. // Fetch blocks again if there are missing blocks.
while (true) { while (true) {
@ -117,7 +117,8 @@ const prefetchBlocks = async (
const missingIndex = res.findIndex(blocks => blocks.length === 0); const missingIndex = res.findIndex(blocks => blocks.length === 0);
if (missingIndex < 0) { if (missingIndex < 0) {
blocks = res.flat(); // Filter null blocks
blocks = res.flat().filter(block => Boolean(block)) as EthFullBlock[];
break; break;
} }

View File

@ -22,12 +22,14 @@ export const indexBlock = async (
console.time('time:index-block#getBlocks-ipld-eth-server'); console.time('time:index-block#getBlocks-ipld-eth-server');
const blocks = await indexer.getBlocks({ blockNumber: argv.block }); const blocks = await indexer.getBlocks({ blockNumber: argv.block });
blockProgressEntities = blocks.map((block: any): Partial<BlockProgressInterface> => { // Filter null blocks and transform to BlockProgress type
block.blockTimestamp = Number(block.timestamp); blockProgressEntities = blocks.filter(block => Boolean(block))
block.blockNumber = Number(block.blockNumber); .map((block: any): Partial<BlockProgressInterface> => {
block.blockTimestamp = Number(block.timestamp);
block.blockNumber = Number(block.blockNumber);
return block; return block;
}); });
console.timeEnd('time:index-block#getBlocks-ipld-eth-server'); console.timeEnd('time:index-block#getBlocks-ipld-eth-server');
} }

View File

@ -291,7 +291,7 @@ export class Indexer {
return res; return res;
} }
async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<EthFullBlock[]> { async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
assert(blockFilter.blockHash || blockFilter.blockNumber); assert(blockFilter.blockHash || blockFilter.blockNumber);
const blocks = await this._ethClient.getFullBlocks(blockFilter); const blocks = await this._ethClient.getFullBlocks(blockFilter);
@ -394,6 +394,7 @@ export class Indexer {
console.time(`time:indexer#fetchAndSaveFilteredEventsAndBlocks-fetch-blocks-txs-${fromBlock}-${toBlock}`); console.time(`time:indexer#fetchAndSaveFilteredEventsAndBlocks-fetch-blocks-txs-${fromBlock}-${toBlock}`);
const blocksPromises = Array.from(blockLogsMap.keys()).map(async (blockHash) => { const blocksPromises = Array.from(blockLogsMap.keys()).map(async (blockHash) => {
const [fullBlock] = await this._ethClient.getFullBlocks({ blockHash }); const [fullBlock] = await this._ethClient.getFullBlocks({ blockHash });
assert(fullBlock);
const block = { const block = {
...fullBlock, ...fullBlock,
@ -401,7 +402,10 @@ export class Indexer {
blockNumber: Number(fullBlock.blockNumber) blockNumber: Number(fullBlock.blockNumber)
}; };
return { block, fullBlock } as { block: DeepPartial<BlockProgressInterface>; fullBlock: EthFullBlock }; return {
block: block as DeepPartial<BlockProgressInterface>,
fullBlock
};
}); });
const ethFullTxPromises = txHashes.map(async txHash => { const ethFullTxPromises = txHashes.map(async txHash => {

View File

@ -521,30 +521,28 @@ export class JobRunner {
const newPriority = (priority || 0) + 1; const newPriority = (priority || 0) + 1;
if (!parentBlock || parentBlock.blockHash !== parentHash) { if (!parentBlock || parentBlock.blockHash !== parentHash) {
const blocks = await this._indexer.getBlocks({ blockHash: parentHash }); const [block] = await this._indexer.getBlocks({ blockHash: parentHash });
if (!blocks.length) { if (!block) {
const message = `No blocks at parentHash ${parentHash}, aborting`; const message = `No blocks at parentHash ${parentHash}, aborting`;
log(message); log(message);
throw new Error(message); throw new Error(message);
} }
blocks.forEach(block => { this._blockAndEventsMap.set(
this._blockAndEventsMap.set( block.blockHash,
block.blockHash, {
{ // block is set later in job when saving to database
// block is set later in job when saving to database block: {} as BlockProgressInterface,
block: {} as BlockProgressInterface, events: [],
events: [], ethFullBlock: block,
ethFullBlock: block, // Transactions are set later in job when fetching events
// Transactions are set later in job when fetching events ethFullTransactions: []
ethFullTransactions: [] }
} );
);
});
const [{ cid: parentCid, blockNumber: parentBlockNumber, parentHash: grandparentHash, timestamp: parentTimestamp }] = blocks; const { cid: parentCid, blockNumber: parentBlockNumber, parentHash: grandparentHash, timestamp: parentTimestamp } = block;
await this.jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, { await this.jobQueue.pushJob(QUEUE_BLOCK_PROCESSING, {
kind: JOB_KIND_INDEX, kind: JOB_KIND_INDEX,

View File

@ -138,7 +138,7 @@ export interface EthClient {
getFullBlocks({ blockNumber, blockHash }: { getFullBlocks({ blockNumber, blockHash }: {
blockNumber?: number; blockNumber?: number;
blockHash?: string; blockHash?: string;
}): Promise<EthFullBlock[]>; }): Promise<Array<EthFullBlock | null>>;
getFullTransaction(txHash: string, blockNumber?: number): Promise<EthFullTransaction>; getFullTransaction(txHash: string, blockNumber?: number): Promise<EthFullTransaction>;
getBlockByHash(blockHash?: string): Promise<any>; getBlockByHash(blockHash?: string): Promise<any>;
getLogs(vars: { getLogs(vars: {
@ -167,7 +167,7 @@ export interface IndexerInterface {
getEvent (id: string): Promise<EventInterface | undefined> getEvent (id: string): Promise<EventInterface | undefined>
getSyncStatus (): Promise<SyncStatusInterface | undefined> getSyncStatus (): Promise<SyncStatusInterface | undefined>
getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined> getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<EthFullBlock[]> getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<Array<EthFullBlock | null>>
getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]> getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgressInterface[]>
getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined> getLatestCanonicalBlock (): Promise<BlockProgressInterface | undefined>
getLatestStateIndexedBlock (): Promise<BlockProgressInterface> getLatestStateIndexedBlock (): Promise<BlockProgressInterface>