cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2025-01-06 19:38:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix job-runner to process missing parent block (#274)

Browse Source 
* Fix job-runner to process missing parent block

* Upgrade package versions to 0.2.18
This commit is contained in:
nikugogoi 2022-12-13 11:59:55 +05:30 committed by GitHub
parent cc66f44bab
commit 9566dff466
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 95 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "0.2.17",
"version": "0.2.18",
"npmClient": "yarn",
"useWorkspaces": true,
"command": {

12
packages/address-watcher/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/address-watcher",
"version": "0.2.17",
"version": "0.2.18",
"description": "Address Watcher",
"private": true,
"scripts": {
@ -26,11 +26,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cache": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/tracing-client": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cache": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/tracing-client": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@types/lodash": "^4.14.168",
"debug": "^4.3.1",
"ethers": "^5.4.4",

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cache",
"version": "0.2.17",
"version": "0.2.18",
"description": "Generic object cache",
"main": "dist/index.js",
"scripts": {

4
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cli",
"version": "0.2.17",
"version": "0.2.18",
"main": "dist/index.js",
"license": "AGPL-3.0",
"scripts": {
@ -10,7 +10,7 @@
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/"
},
"dependencies": {
"@cerc-io/util": "^0.2.17",
"@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^6.0.12",

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/codegen",
"version": "0.2.17",
"version": "0.2.18",
"description": "Code generator",
"private": true,
"main": "index.js",
@ -20,7 +20,7 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/util": "^0.2.17",
"@cerc-io/util": "^0.2.18",
"@graphql-tools/load-files": "^6.5.2",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
"@solidity-parser/parser": "^0.13.2",

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"dependencies": {
"@apollo/client": "^3.3.19",
"@ethersproject/providers": "^5.4.4",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
{{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.17",
"@cerc-io/graph-node": "^0.2.18",
{{/if}}
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",

12
packages/eden-watcher/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/eden-watcher",
"version": "0.2.17",
"version": "0.2.18",
"description": "eden-watcher",
"private": true,
"main": "dist/index.js",
@ -38,11 +38,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/graph-node": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/graph-node": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",

10
packages/erc20-watcher/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/erc20-watcher",
"version": "0.2.17",
"version": "0.2.18",
"description": "ERC20 Watcher",
"private": true,
"main": "dist/index.js",
@ -40,10 +40,10 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",

10
packages/erc721-watcher/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/erc721-watcher",
"version": "0.2.17",
"version": "0.2.18",
"description": "erc721-watcher",
"private": true,
"main": "dist/index.js",
@ -46,10 +46,10 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",

12
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/graph-node",
"version": "0.2.17",
"version": "0.2.18",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
@ -49,11 +49,11 @@
},
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cache": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@types/json-diff": "^0.5.2",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@types/json-diff": "^0.5.2",
"bn.js": "^4.11.9",
"debug": "^4.3.1",
"fs-extra": "^10.0.0",

12
packages/graph-test-watcher/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/graph-test-watcher",
"version": "0.2.17",
"version": "0.2.18",
"description": "graph-test-watcher",
"private": true,
"main": "dist/index.js",
@ -38,11 +38,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/graph-node": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/graph-node": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",

4
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/ipld-eth-client",
"version": "0.2.17",
"version": "0.2.18",
"description": "IPLD ETH Client",
"main": "dist/index.js",
"scripts": {
@ -20,7 +20,7 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.17",
"@cerc-io/cache": "^0.2.18",
"cross-fetch": "^3.1.4",
"debug": "^4.3.1",
"ethers": "^5.4.4",

10
packages/mobymask-watcher/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/mobymask-watcher",
"version": "0.2.17",
"version": "0.2.18",
"description": "mobymask-watcher",
"private": true,
"main": "dist/index.js",
@ -37,10 +37,10 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",

4
packages/solidity-mapper/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/solidity-mapper",
"version": "0.2.17",
"version": "0.2.18",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@ethersproject/abi": "^5.3.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
"@nomiclabs/hardhat-waffle": "^2.0.1",

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/test",
"version": "0.2.17",
"version": "0.2.18",
"main": "dist/index.js",
"license": "AGPL-3.0",
"private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/tracing-client",
"version": "0.2.17",
"version": "0.2.18",
"description": "ETH VM tracing client",
"private": true,
"main": "dist/index.js",

8
packages/util/package.json
View File

@ -1,11 +1,11 @@
{
"name": "@cerc-io/util",
"version": "0.2.17",
"version": "0.2.18",
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.18",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/schema": "^9.0.10",
"@graphql-tools/utils": "^9.1.1",
@ -37,8 +37,8 @@
"yargs": "^17.0.1"
},
"devDependencies": {
"@cerc-io/cache": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/cache": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/express": "^4.17.14",
"@types/fs-extra": "^9.0.11",

3
packages/util/src/common.ts
View File

@ -55,12 +55,11 @@ export const processBlockByNumber = async (
* @param blockAndEventsMap
*/
export const fetchBlocksAtHeight = async (
job: any,
blockNumber: number,
indexer: IndexerInterface,
jobQueueConfig: JobQueueConfig,
blockAndEventsMap: Map<string, PrefetchedBlock>
): Promise<DeepPartial<BlockProgressInterface>[]> => {
const { blockNumber } = job.data;
let blocks = [];
// Check for blocks in cache if prefetchBlocksInMem flag set.

47
packages/util/src/job-runner.ts
View File

@ -4,7 +4,7 @@
import assert from 'assert';
import debug from 'debug';
import { In } from 'typeorm';
import { DeepPartial, In } from 'typeorm';
import { JobQueueConfig } from './config';
import {
@ -19,7 +19,7 @@ import {
QUEUE_HOOKS
} from './constants';
import { JobQueue } from './job-queue';
import { EventInterface, IndexerInterface } from './types';
import { BlockProgressInterface, EventInterface, IndexerInterface } from './types';
import { wait } from './misc';
import {
createPruningJob,
@ -79,14 +79,30 @@ export class JobRunner {
switch (kind) {
case JOB_KIND_INDEX: {
const blocksToBeIndexed = await fetchBlocksAtHeight(
job,
this._indexer,
this._jobQueueConfig,
this._blockAndEventsMap
);
const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed));
await Promise.all(indexBlockPromises);
const { data: { cid, blockHash, blockNumber, parentHash, timestamp } } = job;
// Check if blockHash present in job.
if (blockHash) {
// If blockHash is present it is a job for indexing missing parent block.
await this._indexBlock(job, {
blockTimestamp: timestamp,
cid,
blockHash,
blockNumber,
parentHash
});
} else {
// If blockHash is not present, it is a job to index the next consecutive blockNumber.
const blocksToBeIndexed = await fetchBlocksAtHeight(
blockNumber,
this._indexer,
this._jobQueueConfig,
this._blockAndEventsMap
);
const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed));
await Promise.all(indexBlockPromises);
}
break;
}
@ -277,12 +293,15 @@ export class JobRunner {
console.timeEnd('time:job-runner#_pruneChain');
}
async _indexBlock (job: any, blockToBeIndexed: any): Promise<void> {
async _indexBlock (job: any, blockToBeIndexed: DeepPartial<BlockProgressInterface>): Promise<void> {
const syncStatus = await this._indexer.getSyncStatus();
assert(syncStatus);
const { data: { priority } } = job;
const { cid, blockHash, blockNumber, parentHash, blockTimestamp } = blockToBeIndexed;
assert(blockNumber);
assert(blockHash);
assert(parentHash);
const indexBlockStartTime = new Date();
@ -350,7 +369,8 @@ export class JobRunner {
const message = `Parent block number ${parentBlockNumber} hash ${parentHash} of block number ${blockNumber} hash ${blockHash} not fetched yet, aborting`;
log(message);
throw new Error(message);
// Do not throw error and complete the job as block will be processed after parent block processing.
return;
}
if (!parentBlock.isComplete) {
@ -368,7 +388,8 @@ export class JobRunner {
priority: newPriority
}, { priority: newPriority });
throw new Error(message);
// Do not throw error and complete the job as block will be processed after parent block processing.
return;
} else {
// Remove the unknown events of the parent block if it is marked complete.
console.time('time:job-runner#_indexBlock-remove-unknown-events');