cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2025-01-07 20:08:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix job-runner to process missing parent block (#274)

Browse Source 
* Fix job-runner to process missing parent block

* Upgrade package versions to 0.2.18
This commit is contained in:
nikugogoi 2022-12-13 11:59:55 +05:30 committed by GitHub
parent cc66f44bab
commit 9566dff466
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 95 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [ "packages": [
"packages/*" "packages/*"
], ],
"version": "0.2.17", "version": "0.2.18",
"npmClient": "yarn", "npmClient": "yarn",
"useWorkspaces": true, "useWorkspaces": true,
"command": { "command": {

12
packages/address-watcher/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/address-watcher", "name": "@cerc-io/address-watcher",
"version": "0.2.17", "version": "0.2.18",
"description": "Address Watcher", "description": "Address Watcher",
"private": true, "private": true,
"scripts": { "scripts": {
@ -26,11 +26,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cache": "^0.2.17", "@cerc-io/cache": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/tracing-client": "^0.2.17", "@cerc-io/tracing-client": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@types/lodash": "^4.14.168", "@types/lodash": "^4.14.168",
"debug": "^4.3.1", "debug": "^4.3.1",
"ethers": "^5.4.4", "ethers": "^5.4.4",

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/cache", "name": "@cerc-io/cache",
"version": "0.2.17", "version": "0.2.18",
"description": "Generic object cache", "description": "Generic object cache",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {

4
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/cli", "name": "@cerc-io/cli",
"version": "0.2.17", "version": "0.2.18",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"scripts": { "scripts": {
@ -10,7 +10,7 @@
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/" "copy-assets": "copyfiles -u 1 src/**/*.gql dist/"
}, },
"dependencies": { "dependencies": {
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1", "@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^6.0.12", "@ipld/dag-cbor": "^6.0.12",

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/codegen", "name": "@cerc-io/codegen",
"version": "0.2.17", "version": "0.2.18",
"description": "Code generator", "description": "Code generator",
"private": true, "private": true,
"main": "index.js", "main": "index.js",
@ -20,7 +20,7 @@
}, },
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@graphql-tools/load-files": "^6.5.2", "@graphql-tools/load-files": "^6.5.2",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
"@solidity-parser/parser": "^0.13.2", "@solidity-parser/parser": "^0.13.2",

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@cerc-io/cli": "^0.2.17", "@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
{{#if (subgraphPath)}} {{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.17", "@cerc-io/graph-node": "^0.2.18",
{{/if}} {{/if}}
"apollo-type-bigint": "^0.1.3", "apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1", "debug": "^4.3.1",

12
packages/eden-watcher/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/eden-watcher", "name": "@cerc-io/eden-watcher",
"version": "0.2.17", "version": "0.2.18",
"description": "eden-watcher", "description": "eden-watcher",
"private": true, "private": true,
"main": "dist/index.js", "main": "dist/index.js",
@ -38,11 +38,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17", "@cerc-io/cli": "^0.2.18",
"@cerc-io/graph-node": "^0.2.17", "@cerc-io/graph-node": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3", "apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1", "debug": "^4.3.1",

10
packages/erc20-watcher/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/erc20-watcher", "name": "@cerc-io/erc20-watcher",
"version": "0.2.17", "version": "0.2.18",
"description": "ERC20 Watcher", "description": "ERC20 Watcher",
"private": true, "private": true,
"main": "dist/index.js", "main": "dist/index.js",
@ -40,10 +40,10 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17", "@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3", "apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1", "debug": "^4.3.1",

10
packages/erc721-watcher/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/erc721-watcher", "name": "@cerc-io/erc721-watcher",
"version": "0.2.17", "version": "0.2.18",
"description": "erc721-watcher", "description": "erc721-watcher",
"private": true, "private": true,
"main": "dist/index.js", "main": "dist/index.js",
@ -46,10 +46,10 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17", "@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3", "apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1", "debug": "^4.3.1",

12
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{ {
"name": "@cerc-io/graph-node", "name": "@cerc-io/graph-node",
"version": "0.2.17", "version": "0.2.18",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"devDependencies": { "devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0", "@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2", "@nomiclabs/hardhat-ethers": "^2.0.2",
@ -49,11 +49,11 @@
}, },
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cache": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@types/json-diff": "^0.5.2",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@types/json-diff": "^0.5.2",
"bn.js": "^4.11.9", "bn.js": "^4.11.9",
"debug": "^4.3.1", "debug": "^4.3.1",
"fs-extra": "^10.0.0", "fs-extra": "^10.0.0",

12
packages/graph-test-watcher/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/graph-test-watcher", "name": "@cerc-io/graph-test-watcher",
"version": "0.2.17", "version": "0.2.18",
"description": "graph-test-watcher", "description": "graph-test-watcher",
"private": true, "private": true,
"main": "dist/index.js", "main": "dist/index.js",
@ -38,11 +38,11 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17", "@cerc-io/cli": "^0.2.18",
"@cerc-io/graph-node": "^0.2.17", "@cerc-io/graph-node": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3", "apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1", "debug": "^4.3.1",

4
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/ipld-eth-client", "name": "@cerc-io/ipld-eth-client",
"version": "0.2.17", "version": "0.2.18",
"description": "IPLD ETH Client", "description": "IPLD ETH Client",
"main": "dist/index.js", "main": "dist/index.js",
"scripts": { "scripts": {
@ -20,7 +20,7 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.7.1", "@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.17", "@cerc-io/cache": "^0.2.18",
"cross-fetch": "^3.1.4", "cross-fetch": "^3.1.4",
"debug": "^4.3.1", "debug": "^4.3.1",
"ethers": "^5.4.4", "ethers": "^5.4.4",

10
packages/mobymask-watcher/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/mobymask-watcher", "name": "@cerc-io/mobymask-watcher",
"version": "0.2.17", "version": "0.2.18",
"description": "mobymask-watcher", "description": "mobymask-watcher",
"private": true, "private": true,
"main": "dist/index.js", "main": "dist/index.js",
@ -37,10 +37,10 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme", "homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": { "dependencies": {
"@apollo/client": "^3.3.19", "@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17", "@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.17", "@cerc-io/util": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"apollo-type-bigint": "^0.1.3", "apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1", "debug": "^4.3.1",

4
packages/solidity-mapper/package.json
View File

@ -1,10 +1,10 @@
{ {
"name": "@cerc-io/solidity-mapper", "name": "@cerc-io/solidity-mapper",
"version": "0.2.17", "version": "0.2.18",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"devDependencies": { "devDependencies": {
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@ethersproject/abi": "^5.3.0", "@ethersproject/abi": "^5.3.0",
"@nomiclabs/hardhat-ethers": "^2.0.2", "@nomiclabs/hardhat-ethers": "^2.0.2",
"@nomiclabs/hardhat-waffle": "^2.0.1", "@nomiclabs/hardhat-waffle": "^2.0.1",

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/test", "name": "@cerc-io/test",
"version": "0.2.17", "version": "0.2.18",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"private": true, "private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@cerc-io/tracing-client", "name": "@cerc-io/tracing-client",
"version": "0.2.17", "version": "0.2.18",
"description": "ETH VM tracing client", "description": "ETH VM tracing client",
"private": true, "private": true,
"main": "dist/index.js", "main": "dist/index.js",

8
packages/util/package.json
View File

@ -1,11 +1,11 @@
{ {
"name": "@cerc-io/util", "name": "@cerc-io/util",
"version": "0.2.17", "version": "0.2.18",
"main": "dist/index.js", "main": "dist/index.js",
"license": "AGPL-3.0", "license": "AGPL-3.0",
"dependencies": { "dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1", "@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/solidity-mapper": "^0.2.17", "@cerc-io/solidity-mapper": "^0.2.18",
"@ethersproject/providers": "^5.4.4", "@ethersproject/providers": "^5.4.4",
"@graphql-tools/schema": "^9.0.10", "@graphql-tools/schema": "^9.0.10",
"@graphql-tools/utils": "^9.1.1", "@graphql-tools/utils": "^9.1.1",
@ -37,8 +37,8 @@
"yargs": "^17.0.1" "yargs": "^17.0.1"
}, },
"devDependencies": { "devDependencies": {
"@cerc-io/cache": "^0.2.17", "@cerc-io/cache": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.17", "@cerc-io/ipld-eth-client": "^0.2.18",
"@nomiclabs/hardhat-waffle": "^2.0.1", "@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/express": "^4.17.14", "@types/express": "^4.17.14",
"@types/fs-extra": "^9.0.11", "@types/fs-extra": "^9.0.11",

3
packages/util/src/common.ts
View File

@ -55,12 +55,11 @@ export const processBlockByNumber = async (
* @param blockAndEventsMap * @param blockAndEventsMap
*/ */
export const fetchBlocksAtHeight = async ( export const fetchBlocksAtHeight = async (
job: any, blockNumber: number,
indexer: IndexerInterface, indexer: IndexerInterface,
jobQueueConfig: JobQueueConfig, jobQueueConfig: JobQueueConfig,
blockAndEventsMap: Map<string, PrefetchedBlock> blockAndEventsMap: Map<string, PrefetchedBlock>
): Promise<DeepPartial<BlockProgressInterface>[]> => { ): Promise<DeepPartial<BlockProgressInterface>[]> => {
const { blockNumber } = job.data;
let blocks = []; let blocks = [];
// Check for blocks in cache if prefetchBlocksInMem flag set. // Check for blocks in cache if prefetchBlocksInMem flag set.

47
packages/util/src/job-runner.ts
View File

@ -4,7 +4,7 @@
import assert from 'assert'; import assert from 'assert';
import debug from 'debug'; import debug from 'debug';
import { In } from 'typeorm'; import { DeepPartial, In } from 'typeorm';
import { JobQueueConfig } from './config'; import { JobQueueConfig } from './config';
import { import {
@ -19,7 +19,7 @@ import {
QUEUE_HOOKS QUEUE_HOOKS
} from './constants'; } from './constants';
import { JobQueue } from './job-queue'; import { JobQueue } from './job-queue';
import { EventInterface, IndexerInterface } from './types'; import { BlockProgressInterface, EventInterface, IndexerInterface } from './types';
import { wait } from './misc'; import { wait } from './misc';
import { import {
createPruningJob, createPruningJob,
@ -79,14 +79,30 @@ export class JobRunner {
switch (kind) { switch (kind) {
case JOB_KIND_INDEX: { case JOB_KIND_INDEX: {
const blocksToBeIndexed = await fetchBlocksAtHeight( const { data: { cid, blockHash, blockNumber, parentHash, timestamp } } = job;
job,
this._indexer, // Check if blockHash present in job.
this._jobQueueConfig, if (blockHash) {
this._blockAndEventsMap // If blockHash is present it is a job for indexing missing parent block.
); await this._indexBlock(job, {
const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed)); blockTimestamp: timestamp,
await Promise.all(indexBlockPromises); cid,
blockHash,
blockNumber,
parentHash
});
} else {
// If blockHash is not present, it is a job to index the next consecutive blockNumber.
const blocksToBeIndexed = await fetchBlocksAtHeight(
blockNumber,
this._indexer,
this._jobQueueConfig,
this._blockAndEventsMap
);
const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed));
await Promise.all(indexBlockPromises);
}
break; break;
} }
@ -277,12 +293,15 @@ export class JobRunner {
console.timeEnd('time:job-runner#_pruneChain'); console.timeEnd('time:job-runner#_pruneChain');
} }
async _indexBlock (job: any, blockToBeIndexed: any): Promise<void> { async _indexBlock (job: any, blockToBeIndexed: DeepPartial<BlockProgressInterface>): Promise<void> {
const syncStatus = await this._indexer.getSyncStatus(); const syncStatus = await this._indexer.getSyncStatus();
assert(syncStatus); assert(syncStatus);
const { data: { priority } } = job; const { data: { priority } } = job;
const { cid, blockHash, blockNumber, parentHash, blockTimestamp } = blockToBeIndexed; const { cid, blockHash, blockNumber, parentHash, blockTimestamp } = blockToBeIndexed;
assert(blockNumber);
assert(blockHash);
assert(parentHash);
const indexBlockStartTime = new Date(); const indexBlockStartTime = new Date();
@ -350,7 +369,8 @@ export class JobRunner {
const message = `Parent block number ${parentBlockNumber} hash ${parentHash} of block number ${blockNumber} hash ${blockHash} not fetched yet, aborting`; const message = `Parent block number ${parentBlockNumber} hash ${parentHash} of block number ${blockNumber} hash ${blockHash} not fetched yet, aborting`;
log(message); log(message);
throw new Error(message); // Do not throw error and complete the job as block will be processed after parent block processing.
return;
} }
if (!parentBlock.isComplete) { if (!parentBlock.isComplete) {
@ -368,7 +388,8 @@ export class JobRunner {
priority: newPriority priority: newPriority
}, { priority: newPriority }); }, { priority: newPriority });
throw new Error(message); // Do not throw error and complete the job as block will be processed after parent block processing.
return;
} else { } else {
// Remove the unknown events of the parent block if it is marked complete. // Remove the unknown events of the parent block if it is marked complete.
console.time('time:job-runner#_indexBlock-remove-unknown-events'); console.time('time:job-runner#_indexBlock-remove-unknown-events');