cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2025-01-06 19:38:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix sync status update in historical processing job (#488)

Browse Source 
* Fill watcher start block only in active mode

* Fix sync status update in historical processing

* Upgrade package versions

* Upgrade package versions in codegen template

* Fix graph-node test indexer
This commit is contained in:
Nabarun Gogoi 2023-11-23 20:25:39 +05:30 committed by GitHub
parent ffd1d267d8
commit 937aed139f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 104 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "0.2.74",
"version": "0.2.75",
"npmClient": "yarn",
"useWorkspaces": true,
"command": {

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cache",
"version": "0.2.74",
"version": "0.2.75",
"description": "Generic object cache",
"main": "dist/index.js",
"scripts": {

12
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cli",
"version": "0.2.74",
"version": "0.2.75",
"main": "dist/index.js",
"license": "AGPL-3.0",
"scripts": {
@ -12,13 +12,13 @@
},
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.74",
"@cerc-io/ipld-eth-client": "^0.2.74",
"@cerc-io/cache": "^0.2.75",
"@cerc-io/ipld-eth-client": "^0.2.75",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.74",
"@cerc-io/rpc-eth-client": "^0.2.74",
"@cerc-io/util": "^0.2.74",
"@cerc-io/peer": "^0.2.75",
"@cerc-io/rpc-eth-client": "^0.2.75",
"@cerc-io/util": "^0.2.75",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0",

33
packages/cli/src/server.ts
View File

@ -285,25 +285,26 @@ export class ServerCmd {
assert(indexer);
assert(eventWatcher);
const syncStatus = await indexer.getSyncStatus();
if (!syncStatus) {
const contracts = await this.database.getContracts();
const startBlock = getStartBlock(contracts);
await fillBlocks(
jobQueue,
indexer,
eventWatcher,
config.jobQueue.blockDelayInMilliSecs,
{
startBlock,
endBlock: startBlock
}
);
}
if (config.server.kind === KIND_ACTIVE) {
// Delete all active and pending (before completed) jobs to prevent creating jobs after completion of processing previous block
await jobQueue.deleteAllJobs('completed');
const syncStatus = await indexer.getSyncStatus();
if (!syncStatus) {
const contracts = await this.database.getContracts();
const startBlock = getStartBlock(contracts);
await fillBlocks(
jobQueue,
indexer,
eventWatcher,
config.jobQueue.blockDelayInMilliSecs,
{
startBlock,
endBlock: startBlock
}
);
}
await eventWatcher.start();
}

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/codegen",
"version": "0.2.74",
"version": "0.2.75",
"description": "Code generator",
"private": true,
"main": "index.js",
@ -20,7 +20,7 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/util": "^0.2.74",
"@cerc-io/util": "^0.2.75",
"@graphql-tools/load-files": "^6.5.2",
"@npmcli/package-json": "^5.0.0",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

6
packages/codegen/src/templates/database-template.handlebars
View File

@ -265,6 +265,12 @@ export class Database implements DatabaseInterface {
return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
}
async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
const repo = queryRunner.manager.getRepository(SyncStatus);
return this._baseDatabase.updateSyncStatus(repo, syncStatus);
}
async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
const repo = queryRunner.manager.getRepository(SyncStatus);

4
packages/codegen/src/templates/indexer-template.handlebars
View File

@ -673,6 +673,10 @@ export class Indexer implements IndexerInterface {
return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
}
async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
return this._baseIndexer.updateSyncStatus(syncStatus);
}
async getEvent (id: string): Promise<Event | undefined> {
return this._baseIndexer.getEvent(id);
}

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.74",
"@cerc-io/ipld-eth-client": "^0.2.74",
"@cerc-io/solidity-mapper": "^0.2.74",
"@cerc-io/util": "^0.2.74",
"@cerc-io/cli": "^0.2.75",
"@cerc-io/ipld-eth-client": "^0.2.75",
"@cerc-io/solidity-mapper": "^0.2.75",
"@cerc-io/util": "^0.2.75",
{{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.74",
"@cerc-io/graph-node": "^0.2.75",
{{/if}}
"@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1",

10
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/graph-node",
"version": "0.2.74",
"version": "0.2.75",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.74",
"@cerc-io/solidity-mapper": "^0.2.75",
"@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.74",
"@cerc-io/ipld-eth-client": "^0.2.74",
"@cerc-io/util": "^0.2.74",
"@cerc-io/cache": "^0.2.75",
"@cerc-io/ipld-eth-client": "^0.2.75",
"@cerc-io/util": "^0.2.75",
"@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0",
"bn.js": "^4.11.9",

6
packages/graph-node/test/utils/indexer.ts
View File

@ -190,6 +190,12 @@ export class Indexer implements IndexerInterface {
return undefined;
}
async updateSyncStatus (syncStatus: SyncStatusInterface): Promise<SyncStatusInterface> {
assert(syncStatus);
return {} as SyncStatusInterface;
}
async markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise<void> {
assert(blocks);

6
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/ipld-eth-client",
"version": "0.2.74",
"version": "0.2.75",
"description": "IPLD ETH Client",
"main": "dist/index.js",
"scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.74",
"@cerc-io/util": "^0.2.74",
"@cerc-io/cache": "^0.2.75",
"@cerc-io/util": "^0.2.75",
"cross-fetch": "^3.1.4",
"debug": "^4.3.1",
"ethers": "^5.4.4",

2
packages/peer/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/peer",
"version": "0.2.74",
"version": "0.2.75",
"description": "libp2p module",
"main": "dist/index.js",
"exports": "./dist/index.js",

8
packages/rpc-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/rpc-eth-client",
"version": "0.2.74",
"version": "0.2.75",
"description": "RPC ETH Client",
"main": "dist/index.js",
"scripts": {
@ -19,9 +19,9 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/cache": "^0.2.74",
"@cerc-io/ipld-eth-client": "^0.2.74",
"@cerc-io/util": "^0.2.74",
"@cerc-io/cache": "^0.2.75",
"@cerc-io/ipld-eth-client": "^0.2.75",
"@cerc-io/util": "^0.2.75",
"chai": "^4.3.4",
"ethers": "^5.4.4",
"left-pad": "^1.3.0",

2
packages/solidity-mapper/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/solidity-mapper",
"version": "0.2.74",
"version": "0.2.75",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/test",
"version": "0.2.74",
"version": "0.2.75",
"main": "dist/index.js",
"license": "AGPL-3.0",
"private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/tracing-client",
"version": "0.2.74",
"version": "0.2.75",
"description": "ETH VM tracing client",
"main": "dist/index.js",
"scripts": {

8
packages/util/package.json
View File

@ -1,13 +1,13 @@
{
"name": "@cerc-io/util",
"version": "0.2.74",
"version": "0.2.75",
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.74",
"@cerc-io/solidity-mapper": "^0.2.74",
"@cerc-io/peer": "^0.2.75",
"@cerc-io/solidity-mapper": "^0.2.75",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4",
@ -52,7 +52,7 @@
"yargs": "^17.0.1"
},
"devDependencies": {
"@cerc-io/cache": "^0.2.74",
"@cerc-io/cache": "^0.2.75",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14",

9
packages/util/src/database.ts
View File

@ -211,6 +211,15 @@ export class Database {
return repo.save(entity);
}
async updateSyncStatus (repo: Repository<SyncStatusInterface>, syncStatus: DeepPartial<SyncStatusInterface>): Promise<SyncStatusInterface> {
const entity = await repo.findOne();
return await repo.save({
...entity,
...syncStatus
});
}
async getBlockProgress (repo: Repository<BlockProgressInterface>, blockHash: string): Promise<BlockProgressInterface | undefined> {
return repo.findOne({ where: { blockHash } });
}

17
packages/util/src/indexer.ts
View File

@ -291,6 +291,23 @@ export class Indexer {
return res;
}
async updateSyncStatus (syncStatus: DeepPartial<SyncStatusInterface>): Promise<SyncStatusInterface> {
const dbTx = await this._db.createTransactionRunner();
let res;
try {
res = await this._db.updateSyncStatus(dbTx, syncStatus);
await dbTx.commitTransaction();
} catch (error) {
await dbTx.rollbackTransaction();
throw error;
} finally {
await dbTx.release();
}
return res;
}
async getBlocks (blockFilter: { blockNumber?: number, blockHash?: string }): Promise<Array<EthFullBlock | null>> {
assert(blockFilter.blockHash || blockFilter.blockNumber);
const blocks = await this._ethClient.getFullBlocks(blockFilter);

13
packages/util/src/job-runner.ts
View File

@ -237,11 +237,14 @@ export class JobRunner {
// Update sync status canonical, indexed and chain head block to end block
// Update with zero hash as they won't be used during historical processing
await Promise.all([
this._indexer.updateSyncStatusCanonicalBlock(constants.HashZero, endBlock, true),
this._indexer.updateSyncStatusIndexedBlock(constants.HashZero, endBlock, true),
this._indexer.updateSyncStatusChainHead(constants.HashZero, endBlock, true)
]);
await this._indexer.updateSyncStatus({
latestCanonicalBlockHash: constants.HashZero,
latestIndexedBlockHash: constants.HashZero,
chainHeadBlockHash: constants.HashZero,
latestCanonicalBlockNumber: endBlock,
latestIndexedBlockNumber: endBlock,
chainHeadBlockNumber: endBlock
});
log(`Sync status canonical, indexed and chain head block updated to ${endBlock}`);
this._historicalProcessingCompletedUpto = endBlock;

2
packages/util/src/types.ts
View File

@ -195,6 +195,7 @@ export interface IndexerInterface {
updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface>
updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatusInterface | undefined>
updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatusInterface>
updateSyncStatus (syncStatus: DeepPartial<SyncStatusInterface>): Promise<SyncStatusInterface>;
markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise<void>
saveEventEntity (dbEvent: EventInterface): Promise<EventInterface>
saveEvents (dbEvents: DeepPartial<EventInterface>[]): Promise<void>
@ -258,6 +259,7 @@ export interface DatabaseInterface {
updateSyncStatusCanonicalBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface>;
updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatusInterface | undefined>;
updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface>;
updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatusInterface>): Promise<SyncStatusInterface>;
saveEvents (queryRunner: QueryRunner, events: DeepPartial<EventInterface>[]): Promise<void>;
saveBlockWithEvents (queryRunner: QueryRunner, block: DeepPartial<BlockProgressInterface>, events: DeepPartial<EventInterface>[]): Promise<BlockProgressInterface>;
saveEventEntity (queryRunner: QueryRunner, entity: EventInterface): Promise<EventInterface>;