cerc-io/watcher-ts
11
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2026-09-08 08:54:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize and reduce queries in block processing job (#308)

Browse Source 
* Improvement TODOs

* Get parent and current block in single query

* Remove block progress queries and events query

* Reduce queries in fetching batch events by using query builder

* Implement changes in codegen package
This commit is contained in:
nabarun
2021-12-13 15:38:34 +05:30
committed by GitHub
parent f56f7a823f
commit b345d25bb8
13 changed files with 270 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/codegen/src/templates/database-template.handlebars
+11 -4
View File
@@ -6,7 +6,7 @@ import assert from 'assert';
import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner, FindManyOptions } from 'typeorm';
import path from 'path';
import { Database as BaseDatabase } from '@vulcanize/util';
import { Database as BaseDatabase, QueryOptions, Where } from '@vulcanize/util';
import { Contract } from './entity/Contract';
import { Event } from './entity/Event';
@@ -105,13 +105,14 @@ export class Database {
return this._baseDatabase.saveEventEntity(repo, entity);
}
async getBlockEvents (blockHash: string, where: FindConditions<Event>): Promise<Event[]> {
async getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise<Event[]> {
const repo = this._conn.getRepository(Event);
return this._baseDatabase.getBlockEvents(repo, blockHash, where);
return this._baseDatabase.getBlockEvents(repo, blockHash, options);
return this._baseDatabase.getBlockEvents(repo, blockHash, where, queryOptions);
}
async saveEvents (queryRunner: QueryRunner, block: DeepPartial<BlockProgress>, events: DeepPartial<Event>[]): Promise<void> {
async saveEvents (queryRunner: QueryRunner, block: DeepPartial<BlockProgress>, events: DeepPartial<Event>[]): Promise<BlockProgress> {
const blockRepo = queryRunner.manager.getRepository(BlockProgress);
const eventRepo = queryRunner.manager.getRepository(Event);
@@ -173,6 +174,12 @@ export class Database {
return this._baseDatabase.getBlockProgress(repo, blockHash);
}
async getBlockProgressEntities (where: FindConditions<BlockProgress>, options: FindManyOptions<BlockProgress>): Promise<BlockProgress[]> {
const repo = this._conn.getRepository(BlockProgress);
return this._baseDatabase.getBlockProgressEntities(repo, where, options);
}
async updateBlockProgress (queryRunner: QueryRunner, blockHash: string, lastProcessedEventIndex: number): Promise<void> {
const repo = queryRunner.manager.getRepository(BlockProgress);
packages/codegen/src/templates/indexer-template.handlebars
+14 -8
View File
@@ -5,14 +5,14 @@
import assert from 'assert';
import debug from 'debug';
import { JsonFragment } from '@ethersproject/abi';
import { DeepPartial } from 'typeorm';
import { DeepPartial, FindConditions } from 'typeorm';
import JSONbig from 'json-bigint';
import { ethers } from 'ethers';
import { BaseProvider } from '@ethersproject/providers';
import { EthClient } from '@vulcanize/ipld-eth-client';
import { StorageLayout } from '@vulcanize/solidity-mapper';
import { EventInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME } from '@vulcanize/util';
import { EventInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, Where, QueryOptions } from '@vulcanize/util';
import { Database } from './database';
import { Contract } from './entity/Contract';
@@ -260,16 +260,20 @@ export class Indexer {
return this._baseIndexer.getBlockProgress(blockHash);
}
async getBlockProgressEntities (where: FindConditions<BlockProgress>, options: FindManyOptions<BlockProgress>): Promise<BlockProgress[]> {
return this._baseIndexer.getBlockProgressEntities(where, options);
}
async getBlocksAtHeight (height: number, isPruned: boolean): Promise<BlockProgress[]> {
return this._baseIndexer.getBlocksAtHeight(height, isPruned);
}
async getOrFetchBlockEvents (block: DeepPartial<BlockProgress>): Promise<Array<EventInterface>> {
return this._baseIndexer.getOrFetchBlockEvents(block, this._fetchAndSaveEvents.bind(this));
async fetchBlockEvents (block: DeepPartial<BlockProgress>): Promise<BlockProgress> {
return this._baseIndexer.fetchBlockEvents(block, this._fetchAndSaveEvents.bind(this));
}
async getBlockEvents (blockHash: string): Promise<Array<Event>> {
return this._baseIndexer.getBlockEvents(blockHash);
async getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise<Array<Event>> {
return this._baseIndexer.getBlockEvents(blockHash, where, queryOptions);
}
async removeUnknownEvents (block: BlockProgress): Promise<void> {
@@ -288,7 +292,7 @@ export class Indexer {
return this._baseIndexer.getAncestorAtDepth(blockHash, depth);
}
async _fetchAndSaveEvents ({ blockHash }: DeepPartial<BlockProgress>): Promise<void> {
async _fetchAndSaveEvents ({ blockHash }: DeepPartial<BlockProgress>): Promise<BlockProgress> {
assert(blockHash);
let { block, logs } = await this._ethClient.getLogs({ blockHash });
@@ -377,8 +381,10 @@ export class Indexer {
parentHash: block.parent.hash
};
await this._db.saveEvents(dbTx, block, dbEvents);
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
return blockProgress;
} catch (error) {
await dbTx.rollbackTransaction();
throw error;