cerc-io/watcher-ts
11
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2026-09-12 10:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use prefetching of blocks with events in watchers and codegen (#206)

Browse Source 
* Avoid refetching block while fetching events

* Prefetch a batch of blocks with events while indexing

* Update mock indexer used in graph-node testing

* Process available blocks while prefetching

* Refactor events fetching to a method in util

* Move method to get GQL event query result to util
This commit is contained in:
prathamesh0
2022-10-20 18:46:56 +05:30
committed by GitHub
parent 668875b3a0
commit 306bbb73ca
38 changed files with 436 additions and 1105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/mobymask-watcher/src/cli/import-state.ts
+1 -1
View File
@@ -74,7 +74,7 @@ export const main = async (): Promise<any> => {
jobQueue,
indexer,
eventWatcher,
config.upstream.ethServer.blockDelayInMilliSecs,
jobQueueConfig.blockDelayInMilliSecs,
{
prefetch: true,
startBlock: importData.snapshotBlock.blockNumber,
packages/mobymask-watcher/src/fill.ts
+1 -1
View File
@@ -76,7 +76,7 @@ export const main = async (): Promise<any> => {
const eventWatcher = new EventWatcher(config.upstream, ethClient, indexer, pubsub, jobQueue);
await fillBlocks(jobQueue, indexer, eventWatcher, config.upstream.ethServer.blockDelayInMilliSecs, argv);
await fillBlocks(jobQueue, indexer, eventWatcher, jobQueueConfig.blockDelayInMilliSecs, argv);
};
main().catch(err => {
packages/mobymask-watcher/src/hooks.ts
+2 -2
View File
@@ -5,9 +5,9 @@
import assert from 'assert';
import { utils } from 'ethers';
// import { updateStateForMappingType, updateStateForElementaryType } from '@cerc-io/util';
import { ResultEvent } from '@cerc-io/util';
import { Indexer, KIND_PHISHERREGISTRY, ResultEvent } from './indexer';
import { Indexer, KIND_PHISHERREGISTRY } from './indexer';
const INVOKE_SIGNATURE = 'invoke(((((address,uint256,bytes),((address,bytes32,(address,bytes)[]),bytes)[])[],(uint256,uint256)),bytes)[])';
const CLAIM_IF_MEMBER_SIGNATURE = 'claimIfMember(string,bool)';
packages/mobymask-watcher/src/indexer.ts
+23 -170
View File
@@ -16,7 +16,6 @@ import {
Indexer as BaseIndexer,
IndexerInterface,
ValueResult,
UNKNOWN_EVENT_NAME,
ServerConfig,
JobQueue,
Where,
@@ -26,7 +25,9 @@ import {
BlockHeight,
StateKind,
StateStatus,
getFullTransaction
getFullTransaction,
ResultEvent,
getResultEvent
} from '@cerc-io/util';
import PhisherRegistryArtifacts from './artifacts/PhisherRegistry.json';
@@ -49,30 +50,6 @@ const JSONbigNative = JSONbig({ useNativeBigInt: true });
export const KIND_PHISHERREGISTRY = 'PhisherRegistry';
export type ResultEvent = {
block: {
cid: string;
hash: string;
number: number;
timestamp: number;
parentHash: string;
};
tx: {
hash: string;
from: string;
to: string;
index: number;
};
contract: string;
eventIndex: number;
eventSignature: string;
event: any;
proof: string;
};
export class Indexer implements IndexerInterface {
_db: Database
_ethClient: EthClient
@@ -124,38 +101,7 @@ export class Indexer implements IndexerInterface {
}
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
return {
block: {
cid: block.cid,
hash: block.blockHash,
number: block.blockNumber,
timestamp: block.blockTimestamp,
parentHash: block.parentHash
},
tx: {
hash: event.txHash,
from: tx.src,
to: tx.dst,
index: tx.index
},
contract: event.contract,
eventIndex: event.index,
eventSignature,
event: {
__typename: `${event.eventName}Event`,
...eventFields
},
// TODO: Return proof only if requested.
proof: JSON.parse(event.proof)
};
return getResultEvent(event);
}
async multiNonce (blockHash: string, contractAddress: string, key0: string, key1: bigint, diff = false): Promise<ValueResult> {
@@ -487,12 +433,12 @@ export class Indexer implements IndexerInterface {
const logDescription = contract.parseLog({ data, topics });
const { eventName, eventInfo } = this._baseIndexer.parseEvent(logDescription);
const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription);
return {
eventName,
eventInfo,
eventSignature: logDescription.signature
eventSignature
};
}
@@ -620,8 +566,8 @@ export class Indexer implements IndexerInterface {
return this._baseIndexer.getBlocksAtHeight(height, isPruned);
}
async fetchBlockWithEvents (block: DeepPartial<BlockProgress>): Promise<BlockProgress> {
return this._baseIndexer.fetchBlockWithEvents(block, this._fetchAndSaveEvents.bind(this));
async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
return this._baseIndexer.saveBlockAndFetchEvents(block, this._saveBlockAndFetchEvents.bind(this));
}
async getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise<Array<Event>> {
@@ -661,126 +607,33 @@ export class Indexer implements IndexerInterface {
return this._contractMap.get(kind);
}
async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial<BlockProgress>): Promise<BlockProgress> {
async _saveBlockAndFetchEvents ({
cid: blockCid,
blockHash,
blockNumber,
blockTimestamp,
parentHash
}: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
assert(blockHash);
const transactionsPromise = this._ethClient.getBlockWithTransactions({ blockHash });
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
const addresses = watchedContracts.map((watchedContract): string => {
return watchedContract.address;
});
const logsResult = await this._ethClient.getLogs({
blockHash,
addresses
});
logs = logsResult.logs;
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
{
allEthHeaderCids: {
nodes: [
{
ethTransactionCidsByHeaderId: {
nodes: transactions
}
}
]
}
}
] = await Promise.all([blockPromise, transactionsPromise]);
const transactionMap = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => {
acc[transaction.txHash] = transaction;
return acc;
}, {});
const dbEvents: Array<DeepPartial<Event>> = [];
for (let li = 0; li < logs.length; li++) {
const logObj = logs[li];
const {
topics,
data,
index: logIndex,
cid,
ipldBlock,
account: {
address
},
transaction: {
hash: txHash
},
receiptCID,
status
} = logObj;
if (status) {
let eventName = UNKNOWN_EVENT_NAME;
let eventInfo = {};
const tx = transactionMap[txHash];
const extraInfo: { [key: string]: any } = { topics, data, tx };
const contract = ethers.utils.getAddress(address);
const watchedContract = await this.isWatchedContract(contract);
if (watchedContract) {
const eventDetails = this.parseEventNameAndArgs(watchedContract.kind, logObj);
eventName = eventDetails.eventName;
eventInfo = eventDetails.eventInfo;
extraInfo.eventSignature = eventDetails.eventSignature;
}
dbEvents.push({
index: logIndex,
txHash,
contract,
eventName,
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {
cid,
ipldBlock
}
})
})
});
} else {
log(`Skipping event for receipt ${receiptCID} due to failed transaction.`);
}
}
const dbEvents = await this._baseIndexer.fetchEvents(blockHash, this.parseEventNameAndArgs.bind(this));
const dbTx = await this._db.createTransactionRunner();
try {
block = {
const block = {
cid: blockCid,
blockHash,
blockNumber: block.number,
blockTimestamp: block.timestamp,
parentHash: block.parent.hash
blockNumber,
blockTimestamp,
parentHash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
console.time(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
const blockProgress = await this._db.saveBlockWithEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
return blockProgress;
return [blockProgress, []];
} catch (error) {
await dbTx.rollbackTransaction();
throw error;