Use prefetching of blocks with events in watchers and codegen (#206)

* Avoid refetching block while fetching events

* Prefetch a batch of blocks with events while indexing

* Update mock indexer used in graph-node testing

* Process available blocks while prefetching

* Refactor events fetching to a method in util

* Move method to get GQL event query result to util

packages/codegen/src/templates/config-template.handlebars

@@ -50,7 +50,6 @@
   [upstream.ethServer]
     gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
     rpcProviderEndpoint = "http://127.0.0.1:8081"
-    blockDelayInMilliSecs = 2000
 
   [upstream.cache]
     name = "requests"
@@ -62,3 +61,6 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  blockDelayInMilliSecs = 2000
+  prefetchBlocksInMem = true
+  prefetchBlockCount = 10

packages/codegen/src/templates/fill-template.handlebars

@@ -109,7 +109,7 @@ export const main = async (): Promise<any> => {
 
   const eventWatcher = new EventWatcher(config.upstream, ethClient, indexer, pubsub, jobQueue);
 
-  await fillBlocks(jobQueue, indexer, eventWatcher, config.upstream.ethServer.blockDelayInMilliSecs, argv);
+  await fillBlocks(jobQueue, indexer, eventWatcher, jobQueueConfig.blockDelayInMilliSecs, argv);
 };
 
 main().catch(err => {

packages/codegen/src/templates/hooks-template.handlebars

@@ -4,9 +4,9 @@
 
 import assert from 'assert';
 
-// import { updateStateForMappingType, updateStateForElementaryType } from '@cerc-io/util';
+import { updateStateForMappingType, updateStateForElementaryType, ResultEvent } from '@cerc-io/util';
 
-import { Indexer, ResultEvent } from './indexer';
+import { Indexer } from './indexer';
 
 /**
  * Hook function to store an initial state.

packages/codegen/src/templates/import-state-template.handlebars

@@ -89,7 +89,7 @@ export const main = async (): Promise<any> => {
     jobQueue,
     indexer,
     eventWatcher,
-    config.upstream.ethServer.blockDelayInMilliSecs,
+    jobQueueConfig.blockDelayInMilliSecs,
     {
       prefetch: true,
       startBlock: importData.snapshotBlock.blockNumber,

packages/codegen/src/templates/indexer-template.handlebars

@@ -20,7 +20,6 @@ import {
   Indexer as BaseIndexer,
   IndexerInterface,
   ValueResult,
-  UNKNOWN_EVENT_NAME,
   ServerConfig,
   JobQueue,
   Where,
@@ -31,7 +30,9 @@ import {
   BlockHeight,
   {{/if}}
   StateKind,
-  StateStatus
+  StateStatus,
+  ResultEvent,
+  getResultEvent
 } from '@cerc-io/util';
 {{#if (subgraphPath)}}
 import { GraphWatcher } from '@cerc-io/graph-node';
@@ -64,30 +65,6 @@ const KIND_{{capitalize contract.contractName}} = '{{contract.contractKind}}';
 const {{capitalize event}}_EVENT = '{{event}}';
 
 {{/each}}
-export type ResultEvent = {
-  block: {
-    cid: string;
-    hash: string;
-    number: number;
-    timestamp: number;
-    parentHash: string;
-  };
-  tx: {
-    hash: string;
-    from: string;
-    to: string;
-    index: number;
-  };
-
-  contract: string;
-
-  eventIndex: number;
-  eventSignature: string;
-  event: any;
-
-  proof: string;
-};
-
 export class Indexer implements IndexerInterface {
   _db: Database
   _ethClient: EthClient
@@ -170,38 +147,7 @@ export class Indexer implements IndexerInterface {
   }
 
   getResultEvent (event: Event): ResultEvent {
-    const block = event.block;
-    const eventFields = JSONbigNative.parse(event.eventInfo);
-    const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
-
-    return {
-      block: {
-        cid: block.cid,
-        hash: block.blockHash,
-        number: block.blockNumber,
-        timestamp: block.blockTimestamp,
-        parentHash: block.parentHash
-      },
-
-      tx: {
-        hash: event.txHash,
-        from: tx.src,
-        to: tx.dst,
-        index: tx.index
-      },
-
-      contract: event.contract,
-
-      eventIndex: event.index,
-      eventSignature,
-      event: {
-        __typename: `${event.eventName}Event`,
-        ...eventFields
-      },
-
-      // TODO: Return proof only if requested.
-      proof: JSON.parse(event.proof)
-    };
+    return getResultEvent(event);
   }
 
   {{#each queries as | query |}}
@@ -458,12 +404,12 @@ export class Indexer implements IndexerInterface {
 
     const logDescription = contract.parseLog({ data, topics });
 
-    const { eventName, eventInfo } = this._baseIndexer.parseEvent(logDescription);
+    const { eventName, eventInfo, eventSignature } = this._baseIndexer.parseEvent(logDescription);
 
     return {
       eventName,
       eventInfo,
-      eventSignature: logDescription.signature
+      eventSignature
     };
   }
 
@@ -687,126 +633,33 @@ export class Indexer implements IndexerInterface {
   }
   {{/if}}
 
-  async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial<BlockProgress>): Promise<BlockProgress> {
+  async _saveBlockAndFetchEvents ({
+    cid: blockCid,
+    blockHash,
+    blockNumber,
+    blockTimestamp,
+    parentHash
+  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
     assert(blockHash);
-    const transactionsPromise = this._ethClient.getBlockWithTransactions({ blockHash });
-    const blockPromise = this._ethClient.getBlockByHash(blockHash);
-    let logs: any[];
 
-    console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
-    if (this._serverConfig.filterLogs) {
-      const watchedContracts = this._baseIndexer.getWatchedContracts();
-      const addresses = watchedContracts.map((watchedContract): string => {
-        return watchedContract.address;
-      });
-
-      const logsResult = await this._ethClient.getLogs({
-        blockHash,
-        addresses
-      });
-
-      logs = logsResult.logs;
-    } else {
-      ({ logs } = await this._ethClient.getLogs({ blockHash }));
-    }
-    console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
-
-    let [
-      { block },
-      {
-        allEthHeaderCids: {
-          nodes: [
-            {
-              ethTransactionCidsByHeaderId: {
-                nodes: transactions
-              }
-            }
-          ]
-        }
-      }
-    ] = await Promise.all([blockPromise, transactionsPromise]);
-
-    const transactionMap = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => {
-      acc[transaction.txHash] = transaction;
-      return acc;
-    }, {});
-
-    const dbEvents: Array<DeepPartial<Event>> = [];
-
-    for (let li = 0; li < logs.length; li++) {
-      const logObj = logs[li];
-      const {
-        topics,
-        data,
-        index: logIndex,
-        cid,
-        ipldBlock,
-        account: {
-          address
-        },
-        transaction: {
-          hash: txHash
-        },
-        receiptCID,
-        status
-      } = logObj;
-
-      if (status) {
-        let eventName = UNKNOWN_EVENT_NAME;
-        let eventInfo = {};
-        const tx = transactionMap[txHash];
-        const extraInfo: { [key: string]: any } = { topics, data, tx };
-
-        const contract = ethers.utils.getAddress(address);
-        const watchedContract = await this.isWatchedContract(contract);
-
-        if (watchedContract) {
-          const eventDetails = this.parseEventNameAndArgs(watchedContract.kind, logObj);
-          eventName = eventDetails.eventName;
-          eventInfo = eventDetails.eventInfo;
-          extraInfo.eventSignature = eventDetails.eventSignature;
-        }
-
-        dbEvents.push({
-          index: logIndex,
-          txHash,
-          contract,
-          eventName,
-          eventInfo: JSONbigNative.stringify(eventInfo),
-          extraInfo: JSONbigNative.stringify(extraInfo),
-          proof: JSONbigNative.stringify({
-            data: JSONbigNative.stringify({
-              blockHash,
-              receiptCID,
-              log: {
-                cid,
-                ipldBlock
-              }
-            })
-          })
-        });
-      } else {
-        log(`Skipping event for receipt ${receiptCID} due to failed transaction.`);
-      }
-    }
+    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
-
     try {
-      block = {
+      const block = {
         cid: blockCid,
         blockHash,
-        blockNumber: block.number,
-        blockTimestamp: block.timestamp,
-        parentHash: block.parent.hash
+        blockNumber,
+        blockTimestamp,
+        parentHash
       };
 
-      console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
+      console.time(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
       const blockProgress = await this._db.saveBlockWithEvents(dbTx, block, dbEvents);
       await dbTx.commitTransaction();
-      console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
+      console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return blockProgress;
+      return [blockProgress, []];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;