Implement fetch and save of filtered event logs and required blocks (#445)

* Fix async block size caching for missing blocks in historical processing

* Start historical block processing only if filter logs is set to true

* Fetch filtered logs by topics and save required blocks

* Fix realtime processing start block after historical processing

* Avoid publishing events and blocks in historical processing

* Add new method to graph-node test indexer

* Get full block data for subgraph block handler only if configured

* Add useBlockRanges flag for switching between historical and realtime processing

packages/graph-node/src/watcher.ts

@@ -215,21 +215,13 @@ export class GraphWatcher {
   }
 
   async handleBlock (blockHash: string, blockNumber: number) {
-    // Check if block data is already fetched in handleEvent method for the same block.
-    if (!this._context.block || this._context.block.blockHash !== blockHash) {
-      this._context.block = await getFullBlock(this._ethClient, this._ethProvider, blockHash, blockNumber);
-    }
-
-    const blockData = this._context.block;
-    assert(blockData);
-
     // Clear transactions map on handling new block.
     this._transactionsMap.clear();
 
     // Call block handler(s) for each contract.
     for (const dataSource of this._dataSources) {
       // Reinstantiate WASM after every N blocks.
-      if (Number(blockData.blockNumber) % this._wasmRestartBlocksInterval === 0) {
+      if (Number(blockNumber) % this._wasmRestartBlocksInterval === 0) {
         // The WASM instance allocates memory as required and the limit is 4GB.
         // https://stackoverflow.com/a/40453962
         // https://github.com/AssemblyScript/assemblyscript/pull/1268#issue-618411291
@@ -242,6 +234,14 @@ export class GraphWatcher {
         continue;
       }
 
+      // Check if block data is already fetched in handleEvent method for the same block.
+      if (!this._context.block || this._context.block.blockHash !== blockHash) {
+        this._context.block = await getFullBlock(this._ethClient, this._ethProvider, blockHash, blockNumber);
+      }
+
+      const blockData = this._context.block;
+      assert(blockData);
+
       const { instance } = this._dataSourceMap[dataSource.name];
       assert(instance);
       const { exports: instanceExports } = instance;

packages/graph-node/test/utils/indexer.ts

@@ -93,12 +93,6 @@ export class Indexer implements IndexerInterface {
     return [];
   }
 
-  async getLatestProcessedBlockProgress (isPruned: boolean): Promise<BlockProgressInterface | undefined> {
-    assert(isPruned);
-
-    return undefined;
-  }
-
   async getBlockEvents (blockHash: string): Promise<Array<EventInterface>> {
     assert(blockHash);
 
@@ -118,6 +112,13 @@ export class Indexer implements IndexerInterface {
     return [];
   }
 
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{ blockProgress: BlockProgressInterface, events: DeepPartial<EventInterface>[] }[]> {
+    assert(startBlock);
+    assert(endBlock);
+
+    return [];
+  }
+
   async saveBlockAndFetchEvents (block: BlockProgressInterface): Promise<[BlockProgressInterface, DeepPartial<EventInterface>[]]> {
     return [block, []];
   }