Clear cache of latest entities on event processing error (#201)

* Clear cache of latest entities on event processing error

* Remove lighthouse-watcher and update ethersjs version

* Handle GraphDecimal type in state entity

* Add option for comparing all entities using paginate

* Clear pruned cached entities at intervals

* Move ipld-demo to graph-node package and remove reset-dbs script

* Implement changes in all watchers and codegen

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -16,6 +16,8 @@ import { checkGQLEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, check
 import { Database } from '../../database';
 import { getSubgraphConfig } from '../../utils';
 
+const DEFAULT_ENTITIES_LIMIT = 100;
+
 const log = debug('vulcanize:compare-blocks');
 
 export const main = async (): Promise<void> => {
@@ -70,10 +72,33 @@ export const main = async (): Promise<void> => {
       type: 'boolean',
       describe: 'Compare time taken between GQL queries',
       default: false
+    },
+    queryEntitiesLimit: {
+      type: 'number',
+      default: DEFAULT_ENTITIES_LIMIT,
+      describe: 'Limit for entities returned in query'
+    },
+    paginate: {
+      type: 'boolean',
+      describe: 'Paginate in multiple entities query and compare',
+      default: false
     }
   }).argv;
 
-  const { startBlock, endBlock, batchSize, interval, rawJson, queryDir, fetchIds, configFile, timeDiff } = argv;
+  const {
+    startBlock,
+    endBlock,
+    batchSize,
+    interval,
+    rawJson,
+    queryDir,
+    fetchIds,
+    configFile,
+    timeDiff,
+    queryEntitiesLimit,
+    paginate
+  } = argv;
+
   const config: Config = await getConfig(configFile);
   const snakeNamingStrategy = new SnakeNamingStrategy();
   const clients = await getClients(config, timeDiff, queryDir);
@@ -195,23 +220,41 @@ export const main = async (): Promise<void> => {
           } else {
             if (updatedEntities.has(entityName)) {
               let result;
+              let skip = 0;
 
-              ({ diff: resultDiff, result1: result } = await compareQuery(
-                clients,
-                queryName,
-                { block },
-                rawJson,
-                timeDiff
-              ));
+              do {
+                ({ diff: resultDiff, result1: result } = await compareQuery(
+                  clients,
+                  queryName,
+                  {
+                    block,
+                    skip,
+                    first: queryEntitiesLimit
+                  },
+                  rawJson,
+                  timeDiff
+                ));
 
-              if (config.watcher.verifyState) {
-                const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
+                if (config.watcher.verifyState) {
+                  const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
 
-                if (ipldDiff) {
-                  log('Results mismatch for IPLD state:', ipldDiff);
-                  diffFound = true;
+                  if (ipldDiff) {
+                    log('Results mismatch for IPLD state:', ipldDiff);
+                    diffFound = true;
+                  }
                 }
-              }
+
+                skip += queryEntitiesLimit;
+              } while (
+                // Check if needed to query more entities.
+                result[queryName].length === queryEntitiesLimit &&
+                // Check if diff found.
+                !diffFound &&
+                !resultDiff &&
+                // Check paginate flag
+                // eslint-disable-next-line no-unmodified-loop-condition
+                paginate
+              );
             }
           }
 

packages/graph-node/src/database.ts

@@ -12,6 +12,7 @@ import {
   QueryRunner,
   Repository
 } from 'typeorm';
+import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
 import { SelectionNode } from 'graphql';
 import _ from 'lodash';
 import debug from 'debug';
@@ -647,9 +648,9 @@ export class Database {
     return this.getStateEntityValues(block, stateEntity, entityFields, relations);
   }
 
-  getStateEntityValues (block: BlockProgressInterface, stateEntity: any, entityFields: any, relations: { [key: string]: any } = {}): { [key: string]: any } {
-    const entityValues = entityFields.map((field: any) => {
-      const { propertyName } = field;
+  getStateEntityValues (block: BlockProgressInterface, stateEntity: any, entityFields: ColumnMetadata[], relations: { [key: string]: any } = {}): { [key: string]: any } {
+    const entityValues = entityFields.map((field) => {
+      const { propertyName, transformer } = field;
 
       // Get blockHash property for db entry from block instance.
       if (propertyName === 'blockHash') {
@@ -663,10 +664,10 @@ export class Database {
 
       // Get blockNumber as _blockNumber and blockHash as _blockHash from the entityInstance (wasm).
       if (['_blockNumber', '_blockHash'].includes(propertyName)) {
-        return fromStateEntityValues(stateEntity, propertyName.slice(1), relations);
+        return fromStateEntityValues(stateEntity, propertyName.slice(1), relations, transformer);
       }
 
-      return fromStateEntityValues(stateEntity, propertyName, relations);
+      return fromStateEntityValues(stateEntity, propertyName, relations, transformer);
     }, {});
 
     return entityFields.reduce((acc: { [key: string]: any }, field: any, index: number) => {

packages/graph-node/src/utils.ts

@@ -3,6 +3,7 @@ import path from 'path';
 import fs from 'fs-extra';
 import debug from 'debug';
 import yaml from 'js-yaml';
+import { ValueTransformer } from 'typeorm';
 import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
 import assert from 'assert';
 
@@ -840,7 +841,12 @@ export const prepareEntityState = (updatedEntity: any, entityName: string, relat
   return diffData;
 };
 
-export const fromStateEntityValues = (stateEntity: any, propertyName: string, relations: { [key: string]: any } = {}): any => {
+export const fromStateEntityValues = (
+  stateEntity: any,
+  propertyName: string,
+  relations: { [key: string]: any } = {},
+  transformer?: ValueTransformer | ValueTransformer[]
+): any => {
   // Parse DB data value from state entity data.
   if (relations) {
     const relation = relations[propertyName];
@@ -854,5 +860,16 @@ export const fromStateEntityValues = (stateEntity: any, propertyName: string, re
     }
   }
 
+  if (transformer) {
+    if (Array.isArray(transformer)) {
+      // Apply transformer in reverse order similar to when reading from DB.
+      return transformer.reduceRight((acc, elTransformer) => {
+        return elTransformer.from(acc);
+      }, stateEntity[propertyName]);
+    }
+
+    return transformer.from(stateEntity[propertyName]);
+  }
+
   return stateEntity[propertyName];
 };

packages/graph-node/src/watcher.ts

@@ -12,7 +12,7 @@ import { SelectionNode } from 'graphql';
 
 import { ResultObject } from '@vulcanize/assemblyscript/lib/loader';
 import { EthClient } from '@cerc-io/ipld-eth-client';
-import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface } from '@cerc-io/util';
+import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util';
 
 import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils';
 import { Context, GraphData, instantiate } from './loader';
@@ -186,8 +186,12 @@ export class GraphWatcher {
 
     // Create ethereum event to be passed to the wasm event handler.
     const ethereumEvent = await createEvent(instanceExports, contract, data);
-
-    await this._handleMemoryError(instanceExports[eventHandler.handler](ethereumEvent), dataSource.name);
+    try {
+      await this._handleMemoryError(instanceExports[eventHandler.handler](ethereumEvent), dataSource.name);
+    } catch (error) {
+      this._clearCachedEntities();
+      throw error;
+    }
   }
 
   async handleBlock (blockHash: string) {
@@ -249,7 +253,12 @@ export class GraphWatcher {
           await instanceExports[blockHandler.handler](ethereumBlock);
         });
 
-        await this._handleMemoryError(Promise.all(blockHandlerPromises), dataSource.name);
+        try {
+          await this._handleMemoryError(Promise.all(blockHandlerPromises), dataSource.name);
+        } catch (error) {
+          this._clearCachedEntities();
+          throw error;
+        }
       }
     }
   }
@@ -375,8 +384,18 @@ export class GraphWatcher {
           entities: new Map()
         }
       );
+    }
 
-      log(`Size of cachedEntities.frothyBlocks map: ${this._database.cachedEntities.frothyBlocks.size}`);
+    log(`Size of cachedEntities.frothyBlocks map: ${this._database.cachedEntities.frothyBlocks.size}`);
+    this._measureCachedPrunedEntities();
+
+    assert(this._indexer);
+    // Check if it is time to clear entities cache.
+    if (blockProgress.blockNumber % this._indexer.serverConfig.clearEntitiesCacheInterval === 0) {
+      log(`Clearing cachedEntities.latestPrunedEntities at block ${blockProgress.blockNumber}`);
+      // Clearing only pruned region as frothy region cache gets updated in pruning queue.
+      this._database.cachedEntities.latestPrunedEntities.clear();
+      log(`Cleared cachedEntities.latestPrunedEntities. Map size: ${this._database.cachedEntities.latestPrunedEntities.size}`);
     }
   }
 
@@ -407,6 +426,19 @@ export class GraphWatcher {
     prunedBlockHashes.forEach(blockHash => this._database.cachedEntities.frothyBlocks.delete(blockHash));
   }
 
+  _clearCachedEntities () {
+    this._database.cachedEntities.frothyBlocks.clear();
+    this._database.cachedEntities.latestPrunedEntities.clear();
+  }
+
+  _measureCachedPrunedEntities () {
+    const totalEntities = Array.from(this._database.cachedEntities.latestPrunedEntities.values())
+      .reduce((acc, idEntitiesMap) => acc + idEntitiesMap.size, 0);
+
+    log(`Total entities in cachedEntities.latestPrunedEntities map: ${totalEntities}`);
+    cachePrunedEntitiesCount.set(totalEntities);
+  }
+
   /**
    * Method to reinstantiate WASM instance for specified dataSource.
    * @param dataSourceName