Clear cache of latest entities on event processing error (#201)

* Clear cache of latest entities on event processing error

* Remove lighthouse-watcher and update ethersjs version

* Handle GraphDecimal type in state entity

* Add option for comparing all entities using paginate

* Clear pruned cached entities at intervals

* Move ipld-demo to graph-node package and remove reset-dbs script

* Implement changes in all watchers and codegen

packages/util/src/config.ts

@@ -42,6 +42,7 @@ export interface ServerConfig {
   wasmRestartBlocksInterval: number;
   filterLogs: boolean;
   maxEventsBlockRange: number;
+  clearEntitiesCacheInterval: number;
 }
 
 export interface UpstreamConfig {

packages/util/src/indexer.ts

@@ -404,6 +404,7 @@ export class Indexer {
 
   async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
     assert(this._db.saveContract);
+    this.updateIPLDStatusMap(address, {});
     const dbTx = await this._db.createTransactionRunner();
 
     // Use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress) if input to address is a contract address.
@@ -838,14 +839,14 @@ export class Indexer {
     try {
       res = await this._db.saveOrUpdateIPLDBlock(dbTx, ipldBlock);
 
-      await dbTx.commitTransaction();
-
       // Get IPLD status for the contract.
       const ipldStatus = this._ipldStatusMap[res.contractAddress];
       assert(ipldStatus, `IPLD status for contract ${res.contractAddress} not found`);
 
       // Update the IPLD status for the kind.
       ipldStatus[res.kind] = res.block.blockNumber;
+
+      await dbTx.commitTransaction();
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;
@@ -888,7 +889,7 @@ export class Indexer {
     }
   }
 
-  async updateIPLDStatusMap (address: string, ipldStatus: IpldStatus): Promise<void> {
+  updateIPLDStatusMap (address: string, ipldStatus: IpldStatus): void {
     // Get and update IPLD status for the contract.
     const ipldStatusOld = this._ipldStatusMap[address];
     this._ipldStatusMap[address] = _.merge(ipldStatusOld, ipldStatus);

packages/util/src/metrics.ts

@@ -68,6 +68,11 @@ export const eventProcessingLoadEntityDBQueryDuration = new client.Histogram({
   help: 'Duration of DB query made in event processing'
 });
 
+export const cachePrunedEntitiesCount = new client.Gauge({
+  name: 'cached_pruned_entities_total',
+  help: 'Total entities in pruned region of cache'
+});
+
 export const eventProcessingEthCallDuration = new client.Histogram({
   name: 'event_processing_eth_call_duration_seconds',
   help: 'Duration of eth_calls made in event processing'