Fixes and improvements for eden-watcher job-runner and compare CLI (#165)

* Compare IPLD state entity without derived fields

* Apply default limit to array relation fields in IPLD state entity

* Mark block as complete after processing of block handler

* Avoid re processing of block handler

* Use LIMIT 1 in the query to get latest IPLD block

* Replace eth_calls in eden-watcher with getStorageValue

* Add checkpoint verification to export state CLI

* Fix get diff blocks query when creating checkpoint

* Fix subgraph staker sort and remove entities sequentially in reset CLI

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>

packages/graph-test-watcher/src/cli/export-state.ts

@@ -9,7 +9,7 @@ import debug from 'debug';
 import fs from 'fs';
 import path from 'path';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@vulcanize/util';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind, verifyCheckpointData } from '@vulcanize/util';
 import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
 import * as codec from '@ipld/dag-cbor';
 
@@ -34,6 +34,18 @@ const main = async (): Promise<void> => {
       alias: 'o',
       type: 'string',
       describe: 'Export file path'
+    },
+    createCheckpoint: {
+      alias: 'c',
+      type: 'boolean',
+      describe: 'Create new checkpoint',
+      default: false
+    },
+    verify: {
+      alias: 'v',
+      type: 'boolean',
+      describe: 'Verify checkpoint',
+      default: true
     }
   }).argv;
 
@@ -92,13 +104,22 @@ const main = async (): Promise<void> => {
 
     // Create and export checkpoint if checkpointing is on for the contract.
     if (contract.checkpoint) {
-      await indexer.createCheckpoint(contract.address, block.blockHash);
+      if (argv.createCheckpoint) {
+        log(`Creating checkpoint at block ${block.blockNumber}`);
+        await indexer.createCheckpoint(contract.address, block.blockHash);
+      }
 
       const ipldBlock = await indexer.getLatestIPLDBlock(contract.address, StateKind.Checkpoint, block.blockNumber);
       assert(ipldBlock);
 
       const data = indexer.getIPLDData(ipldBlock);
 
+      if (argv.verify) {
+        log(`Verifying checkpoint data for contract ${contract.address}`);
+        await verifyCheckpointData(graphDb, ipldBlock.block, data);
+        log('Checkpoint data verified');
+      }
+
       if (indexer.isIPFSConfigured()) {
         await indexer.pushToIPFS(data);
       }

packages/graph-test-watcher/src/cli/reset-cmds/state.ts

@@ -71,11 +71,9 @@ export const handler = async (argv: any): Promise<void> => {
   try {
     const entities = [BlockProgress, GetMethod, _Test, Author, Category, Blog];
 
-    const removeEntitiesPromise = entities.map(async entityClass => {
-      return db.removeEntities<any>(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) });
-    });
-
-    await Promise.all(removeEntitiesPromise);
+    for (const entity of entities) {
+      await db.removeEntities<any>(dbTx, entity, { blockNumber: MoreThan(argv.blockNumber) });
+    }
 
     const syncStatus = await indexer.getSyncStatus();
     assert(syncStatus, 'Missing syncStatus');

packages/graph-test-watcher/src/database.ts

@@ -96,10 +96,10 @@ export class Database implements IPLDDatabaseInterface {
   }
 
   // Fetch all diff IPLDBlocks after the specified block number.
-  async getDiffIPLDBlocksByBlocknumber (contractAddress: string, blockNumber: number): Promise<IPLDBlock[]> {
+  async getDiffIPLDBlocksInRange (contractAddress: string, startBlock: number, endBlock: number): Promise<IPLDBlock[]> {
     const repo = this._conn.getRepository(IPLDBlock);
 
-    return this._baseDatabase.getDiffIPLDBlocksByBlocknumber(repo, contractAddress, blockNumber);
+    return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startBlock, endBlock);
   }
 
   async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {