Update CLI to compare only updated entities and verify IPLD state (#161)

* Change compare CLI to verify only updated entities

* Implement IPLD state verification in compare CLI

* Changes to IPLD state to match with GQL result entity

packages/graph-node/README.md

@@ -75,9 +75,9 @@
       ./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10
       ```
     
-    * For comparing entities after fetching ids from one of the endpoints and then querying individually by ids:
+    * For comparing entities after fetching updated entity ids from watcher database:
 
-      * Set the `idsEndpoint` to choose which endpoint the ids should be fetched from.
+      * Set the watcher config file path and entities directory.
 
         ```toml
         [endpoints]
@@ -90,7 +90,20 @@
             "author",
             "blog"
           ]
-          idsEndpoint = "gqlEndpoint1"
+        
+        [watcher]
+          configPath = "../../graph-test-watcher/environments/local.toml"
+          entitiesDir = "../../graph-test-watcher/dist/entity/*"
+        ```
+      
+      * To verify diff IPLD state generated at each block, set the watcher endpoint and `verifyState` flag to true
+
+        ```toml
+        [watcher]
+          configPath = "../../graph-test-watcher/environments/local.toml"
+          entitiesDir = "../../graph-test-watcher/dist/entity/*"
+          endpoint = "gqlEndpoint2"
+          verifyState = true
         ```
       
       * Run the CLI with `fetch-ids` flag set to true:\

packages/graph-node/environments/compare-cli-config.toml

@@ -6,6 +6,13 @@
   queryDir = "../graph-test-watcher/src/gql/queries"
   names = []
   idsEndpoint = "gqlEndpoint1"
+  blockDelayInMs = 250
+
+[watcher]
+  configpath = "../../graph-test-watcher/environments/local.toml"
+  entitiesDir = "../../graph-test-watcher/src/entity"
+  endpoint = "gqlEndpoint2"
+  verifyState = true
 
 [cache]
   endpoint = "gqlEndpoint1"

packages/graph-node/package.json

@@ -58,6 +58,7 @@
     "js-yaml": "^4.1.0",
     "json-bigint": "^1.0.0",
     "json-diff": "^0.5.4",
+    "omit-deep": "^0.3.0",
     "pluralize": "^8.0.0",
     "reflect-metadata": "^0.1.13",
     "toml": "^3.0.0",

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -5,10 +5,17 @@
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
+import path from 'path';
 import assert from 'assert';
+import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
+import _ from 'lodash';
+import omitDeep from 'omit-deep';
+import { getConfig as getWatcherConfig, wait } from '@vulcanize/util';
+import { GraphQLClient } from '@vulcanize/ipld-eth-client';
 
-import { compareQuery, Config, getClients, getConfig } from './utils';
-import { Client } from './client';
+import { compareObjects, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils';
+import { Database } from '../../database';
+import { getSubgraphConfig } from '../../utils';
 
 const log = debug('vulcanize:compare-blocks');
 
@@ -50,48 +57,109 @@ export const main = async (): Promise<void> => {
     }
   }).argv;
 
-  const config: Config = await getConfig(argv.configFile);
-
-  const { startBlock, endBlock, rawJson, queryDir, fetchIds } = argv;
+  const { startBlock, endBlock, rawJson, queryDir, fetchIds, configFile } = argv;
+  const config: Config = await getConfig(configFile);
+  const snakeNamingStrategy = new SnakeNamingStrategy();
+  const clients = await getClients(config, queryDir);
   const queryNames = config.queries.names;
   let diffFound = false;
+  let blockDelay = wait(0);
+  let subgraphContracts: string[] = [];
+  let db: Database | undefined, subgraphGQLClient: GraphQLClient | undefined;
 
-  const clients = await getClients(config, queryDir);
+  if (config.watcher) {
+    const watcherConfigPath = path.resolve(path.dirname(configFile), config.watcher.configPath);
+    const entitiesDir = path.resolve(path.dirname(configFile), config.watcher.entitiesDir);
+    const watcherConfig = await getWatcherConfig(watcherConfigPath);
+    db = new Database(watcherConfig.database, entitiesDir);
+    await db.init();
+
+    if (config.watcher.verifyState) {
+      const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath);
+      subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address);
+      const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string;
+      subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint });
+    }
+  }
 
   for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) {
     const block = { number: blockNumber };
+    let updatedEntityIds: string[][] = [];
+    let ipldStateByBlock = {};
     console.time(`time:compare-block-${blockNumber}`);
 
-    for (const queryName of queryNames) {
+    if (fetchIds) {
+      // Fetch entity ids updated at block.
+      console.time(`time:fetch-updated-ids-${blockNumber}`);
+
+      const updatedEntityIdPromises = queryNames.map(
+        queryName => {
+          assert(db);
+
+          return db.getEntityIdsAtBlockNumber(
+            blockNumber,
+            snakeNamingStrategy.tableName(queryName, '')
+          );
+        }
+      );
+
+      updatedEntityIds = await Promise.all(updatedEntityIdPromises);
+      console.timeEnd(`time:fetch-updated-ids-${blockNumber}`);
+    }
+
+    if (config.watcher.verifyState) {
+      assert(db);
+      const [block] = await db?.getBlocksAtHeight(blockNumber, false);
+      assert(subgraphGQLClient);
+      ipldStateByBlock = await getIPLDStateByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
+    }
+
+    await blockDelay;
+    for (const [index, queryName] of queryNames.entries()) {
       try {
         log(`At block ${blockNumber} for query ${queryName}:`);
+        let resultDiff = '';
 
         if (fetchIds) {
-          const { idsEndpoint } = config.queries;
-          assert(idsEndpoint, 'Specify endpoint for fetching ids when fetchId is true');
-          const client = Object.values(clients).find(client => client.endpoint === config.endpoints[idsEndpoint]);
-          assert(client);
-          const ids = await client.getIds(queryName, blockNumber);
+          for (const id of updatedEntityIds[index]) {
+            const { diff, result1: result } = await compareQuery(
+              clients,
+              queryName,
+              { block, id },
+              rawJson
+            );
 
-          for (const id of ids) {
-            const isDiff = await compareAndLog(clients, queryName, { block, id }, rawJson);
+            if (config.watcher.verifyState) {
+              await checkEntityInIPLDState(ipldStateByBlock, queryName, result, id, rawJson);
+            }
 
-            if (isDiff) {
-              diffFound = isDiff;
+            if (diff) {
+              resultDiff = diff;
             }
           }
         } else {
-          const isDiff = await compareAndLog(clients, queryName, { block }, rawJson);
+          ({ diff: resultDiff } = await compareQuery(
+            clients,
+            queryName,
+            { block },
+            rawJson
+          ));
+        }
 
-          if (isDiff) {
-            diffFound = isDiff;
-          }
+        if (resultDiff) {
+          log('Results mismatch:', resultDiff);
+          diffFound = true;
+        } else {
+          log('Results match.');
         }
       } catch (err: any) {
         log('Error:', err.message);
       }
     }
 
+    // Set delay between requests for a block.
+    blockDelay = wait(config.queries.blockDelayInMs || 0);
+
     console.timeEnd(`time:compare-block-${blockNumber}`);
   }
 
@@ -100,24 +168,21 @@ export const main = async (): Promise<void> => {
   }
 };
 
-const compareAndLog = async (
-  clients: { client1: Client, client2: Client },
+const checkEntityInIPLDState = async (
+  ipldState: {[key: string]: any},
   queryName: string,
-  params: { [key: string]: any },
+  entityResult: {[key: string]: any},
+  id: string,
   rawJson: boolean
-): Promise<boolean> => {
-  const resultDiff = await compareQuery(
-    clients,
-    queryName,
-    params,
-    rawJson
-  );
+) => {
+  const entityName = _.startCase(queryName);
+  const ipldEntity = ipldState[entityName][id];
 
-  if (resultDiff) {
-    log('Results mismatch:', resultDiff);
-    return true;
+  // Filter __typename key in GQL result.
+  const resultEntity = omitDeep(entityResult[queryName], '__typename');
+  const diff = compareObjects(ipldEntity, resultEntity, rawJson);
+
+  if (diff) {
+    log('Results mismatch for IPLD state:', diff);
   }
-
-  log('Results match.');
-  return false;
 };

packages/graph-node/src/cli/compare/compare-entity.ts

@@ -4,9 +4,12 @@
 
 import yargs from 'yargs';
 import 'reflect-metadata';
+import debug from 'debug';
 
 import { compareQuery, Config, getClients, getConfig } from './utils';
 
+const log = debug('vulcanize:compare-entity');
+
 export const main = async (): Promise<void> => {
   const argv = await yargs.parserConfiguration({
     'parse-numbers': false
@@ -62,10 +65,10 @@ export const main = async (): Promise<void> => {
 
   const clients = await getClients(config, argv.queryDir);
 
-  const resultDiff = await compareQuery(clients, queryName, { id, block }, argv.rawJson);
+  const { diff } = await compareQuery(clients, queryName, { id, block }, argv.rawJson);
 
-  if (resultDiff) {
-    console.log(resultDiff);
+  if (diff) {
+    log(diff);
     process.exit(1);
   }
 };

packages/graph-node/src/cli/compare/utils.ts

@@ -8,25 +8,43 @@ import path from 'path';
 import toml from 'toml';
 import fs from 'fs-extra';
 import { diffString, diff } from 'json-diff';
+import _ from 'lodash';
 
 import { Config as CacheConfig, getCache } from '@vulcanize/cache';
+import { GraphQLClient } from '@vulcanize/ipld-eth-client';
+import { gql } from '@apollo/client/core';
 
 import { Client } from './client';
 
+const IPLD_STATE_QUERY = `
+query getState($blockHash: String!, $contractAddress: String!, $kind: String){
+  getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){
+    data
+  }
+}
+`;
+
 interface EndpointConfig {
   gqlEndpoint1: string;
   gqlEndpoint2: string;
+  requestDelayInMs: number;
 }
 
 interface QueryConfig {
   queryDir: string;
   names: string[];
-  idsEndpoint: keyof EndpointConfig;
+  blockDelayInMs: number;
 }
 
 export interface Config {
   endpoints: EndpointConfig;
   queries: QueryConfig;
+  watcher: {
+    configPath: string;
+    entitiesDir: string;
+    verifyState: boolean;
+    endpoint: keyof EndpointConfig;
+  }
   cache: {
     endpoint: keyof EndpointConfig;
     config: CacheConfig;
@@ -52,6 +70,12 @@ export const getConfig = async (configFile: string): Promise<Config> => {
   return config;
 };
 
+interface CompareResult {
+  diff: string,
+  result1: any,
+  result2: any
+}
+
 export const compareQuery = async (
   clients: {
     client1: Client,
@@ -60,27 +84,22 @@ export const compareQuery = async (
   queryName: string,
   params: { [key: string]: any },
   rawJson: boolean
-): Promise<string> => {
+): Promise<CompareResult> => {
   const { client1, client2 } = clients;
 
-  const result2 = await client2.getResult(queryName, params);
-  const result1 = await client1.getResult(queryName, params);
+  const [result1, result2] = await Promise.all([
+    client1.getResult(queryName, params),
+    client2.getResult(queryName, params)
+  ]);
 
   // Getting the diff of two result objects.
-  let resultDiff;
+  const resultDiff = compareObjects(result1, result2, rawJson);
 
-  if (rawJson) {
-    resultDiff = diff(result1, result2);
-
-    if (resultDiff) {
-      // Use util.inspect to extend depth limit in the output.
-      resultDiff = util.inspect(diff(result1, result2), false, null);
-    }
-  } else {
-    resultDiff = diffString(result1, result2);
-  }
-
-  return resultDiff;
+  return {
+    diff: resultDiff,
+    result1,
+    result2
+  };
 };
 
 export const getClients = async (config: Config, queryDir?: string):Promise<{
@@ -121,3 +140,40 @@ export const getClients = async (config: Config, queryDir?: string):Promise<{
     client2
   };
 };
+
+export const getBlockIPLDState = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}> => {
+  const contractIPLDStates: {[key: string]: any}[] = await Promise.all(contracts.map(async contract => {
+    const { getState } = await client.query(
+      gql(IPLD_STATE_QUERY),
+      {
+        blockHash,
+        contractAddress: contract,
+        kind: 'diff'
+      }
+    );
+
+    if (getState) {
+      const data = JSON.parse(getState.data);
+      return data.state;
+    }
+
+    return {};
+  }));
+
+  return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
+};
+
+export const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => {
+  if (rawJson) {
+    const diffObj = diff(obj1, obj2);
+
+    if (diffObj) {
+      // Use util.inspect to extend depth limit in the output.
+      return util.inspect(diffObj, false, null);
+    }
+
+    return '';
+  } else {
+    return diffString(obj1, obj2);
+  }
+};

packages/graph-node/src/database.ts

@@ -7,11 +7,13 @@ import {
   Connection,
   ConnectionOptions,
   FindOneOptions,
-  LessThanOrEqual
+  LessThanOrEqual,
+  Repository
 } from 'typeorm';
 
 import {
   BlockHeight,
+  BlockProgressInterface,
   Database as BaseDatabase
 } from '@vulcanize/util';
 
@@ -77,6 +79,19 @@ export class Database {
     }
   }
 
+  async getEntityIdsAtBlockNumber (blockNumber: number, tableName: string): Promise<string[]> {
+    const repo = this._conn.getRepository(tableName);
+
+    const entities = await repo.find({
+      select: ['id'],
+      where: {
+        blockNumber
+      }
+    });
+
+    return entities.map((entity: any) => entity.id);
+  }
+
   async getEntityWithRelations<Entity> (entity: (new () => Entity) | string, id: string, relations: { [key: string]: any }, block: BlockHeight = {}): Promise<Entity | undefined> {
     const queryRunner = this._conn.createQueryRunner();
     let { hash: blockHash, number: blockNumber } = block;
@@ -265,4 +280,10 @@ export class Database {
       return acc;
     }, {});
   }
+
+  async getBlocksAtHeight (height: number, isPruned: boolean) {
+    const repo: Repository<BlockProgressInterface> = this._conn.getRepository('block_progress');
+
+    return this._baseDatabase.getBlocksAtHeight(repo, height, isPruned);
+  }
 }

packages/graph-node/src/loader.ts

@@ -10,7 +10,6 @@ import {
   Contract,
   ContractInterface
 } from 'ethers';
-import JSONbig from 'json-bigint';
 import BN from 'bn.js';
 import debug from 'debug';
 
@@ -26,12 +25,11 @@ import {
   resolveEntityFieldConflicts,
   getEthereumTypes,
   jsonFromBytes,
-  getStorageValueType
+  getStorageValueType,
+  jsonBigIntStringReplacer
 } from './utils';
 import { Database } from './database';
 
-const JSONbigString = JSONbig({ storeAsString: true });
-
 // Endianness of BN used in bigInt store host API.
 // Negative bigInt is being stored in wasm in 2's compliment, 'le' representation.
 // (for eg. bigInt.fromString(negativeI32Value))
@@ -104,14 +102,39 @@ export const instantiate = async (
 
         // Prepare the diff data.
         const diffData: any = { state: {} };
+        assert(indexer.getRelationsMap);
+
+        const result = Array.from(indexer.getRelationsMap().entries())
+          .find(([key]) => key.name === entityName);
+
+        if (result) {
+          // Update dbData if relations exist.
+          const [_, relations] = result;
+
+          // Update relation fields for diff data to be similar to GQL query entities.
+          Object.entries(relations).forEach(([relation, { isArray, isDerived }]) => {
+            if (isDerived || !dbData[relation]) {
+              // Field is not present in dbData for derived relations
+              return;
+            }
+
+            if (isArray) {
+              dbData[relation] = dbData[relation]
+                .map((id: string) => ({ id }))
+                .sort((a: any, b: any) => a.id.localeCompare(b.id));
+            } else {
+              dbData[relation] = { id: dbData[relation] };
+            }
+          });
+        }
 
         // JSON stringify and parse data for handling unknown types when encoding.
         // For example, decimal.js values are converted to string in the diff data.
         diffData.state[entityName] = {
-          // Using JSONbigString to store bigints as string values to be encoded by IPLD dag-cbor.
+          // Using custom replacer to store bigints as string values to be encoded by IPLD dag-cbor.
           // TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode.
           // https://github.com/rvagg/cborg#type-encoders
-          [dbData.id]: JSONbigString.parse(JSONbigString.stringify(dbData))
+          [dbData.id]: JSON.parse(JSON.stringify(dbData, jsonBigIntStringReplacer))
         };
 
         // Create an auto-diff.

packages/graph-node/src/types/common/main.d.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+declare module 'omit-deep';

packages/graph-node/src/types/common/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "common",
+  "version": "0.1.0",
+  "license": "AGPL-3.0",
+  "typings": "main.d.ts"
+}

packages/graph-node/src/utils.ts

@@ -798,3 +798,11 @@ const getEthereumType = (storageTypes: StorageLayout['types'], type: string, map
 
   return utils.ParamType.from(label);
 };
+
+export const jsonBigIntStringReplacer = (_: string, value: any) => {
+  if (typeof value === 'bigint') {
+    return value.toString();
+  }
+
+  return value;
+};

packages/graph-node/tsconfig.json

@@ -52,7 +52,7 @@
     // "noEmit": true,                                   /* Disable emitting files from a compilation. */
     // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
     // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types */
-    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    "downlevelIteration": true,                          /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
     // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
     // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
     // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */