Implement compare CLI for multiple entities for given block range (#108)

* Implement compare CLI for multiple entities at blocks

* Implement caching for gql requests

packages/codegen/src/schema.ts

@@ -194,7 +194,7 @@ export class Schema {
         // Get type composer object for return type from the schema composer.
         type: this._composer.getAnyTC(subgraphType).NonNull,
         args: {
-          id: 'String!',
+          id: 'ID!',
           block: 'Block_height'
         }
       };

packages/codegen/src/templates/reset-state-template.handlebars

@@ -92,7 +92,7 @@ export const handler = async (argv: any): Promise<void> => {
 
     const ipldStatus = await indexer.getIPLDStatus();
 
-    if(ipldStatus) {
+    if (ipldStatus) {
       if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) {
         await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true);
       }

packages/graph-node/assembly/tsconfig.json

@@ -1,5 +1,8 @@
 {
-  "extends": "@vulcanize/assemblyscript/std/assembly.json",
+  "extends": "@graphprotocol/graph-ts/tsconfig",
+  "compilerOptions": {
+    "types": ["@graphprotocol/graph-ts"]
+  },
   "include": [
     "./**/*.ts"
   ]

packages/graph-node/bin/compare-blocks

@@ -0,0 +1,9 @@
+#! /usr/bin/env node
+
+const { main } = require('../dist/cli/compare/compare-blocks')
+
+main().catch(err => {
+  console.log(err);
+}).finally(() => {
+  process.exit(0);
+});

packages/graph-node/environments/compare-cli-config.toml

@@ -4,3 +4,12 @@
 
 [queries]
   queryDir = "../graph-test-watcher/src/gql/queries"
+  names = []
+
+[cache]
+  endpoint = "gqlEndpoint1"
+
+  [cache.config]
+    name = "subgraph-requests"
+    enabled = true
+    deleteOnStart = false

packages/graph-node/src/cli/compare/client.ts

@@ -7,32 +7,58 @@ import fs from 'fs';
 import path from 'path';
 
 import { gql } from '@apollo/client/core';
-import { GraphQLClient, GraphQLConfig } from '@vulcanize/ipld-eth-client';
+import { GraphQLClient, Config } from '@vulcanize/ipld-eth-client';
+import { Cache } from '@vulcanize/cache';
 
 export class Client {
-  _config: GraphQLConfig;
+  _config: Config;
   _graphqlClient: GraphQLClient;
   _queryDir: string;
+  _cache: Cache | undefined;
 
-  constructor (config: GraphQLConfig, queryDir: string) {
+  constructor (config: Config, queryDir: string) {
     this._config = config;
     this._queryDir = path.resolve(process.cwd(), queryDir);
 
-    const { gqlEndpoint } = config;
+    const { gqlEndpoint, cache } = config;
     assert(gqlEndpoint, 'Missing gql endpoint');
 
     this._graphqlClient = new GraphQLClient(config);
+
+    this._cache = cache;
   }
 
-  async getEntity ({ blockHash, queryName, id }: { blockHash: string, queryName: string, id: string }): Promise<any> {
+  async getResult (queryName: string, params: { [key: string]: any }): Promise<any> {
+    return this._getCachedOrFetch(queryName, params);
+  }
+
+  async _getCachedOrFetch (queryName: string, params: {[key: string]: any}): Promise<any> {
+    const keyObj = {
+      queryName,
+      params
+    };
+
+    // Check if request cached in db, if cache is enabled.
+    if (this._cache) {
+      const [value, found] = await this._cache.get(keyObj) || [undefined, false];
+      if (found) {
+        return value;
+      }
+    }
+
     const entityQuery = fs.readFileSync(path.resolve(this._queryDir, `${queryName}.gql`), 'utf8');
 
-    return this._graphqlClient.query(
+    // Result not cached or cache disabled, need to perform an upstream GQL query.
+    const result = await this._graphqlClient.query(
       gql(entityQuery),
-      {
-        id,
-        blockHash
-      }
+      params
     );
+
+    // Cache the result and return it, if cache is enabled.
+    if (this._cache) {
+      await this._cache.put(keyObj, result);
+    }
+
+    return result;
   }
 }

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -0,0 +1,80 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import yargs from 'yargs';
+import 'reflect-metadata';
+import debug from 'debug';
+
+import { compareQuery, Config, getClients, getConfig } from './utils';
+
+const log = debug('vulcanize:compare-blocks');
+
+export const main = async (): Promise<void> => {
+  const argv = await yargs.parserConfiguration({
+    'parse-numbers': false
+  }).options({
+    configFile: {
+      alias: 'cf',
+      type: 'string',
+      demandOption: true,
+      describe: 'Configuration file path (toml)'
+    },
+    queryDir: {
+      alias: 'qf',
+      type: 'string',
+      describe: 'Path to queries directory'
+    },
+    startBlock: {
+      type: 'number',
+      demandOption: true,
+      describe: 'Start block number'
+    },
+    endBlock: {
+      type: 'number',
+      demandOption: true,
+      describe: 'End block number'
+    },
+    rawJson: {
+      alias: 'j',
+      type: 'boolean',
+      describe: 'Whether to print out raw diff object',
+      default: false
+    }
+  }).argv;
+
+  const config: Config = await getConfig(argv.configFile);
+
+  const { startBlock, endBlock, rawJson, queryDir } = argv;
+  const queryNames = config.queries.names;
+  let diffFound = false;
+
+  const clients = await getClients(config, queryDir);
+
+  for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) {
+    const block = { number: blockNumber };
+    console.time(`time:compare-block-${blockNumber}`);
+
+    for (const queryName of queryNames) {
+      try {
+        log(`At block ${blockNumber} for query ${queryName}:`);
+        const resultDiff = await compareQuery(clients, queryName, { block }, rawJson);
+
+        if (resultDiff) {
+          diffFound = true;
+          log('Results mismatch:', resultDiff);
+        } else {
+          log('Results match.');
+        }
+      } catch (err: any) {
+        log('Error:', err.message);
+      }
+    }
+
+    console.timeEnd(`time:compare-block-${blockNumber}`);
+  }
+
+  if (diffFound) {
+    process.exit(1);
+  }
+};

packages/graph-node/src/cli/compare/compare-entity.ts

@@ -4,28 +4,8 @@
 
 import yargs from 'yargs';
 import 'reflect-metadata';
-import path from 'path';
-import toml from 'toml';
-import fs from 'fs-extra';
-import assert from 'assert';
-import util from 'util';
-import { diffString, diff } from 'json-diff';
 
-import { Client } from './client';
-
-interface EndpointConfig {
-  gqlEndpoint1: string;
-  gqlEndpoint2: string;
-}
-
-interface QueryConfig {
-  queryDir: string;
-}
-
-interface Config {
-  endpoints: EndpointConfig;
-  queries: QueryConfig;
-}
+import { compareQuery, Config, getClients, getConfig } from './utils';
 
 export const main = async (): Promise<void> => {
   const argv = await yargs.parserConfiguration({
@@ -45,8 +25,11 @@ export const main = async (): Promise<void> => {
     blockHash: {
       alias: 'b',
       type: 'string',
-      demandOption: true,
-      describe: 'Blockhash'
+      describe: 'Block hash'
+    },
+    blockNumber: {
+      type: 'number',
+      describe: 'Block number'
     },
     queryName: {
       alias: 'q',
@@ -57,7 +40,6 @@ export const main = async (): Promise<void> => {
     entityId: {
       alias: 'i',
       type: 'string',
-      demandOption: true,
       describe: 'Id of the entity to be queried'
     },
     rawJson: {
@@ -70,75 +52,21 @@ export const main = async (): Promise<void> => {
 
   const config: Config = await getConfig(argv.configFile);
 
-  const { client1, client2 } = await getClients(config, argv.queryDir);
-
   const queryName = argv.queryName;
   const id = argv.entityId;
   const blockHash = argv.blockHash;
 
-  const result1 = await client1.getEntity({ blockHash, queryName, id });
-  const result2 = await client2.getEntity({ blockHash, queryName, id });
+  const block = {
+    number: argv.blockNumber,
+    hash: blockHash
+  };
 
-  // Getting the diff of two result objects.
-  let resultDiff;
-  if (argv.rawJson) {
-    resultDiff = diff(result1, result2);
+  const clients = await getClients(config, argv.queryDir);
 
-    if (resultDiff) {
-      // Use util.inspect to extend depth limit in the output.
-      resultDiff = util.inspect(diff(result1, result2), false, null);
-    }
-  } else {
-    resultDiff = diffString(result1, result2);
-  }
+  const resultDiff = await compareQuery(clients, queryName, { id, block }, argv.rawJson);
 
   if (resultDiff) {
     console.log(resultDiff);
     process.exit(1);
   }
 };
-
-async function getConfig (configFile: string): Promise<Config> {
-  const configFilePath = path.resolve(configFile);
-  const fileExists = await fs.pathExists(configFilePath);
-  if (!fileExists) {
-    throw new Error(`Config file not found: ${configFilePath}`);
-  }
-
-  const config = toml.parse(await fs.readFile(configFilePath, 'utf8'));
-
-  return config;
-}
-
-async function getClients (config: Config, queryDir?: string): Promise<{
-  client1: Client,
-  client2: Client
-}> {
-  assert(config.endpoints, 'Missing endpoints config');
-
-  const gqlEndpoint1 = config.endpoints.gqlEndpoint1;
-  const gqlEndpoint2 = config.endpoints.gqlEndpoint2;
-
-  assert(gqlEndpoint1, 'Missing endpoint one');
-  assert(gqlEndpoint2, 'Missing endpoint two');
-
-  if (!queryDir) {
-    assert(config.queries, 'Missing queries config');
-    queryDir = config.queries.queryDir;
-  }
-
-  assert(queryDir, 'Query directory not provided');
-
-  const client1 = new Client({
-    gqlEndpoint: gqlEndpoint1
-  }, queryDir);
-
-  const client2 = new Client({
-    gqlEndpoint: gqlEndpoint2
-  }, queryDir);
-
-  return {
-    client1,
-    client2
-  };
-}

packages/graph-node/src/cli/compare/utils.ts

@@ -0,0 +1,122 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import assert from 'assert';
+import util from 'util';
+import path from 'path';
+import toml from 'toml';
+import fs from 'fs-extra';
+import { diffString, diff } from 'json-diff';
+
+import { Config as CacheConfig, getCache } from '@vulcanize/cache';
+
+import { Client } from './client';
+
+interface EndpointConfig {
+  gqlEndpoint1: string;
+  gqlEndpoint2: string;
+}
+
+interface QueryConfig {
+  queryDir: string;
+  names: string[];
+}
+
+export interface Config {
+  endpoints: EndpointConfig;
+  queries: QueryConfig;
+  cache: {
+    endpoint: string;
+    config: CacheConfig;
+  }
+}
+
+export const getConfig = async (configFile: string): Promise<Config> => {
+  const configFilePath = path.resolve(configFile);
+  const fileExists = await fs.pathExists(configFilePath);
+
+  if (!fileExists) {
+    throw new Error(`Config file not found: ${configFilePath}`);
+  }
+
+  const config = toml.parse(await fs.readFile(configFilePath, 'utf8'));
+
+  if (config.queries.queryDir) {
+    // Resolve path from config file path.
+    const configFileDir = path.dirname(configFilePath);
+    config.queries.queryDir = path.resolve(configFileDir, config.queries.queryDir);
+  }
+
+  return config;
+};
+
+export const compareQuery = async (
+  clients: {
+    client1: Client,
+    client2: Client
+  },
+  queryName: string,
+  params: { [key: string]: any },
+  rawJson: boolean
+): Promise<string> => {
+  const { client1, client2 } = clients;
+
+  const result2 = await client2.getResult(queryName, params);
+  const result1 = await client1.getResult(queryName, params);
+
+  // Getting the diff of two result objects.
+  let resultDiff;
+
+  if (rawJson) {
+    resultDiff = diff(result1, result2);
+
+    if (resultDiff) {
+      // Use util.inspect to extend depth limit in the output.
+      resultDiff = util.inspect(diff(result1, result2), false, null);
+    }
+  } else {
+    resultDiff = diffString(result1, result2);
+  }
+
+  return resultDiff;
+};
+
+export const getClients = async (config: Config, queryDir?: string):Promise<{
+  client1: Client,
+  client2: Client
+}> => {
+  assert(config.endpoints, 'Missing endpoints config');
+
+  const {
+    endpoints: { gqlEndpoint1, gqlEndpoint2 },
+    cache: { endpoint, config: cacheConfig }
+  } = config;
+
+  assert(gqlEndpoint1, 'Missing endpoint one');
+  assert(gqlEndpoint2, 'Missing endpoint two');
+
+  if (!queryDir) {
+    assert(config.queries, 'Missing queries config');
+    queryDir = config.queries.queryDir;
+  }
+
+  assert(queryDir, 'Query directory not provided');
+  assert(cacheConfig, 'Cache config not provided');
+  const cache = await getCache(cacheConfig);
+
+  const client1 = new Client({
+    gqlEndpoint: gqlEndpoint1,
+    cache: endpoint === 'gqlEndpoint1' ? cache : undefined
+  }, queryDir);
+
+  const client2 = new Client({
+    gqlEndpoint: gqlEndpoint2,
+    cache: endpoint === 'gqlEndpoint2' ? cache : undefined
+  }, queryDir);
+
+  return {
+    client1,
+    client2
+  };
+};

packages/ipld-eth-client/src/eth-client.ts

@@ -11,7 +11,7 @@ import ethQueries from './eth-queries';
 import { padKey } from './utils';
 import { GraphQLClient, GraphQLConfig } from './graphql-client';
 
-interface Config extends GraphQLConfig {
+export interface Config extends GraphQLConfig {
   cache: Cache | undefined;
 }