CLI to compare eden-watcher entities and fix mapping code (#149)

* Make sumStaked variable local in eden network mapping

* Implement compare CLI to fetch and query by ids

* Set filterLogs to true for eden-watcher

* Use varchar for bigint array type in eden-watcher

* Store subgraph entities by id in IPLD state

* Store bigint vales as string in IPLD state

* Update eden watcher hook to store single Block entity in IPLD checkpoint

* Fix entity enum type property

* Fix parsing big numbers in event params

* Fix event bigint params parsing in all watchers

* Set default limit to query result and process block after events

packages/graph-node/src/cli/compare/client.ts

@@ -5,6 +5,7 @@
 import assert from 'assert';
 import fs from 'fs';
 import path from 'path';
+import pluralize from 'pluralize';
 
 import { gql } from '@apollo/client/core';
 import { GraphQLClient, Config } from '@vulcanize/ipld-eth-client';
@@ -15,6 +16,7 @@ export class Client {
   _graphqlClient: GraphQLClient;
   _queryDir: string;
   _cache: Cache | undefined;
+  _endpoint: string;
 
   constructor (config: Config, queryDir: string) {
     this._config = config;
@@ -22,16 +24,56 @@ export class Client {
 
     const { gqlEndpoint, cache } = config;
     assert(gqlEndpoint, 'Missing gql endpoint');
+    this._endpoint = gqlEndpoint;
 
     this._graphqlClient = new GraphQLClient(config);
 
     this._cache = cache;
   }
 
+  get endpoint () {
+    return this._endpoint;
+  }
+
   async getResult (queryName: string, params: { [key: string]: any }): Promise<any> {
     return this._getCachedOrFetch(queryName, params);
   }
 
+  async getIds (queryName: string, blockNumber: number): Promise<string[]> {
+    const keyObj = { queryName, blockNumber };
+
+    if (this._cache) {
+      const [value, found] = await this._cache.get(keyObj) || [undefined, false];
+      if (found) {
+        return value;
+      }
+    }
+
+    const result = await this._graphqlClient.query(
+      gql(
+        `query($blockNumber: Int){
+          ${pluralize(queryName)}(
+            block: { number: $blockNumber }
+          ) {
+            id
+          }
+        }`
+      ),
+      {
+        blockNumber
+      }
+    );
+
+    const ids = result[pluralize(queryName)].map((entity: { id: string }) => entity.id);
+
+    // Cache the result and return it, if cache is enabled.
+    if (this._cache) {
+      await this._cache.put(keyObj, ids);
+    }
+
+    return ids;
+  }
+
   async _getCachedOrFetch (queryName: string, params: {[key: string]: any}): Promise<any> {
     const keyObj = {
       queryName,

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -5,8 +5,10 @@
 import yargs from 'yargs';
 import 'reflect-metadata';
 import debug from 'debug';
+import assert from 'assert';
 
 import { compareQuery, Config, getClients, getConfig } from './utils';
+import { Client } from './client';
 
 const log = debug('vulcanize:compare-blocks');
 
@@ -40,12 +42,17 @@ export const main = async (): Promise<void> => {
       type: 'boolean',
       describe: 'Whether to print out raw diff object',
       default: false
+    },
+    fetchIds: {
+      type: 'boolean',
+      describe: 'Fetch ids and compare multiple entities',
+      default: false
     }
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
 
-  const { startBlock, endBlock, rawJson, queryDir } = argv;
+  const { startBlock, endBlock, rawJson, queryDir, fetchIds } = argv;
   const queryNames = config.queries.names;
   let diffFound = false;
 
@@ -58,13 +65,27 @@ export const main = async (): Promise<void> => {
     for (const queryName of queryNames) {
       try {
         log(`At block ${blockNumber} for query ${queryName}:`);
-        const resultDiff = await compareQuery(clients, queryName, { block }, rawJson);
 
-        if (resultDiff) {
-          diffFound = true;
-          log('Results mismatch:', resultDiff);
+        if (fetchIds) {
+          const { idsEndpoint } = config.queries;
+          assert(idsEndpoint, 'Specify endpoint for fetching ids when fetchId is true');
+          const client = Object.values(clients).find(client => client.endpoint === config.endpoints[idsEndpoint]);
+          assert(client);
+          const ids = await client.getIds(queryName, blockNumber);
+
+          for (const id of ids) {
+            const isDiff = await compareAndLog(clients, queryName, { block, id }, rawJson);
+
+            if (isDiff) {
+              diffFound = isDiff;
+            }
+          }
         } else {
-          log('Results match.');
+          const isDiff = await compareAndLog(clients, queryName, { block }, rawJson);
+
+          if (isDiff) {
+            diffFound = isDiff;
+          }
         }
       } catch (err: any) {
         log('Error:', err.message);
@@ -78,3 +99,25 @@ export const main = async (): Promise<void> => {
     process.exit(1);
   }
 };
+
+const compareAndLog = async (
+  clients: { client1: Client, client2: Client },
+  queryName: string,
+  params: { [key: string]: any },
+  rawJson: boolean
+): Promise<boolean> => {
+  const resultDiff = await compareQuery(
+    clients,
+    queryName,
+    params,
+    rawJson
+  );
+
+  if (resultDiff) {
+    log('Results mismatch:', resultDiff);
+    return true;
+  }
+
+  log('Results match.');
+  return false;
+};

packages/graph-node/src/cli/compare/compare-entity.ts

@@ -54,11 +54,10 @@ export const main = async (): Promise<void> => {
 
   const queryName = argv.queryName;
   const id = argv.entityId;
-  const blockHash = argv.blockHash;
 
   const block = {
     number: argv.blockNumber,
-    hash: blockHash
+    hash: argv.blockHash
   };
 
   const clients = await getClients(config, argv.queryDir);

packages/graph-node/src/cli/compare/utils.ts

@@ -21,13 +21,14 @@ interface EndpointConfig {
 interface QueryConfig {
   queryDir: string;
   names: string[];
+  idsEndpoint: keyof EndpointConfig;
 }
 
 export interface Config {
   endpoints: EndpointConfig;
   queries: QueryConfig;
   cache: {
-    endpoint: string;
+    endpoint: keyof EndpointConfig;
     config: CacheConfig;
   }
 }

packages/graph-node/src/database.ts

@@ -17,6 +17,8 @@ import {
 
 import { Block, fromEntityValue, toEntityValue } from './utils';
 
+const DEFAULT_LIMIT = 100;
+
 export class Database {
   _config: ConnectionOptions
   _conn!: Connection
@@ -123,7 +125,8 @@ export class Database {
 
             if (isArray) {
               selectQueryBuilder = selectQueryBuilder.distinctOn(['entity.id'])
-                .orderBy('entity.id');
+                .orderBy('entity.id')
+                .limit(DEFAULT_LIMIT);
             } else {
               selectQueryBuilder = selectQueryBuilder.limit(1);
             }
@@ -132,7 +135,8 @@ export class Database {
               // For one to many relational field.
               selectQueryBuilder = selectQueryBuilder.where('entity.id IN (:...ids)', { ids: entityData[field] })
                 .distinctOn(['entity.id'])
-                .orderBy('entity.id');
+                .orderBy('entity.id')
+                .limit(DEFAULT_LIMIT);
 
               // Subquery example if distinctOn is not performant.
               //

packages/graph-node/src/loader.ts

@@ -29,6 +29,8 @@ import {
 } from './utils';
 import { Database } from './database';
 
+const JSONbigString = JSONbig({ storeAsString: true });
+
 // Endianness of BN used in bigInt store host API.
 // Negative bigInt is being stored in wasm in 2's compliment, 'le' representation.
 // (for eg. bigInt.fromString(negativeI32Value))
@@ -104,7 +106,12 @@ export const instantiate = async (
 
         // JSON stringify and parse data for handling unknown types when encoding.
         // For example, decimal.js values are converted to string in the diff data.
-        diffData.state[entityName] = JSONbig.parse(JSONbig.stringify(dbData));
+        diffData.state[entityName] = {
+          // Using JSONbigString to store bigints as string values to be encoded by IPLD dag-cbor.
+          // TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode.
+          // https://github.com/rvagg/cborg#type-encoders
+          [dbData.id]: JSONbigString.parse(JSONbigString.stringify(dbData))
+        };
 
         // Create an auto-diff.
         assert(indexer.createDiffStaged);

packages/graph-node/src/watcher.ts

@@ -123,7 +123,7 @@ export class GraphWatcher {
   async handleEvent (eventData: any) {
     const { contract, event, eventSignature, block, tx: { hash: txHash }, eventIndex } = eventData;
 
-    if (!this._context.block) {
+    if (!this._context.block || this._context.block.blockHash !== block.hash) {
       this._context.block = await getFullBlock(this._ethClient, this._ethProvider, block.hash);
     }