Fixes and improvements for eden-watcher job-runner and compare CLI (#165)

* Compare IPLD state entity without derived fields

* Apply default limit to array relation fields in IPLD state entity

* Mark block as complete after processing of block handler

* Avoid re processing of block handler

* Use LIMIT 1 in the query to get latest IPLD block

* Replace eth_calls in eden-watcher with getStorageValue

* Add checkpoint verification to export state CLI

* Fix get diff blocks query when creating checkpoint

* Fix subgraph staker sort and remove entities sequentially in reset CLI

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>

packages/graph-node/environments/compare-cli-config.toml

@@ -5,14 +5,14 @@
 [queries]
   queryDir = "../graph-test-watcher/src/gql/queries"
   names = []
-  idsEndpoint = "gqlEndpoint1"
   blockDelayInMs = 250
 
 [watcher]
-  configpath = "../../graph-test-watcher/environments/local.toml"
+  configPath = "../../graph-test-watcher/environments/local.toml"
   entitiesDir = "../../graph-test-watcher/src/entity"
   endpoint = "gqlEndpoint2"
   verifyState = true
+  derivedFields = []
 
 [cache]
   endpoint = "gqlEndpoint1"

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -9,11 +9,10 @@ import path from 'path';
 import assert from 'assert';
 import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
 import _ from 'lodash';
-import omitDeep from 'omit-deep';
 import { getConfig as getWatcherConfig, wait } from '@vulcanize/util';
 import { GraphQLClient } from '@vulcanize/ipld-eth-client';
 
-import { compareObjects, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils';
+import { checkEntityInIPLDState, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils';
 import { Database } from '../../database';
 import { getSubgraphConfig } from '../../utils';
 
@@ -130,7 +129,12 @@ export const main = async (): Promise<void> => {
             );
 
             if (config.watcher.verifyState) {
-              await checkEntityInIPLDState(ipldStateByBlock, queryName, result, id, rawJson);
+              const ipldDiff = await checkEntityInIPLDState(ipldStateByBlock, queryName, result, id, rawJson, config.watcher.derivedFields);
+
+              if (ipldDiff) {
+                log('Results mismatch for IPLD state:', ipldDiff);
+                diffFound = true;
+              }
             }
 
             if (diff) {
@@ -167,22 +171,3 @@ export const main = async (): Promise<void> => {
     process.exit(1);
   }
 };
-
-const checkEntityInIPLDState = async (
-  ipldState: {[key: string]: any},
-  queryName: string,
-  entityResult: {[key: string]: any},
-  id: string,
-  rawJson: boolean
-) => {
-  const entityName = _.startCase(queryName);
-  const ipldEntity = ipldState[entityName][id];
-
-  // Filter __typename key in GQL result.
-  const resultEntity = omitDeep(entityResult[queryName], '__typename');
-  const diff = compareObjects(ipldEntity, resultEntity, rawJson);
-
-  if (diff) {
-    log('Results mismatch for IPLD state:', diff);
-  }
-};

packages/graph-node/src/cli/compare/utils.ts

@@ -9,12 +9,14 @@ import toml from 'toml';
 import fs from 'fs-extra';
 import { diffString, diff } from 'json-diff';
 import _ from 'lodash';
+import omitDeep from 'omit-deep';
 
 import { Config as CacheConfig, getCache } from '@vulcanize/cache';
 import { GraphQLClient } from '@vulcanize/ipld-eth-client';
 import { gql } from '@apollo/client/core';
 
 import { Client } from './client';
+import { DEFAULT_LIMIT } from '../../database';
 
 const IPLD_STATE_QUERY = `
 query getState($blockHash: String!, $contractAddress: String!, $kind: String){
@@ -36,6 +38,11 @@ interface QueryConfig {
   blockDelayInMs: number;
 }
 
+interface EntityDerivedFields {
+  entity: string;
+  fields: string[];
+}
+
 export interface Config {
   endpoints: EndpointConfig;
   queries: QueryConfig;
@@ -44,6 +51,7 @@ export interface Config {
     entitiesDir: string;
     verifyState: boolean;
     endpoint: keyof EndpointConfig;
+    derivedFields: EntityDerivedFields[]
   }
   cache: {
     endpoint: keyof EndpointConfig;
@@ -154,6 +162,25 @@ export const getBlockIPLDState = async (client: GraphQLClient, contracts: string
 
     if (getState) {
       const data = JSON.parse(getState.data);
+
+      // Apply default limit on array type relation fields.
+      Object.values(data.state)
+        .forEach((idEntityMap: any) => {
+          Object.values(idEntityMap)
+            .forEach((entity: any) => {
+              Object.values(entity)
+                .forEach(fieldValue => {
+                  if (
+                    Array.isArray(fieldValue) &&
+                    fieldValue.length &&
+                    fieldValue[0].id
+                  ) {
+                    fieldValue.splice(DEFAULT_LIMIT);
+                  }
+                });
+            });
+        });
+
       return data.state;
     }
 
@@ -163,7 +190,35 @@ export const getBlockIPLDState = async (client: GraphQLClient, contracts: string
   return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
 };
 
-export const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => {
+export const checkEntityInIPLDState = async (
+  ipldState: {[key: string]: any},
+  queryName: string,
+  entityResult: {[key: string]: any},
+  id: string,
+  rawJson: boolean,
+  derivedFields: EntityDerivedFields[] = []
+): Promise<string> => {
+  const entityName = _.upperFirst(queryName);
+  const ipldEntity = ipldState[entityName][id];
+
+  // Filter __typename key in GQL result.
+  const resultEntity = omitDeep(entityResult[queryName], '__typename');
+
+  // Filter derived fields in GQL result.
+  derivedFields.forEach(({ entity, fields }) => {
+    if (entityName === entity) {
+      fields.forEach(field => {
+        delete resultEntity[field];
+      });
+    }
+  });
+
+  const diff = compareObjects(ipldEntity, resultEntity, rawJson);
+
+  return diff;
+};
+
+const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => {
   if (rawJson) {
     const diffObj = diff(obj1, obj2);
 

packages/graph-node/src/database.ts

@@ -19,7 +19,7 @@ import {
 
 import { Block, fromEntityValue, toEntityValue } from './utils';
 
-const DEFAULT_LIMIT = 100;
+export const DEFAULT_LIMIT = 100;
 
 export class Database {
   _config: ConnectionOptions

packages/graph-node/src/eden.test.ts

@@ -77,7 +77,8 @@ xdescribe('eden wasm loader tests', async () => {
       },
       dataSource: {
         address: contractAddress,
-        network: 'mainnet'
+        network: 'mainnet',
+        name: 'EdenNetwork'
       }
     };
 
@@ -197,7 +198,8 @@ xdescribe('eden wasm loader tests', async () => {
       },
       dataSource: {
         address: contractAddress,
-        network: 'mainnet'
+        network: 'mainnet',
+        name: 'EdenNetworkDistribution'
       }
     };
 
@@ -313,7 +315,8 @@ xdescribe('eden wasm loader tests', async () => {
       },
       dataSource: {
         address: contractAddress,
-        network: 'mainnet'
+        network: 'mainnet',
+        name: 'EdenNetworkGovernance'
       }
     };
 

packages/graph-node/src/eth-call.test.ts

@@ -29,7 +29,8 @@ xdescribe('eth-call wasm tests', () => {
     },
     dataSource: {
       address: contractAddress,
-      network: 'mainnet'
+      network: 'mainnet',
+      name: 'Example1'
     }
   };
 

packages/graph-node/src/loader.ts

@@ -15,7 +15,7 @@ import debug from 'debug';
 
 import { BaseProvider } from '@ethersproject/providers';
 import loader from '@vulcanize/assemblyscript/lib/loader';
-import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp } from '@vulcanize/util';
+import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp, jsonBigIntStringReplacer } from '@vulcanize/util';
 
 import { TypeId, Level } from './types';
 import {
@@ -25,8 +25,7 @@ import {
   resolveEntityFieldConflicts,
   getEthereumTypes,
   jsonFromBytes,
-  getStorageValueType,
-  jsonBigIntStringReplacer
+  getStorageValueType
 } from './utils';
 import { Database } from './database';
 
@@ -41,6 +40,7 @@ export interface GraphData {
   abis?: {[key: string]: ContractInterface};
   dataSource: {
     network: string;
+    name: string;
   };
 }
 
@@ -261,10 +261,9 @@ export const instantiate = async (
 
         return toEthereumValue(instanceExports, utils.ParamType.from(typesString), decoded);
       },
-      'ethereum.storageValue': async (contractName: number, contractAddress: number, variable: number, mappingKeys: number) => {
-        const contractNameString = __getString(contractName);
-        const address = await Address.wrap(contractAddress);
-        const addressStringPtr = await address.toHexString();
+      'ethereum.storageValue': async (variable: number, mappingKeys: number) => {
+        assert(context.contractAddress);
+        const addressStringPtr = await __newString(context.contractAddress);
         const addressString = __getString(addressStringPtr);
 
         const variableString = __getString(variable);
@@ -276,7 +275,7 @@ export const instantiate = async (
         });
 
         const mappingKeyValues = await Promise.all(mappingKeyPromises);
-        const storageLayout = indexer.storageLayoutMap.get(contractNameString);
+        const storageLayout = indexer.storageLayoutMap.get(dataSource.name);
         assert(storageLayout);
         assert(context.block);
 

packages/graph-node/src/storage-call.test.ts

@@ -30,7 +30,8 @@ xdescribe('storage-call wasm tests', () => {
     },
     dataSource: {
       address: contractAddress,
-      network: 'mainnet'
+      network: 'mainnet',
+      name: 'Example1'
     }
   };
 

packages/graph-node/src/utils.ts

@@ -798,11 +798,3 @@ const getEthereumType = (storageTypes: StorageLayout['types'], type: string, map
 
   return utils.ParamType.from(label);
 };
-
-export const jsonBigIntStringReplacer = (_: string, value: any) => {
-  if (typeof value === 'bigint') {
-    return value.toString();
-  }
-
-  return value;
-};

packages/graph-node/src/watcher.ts

@@ -52,7 +52,7 @@ export class GraphWatcher {
 
     // Create wasm instance and contract interface for each dataSource and template in subgraph yaml.
     const dataPromises = this._dataSources.map(async (dataSource: any) => {
-      const { source: { abi }, mapping, network } = dataSource;
+      const { source: { abi }, mapping, network, name } = dataSource;
       const { abis, file } = mapping;
 
       const abisMap = abis.reduce((acc: {[key: string]: ContractInterface}, abi: any) => {
@@ -68,7 +68,8 @@ export class GraphWatcher {
       const data = {
         abis: abisMap,
         dataSource: {
-          network
+          network,
+          name
         }
       };
 

packages/graph-node/test/subgraph/example1/package.json

@@ -10,7 +10,7 @@
     "deploy-local": "graph deploy --node http://localhost:8020/ --ipfs http://localhost:5001 example1"
   },
   "dependencies": {
-    "@graphprotocol/graph-ts": "npm:@vulcanize/graph-ts@0.22.1",
+    "@graphprotocol/graph-ts": "npm:@vulcanize/graph-ts@0.22.2",
     "@vulcanize/graph-cli": "0.22.5"
   }
 }

packages/graph-node/test/subgraph/example1/src/mapping.ts

@@ -165,7 +165,7 @@ export function testGetStorageValue (): void {
   // Bind the contract to the address.
   const contractAddress = dataSource.address();
   const contract = Example1.bind(contractAddress);
-  const res = contract.getStorageValue('_test', []);
+  const res = ethereum.getStorageValue('_test', []);
   log.debug('Storage call result: {}', [res!.toBigInt().toString()]);
 }
 
@@ -176,7 +176,7 @@ export function testMapStorageValue (): void {
   const contractAddress = dataSource.address();
   const contract = Example1.bind(contractAddress);
   const addressValue = ethereum.Value.fromAddress(Address.zero());
-  const res = contract.getStorageValue('addressUintMap', [addressValue]);
+  const res = ethereum.getStorageValue('addressUintMap', [addressValue]);
   log.debug('Storage call result: {}', [res!.toBigInt().toString()]);
 }
 

packages/graph-node/test/subgraph/example1/yarn.lock

@@ -23,10 +23,10 @@
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
-"@graphprotocol/graph-ts@npm:@vulcanize/graph-ts@0.22.1":
-  version "0.22.1"
-  resolved "https://npm.pkg.github.com/download/@vulcanize/graph-ts/0.22.1/7a14baaab8b99d4a88e19620dc7200aa501fbecf#7a14baaab8b99d4a88e19620dc7200aa501fbecf"
-  integrity sha512-0CoKeFezskYjAsLmqfdxmS7q+gWy1V1wFgiNB4tMJSa2EiPTVG62qlPKkqTduApK2gZX9//rmE5Vb2xcF/v2+w==
+"@graphprotocol/graph-ts@npm:@vulcanize/graph-ts@0.22.2":
+  version "0.22.2"
+  resolved "https://npm.pkg.github.com/download/@vulcanize/graph-ts/0.22.2/a403a4ef6a5742246c4a1c97695a2f55943eb3a7#a403a4ef6a5742246c4a1c97695a2f55943eb3a7"
+  integrity sha512-Fscv1owyoeAkS9QsLGXOalMZlb3j0Ge22z+wmpqA6zJHRiSUyyIyiarSz6e0ZTs761oFqqvt00dR6A/4xxf40A==
   dependencies:
     assemblyscript "0.19.10"