Update compare CLI to verify IPLD block meta data (#179)

* Update compare CLI to compare IPLD block meta data

* Include init and checkpoints in IPLD blocks verification

* Accomodate changes in codegen

packages/codegen/src/templates/resolvers-template.handlebars

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
+import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -125,7 +125,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
         return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
       },
 
-      getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
+      getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
         log('getState', blockHash, contractAddress, kind);
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('getState').inc(1);

packages/eden-watcher/src/resolvers.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { BlockHeight, OrderDirection, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
+import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -311,7 +311,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
         return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
       },
 
-      getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
+      getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
         log('getState', blockHash, contractAddress, kind);
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('getState').inc(1);

packages/erc721-watcher/src/resolvers.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { ValueResult, BlockHeight, StateKind } from '@cerc-io/util';
+import { ValueResult, BlockHeight } from '@cerc-io/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -166,7 +166,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
         return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
       },
 
-      getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
+      getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
         log('getState', blockHash, contractAddress, kind);
 
         const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -12,7 +12,7 @@ import _ from 'lodash';
 import { getConfig as getWatcherConfig, wait } from '@cerc-io/util';
 import { GraphQLClient } from '@cerc-io/ipld-eth-client';
 
-import { checkEntityInIPLDState, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils';
+import { checkEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig } from './utils';
 import { Database } from '../../database';
 import { getSubgraphConfig } from '../../utils';
 
@@ -64,6 +64,7 @@ export const main = async (): Promise<void> => {
   let diffFound = false;
   let blockDelay = wait(0);
   let subgraphContracts: string[] = [];
+  const contractLatestStateCIDMap: Map<string, string> = new Map();
   let db: Database | undefined, subgraphGQLClient: GraphQLClient | undefined;
 
   if (config.watcher) {
@@ -79,6 +80,10 @@ export const main = async (): Promise<void> => {
       const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string;
       subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint });
     }
+
+    subgraphContracts.forEach(subgraphContract => {
+      contractLatestStateCIDMap.set(subgraphContract, '');
+    });
   }
 
   for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) {
@@ -110,7 +115,18 @@ export const main = async (): Promise<void> => {
       assert(db);
       const [block] = await db?.getBlocksAtHeight(blockNumber, false);
       assert(subgraphGQLClient);
-      ipldStateByBlock = await getIPLDStateByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
+      const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
+
+      // Check meta data for each IPLD block found
+      contractIPLDsByBlock.flat().forEach(contractIPLD => {
+        const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
+        if (ipldMetaDataDiff) {
+          log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
+          diffFound = true;
+        }
+      });
+
+      ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
     }
 
     await blockDelay;
@@ -166,6 +182,7 @@ export const main = async (): Promise<void> => {
         }
       } catch (err: any) {
         log('Error:', err.message);
+        log('Error:', err);
       }
     }
 

packages/graph-node/src/cli/compare/utils.ts

@@ -21,6 +21,14 @@ import { DEFAULT_LIMIT } from '../../database';
 const IPLD_STATE_QUERY = `
 query getState($blockHash: String!, $contractAddress: String!, $kind: String){
   getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){
+    block {
+      cid
+      number
+      hash
+    }
+    contractAddress
+    cid
+    kind
     data
   }
 }
@@ -150,43 +158,131 @@ export const getClients = async (config: Config, queryDir?: string):Promise<{
   };
 };
 
-export const getBlockIPLDState = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}> => {
+export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
-  const contractIPLDStates: {[key: string]: any}[] = await Promise.all(contracts.map(async contract => {
+  // Fetch IPLD states for all contracts
+  return Promise.all(contracts.map(async contract => {
     const { getState } = await client.query(
       gql(IPLD_STATE_QUERY),
       {
         blockHash,
-        contractAddress: contract,
+        contractAddress: contract
-        kind: 'diff'
       }
     );
 
-    if (getState) {
+    const stateIPLDs = [];
-      const data = JSON.parse(getState.data);
 
-      // Apply default limit on array type relation fields.
+    // If 'checkpoint' is found at the same block, fetch 'diff' as well
-      Object.values(data.state)
+    if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) {
-        .forEach((idEntityMap: any) => {
+      // Check if 'init' present at the same block
-          Object.values(idEntityMap)
+      const { getState: getInitState } = await client.query(
-            .forEach((entity: any) => {
+        gql(IPLD_STATE_QUERY),
-              Object.values(entity)
+        {
-                .forEach(fieldValue => {
+          blockHash,
-                  if (
+          contractAddress: contract,
-                    Array.isArray(fieldValue) &&
+          kind: 'init'
-                    fieldValue.length &&
+        }
-                    fieldValue[0].id
+      );
-                  ) {
-                    fieldValue.splice(DEFAULT_LIMIT);
-                  }
-                });
-            });
-        });
 
-      return data.state;
+      if (getInitState && getInitState.block.hash === blockHash) {
+        // Append the 'init' IPLD block to the result
+        stateIPLDs.push(getInitState);
+      }
+
+      // Check if 'diff' present at the same block
+      const { getState: getDiffState } = await client.query(
+        gql(IPLD_STATE_QUERY),
+        {
+          blockHash,
+          contractAddress: contract,
+          kind: 'diff'
+        }
+      );
+
+      if (getDiffState && getDiffState.block.hash === blockHash) {
+        // Append the 'diff' IPLD block to the result
+        stateIPLDs.push(getDiffState);
+      }
     }
 
-    return {};
+    // Append the IPLD block to the result
+    stateIPLDs.push(getState);
+
+    return stateIPLDs;
   }));
+};
+
+export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map<string, string>, rawJson: boolean) => {
+  // Return if IPLD for a contract not found
+  if (!contractIPLD) {
+    return;
+  }
+
+  const { contractAddress, cid, kind, block } = contractIPLD;
+
+  let parentCID = contractLatestStateCIDMap.get(contractAddress);
+  // If CID is same as the parent CID, skip the check
+  if (cid === parentCID) {
+    return;
+  }
+
+  // Update the parent CID in the map
+  contractLatestStateCIDMap.set(contractAddress, cid);
+
+  // Actual meta data from the GQL result
+  const data = JSON.parse(contractIPLD.data);
+
+  // If parentCID not initialized (is empty at start)
+  // Take the expected parentCID from the actual data itself
+  if (parentCID === '') {
+    parentCID = data.meta.parent['/'];
+  }
+
+  // Expected meta data
+  const expectedMetaData = {
+    id: contractAddress,
+    kind,
+    parent: {
+      '/': parentCID
+    },
+    ethBlock: {
+      cid: {
+        '/': block.cid
+      },
+      num: block.number
+    }
+  };
+
+  return compareObjects(expectedMetaData, data.meta, rawJson);
+};
+
+export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => {
+  const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => {
+    if (!contractIPLD) {
+      return {};
+    }
+
+    const data = JSON.parse(contractIPLD.data);
+
+    // Apply default limit on array type relation fields.
+    Object.values(data.state)
+      .forEach((idEntityMap: any) => {
+        Object.values(idEntityMap)
+          .forEach((entity: any) => {
+            Object.values(entity)
+              .forEach(fieldValue => {
+                if (
+                  Array.isArray(fieldValue) &&
+                  fieldValue.length &&
+                  fieldValue[0].id
+                ) {
+                  fieldValue.splice(DEFAULT_LIMIT);
+                }
+              });
+          });
+      });
+
+    return data.state;
+  });
 
   return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
 };
@@ -219,6 +315,8 @@ export const checkEntityInIPLDState = async (
   return diff;
 };
 
+// obj1: expected
+// obj2: actual
 const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => {
   if (rawJson) {
     const diffObj = diff(obj1, obj2);

packages/graph-node/src/loader.ts

@@ -15,7 +15,7 @@ import debug from 'debug';
 
 import { BaseProvider } from '@ethersproject/providers';
 import loader from '@vulcanize/assemblyscript/lib/loader';
-import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp, jsonBigIntStringReplacer } from '@cerc-io/util';
+import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp } from '@cerc-io/util';
 
 import { TypeId, Level } from './types';
 import {

packages/graph-node/tsconfig.json

@@ -11,7 +11,7 @@
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
     /* Language and Environment */
-    "target": "es5",                                     /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    "target": "ES2019",                                     /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */

packages/graph-test-watcher/src/resolvers.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { ValueResult, BlockHeight, StateKind, jsonBigIntStringReplacer } from '@cerc-io/util';
+import { ValueResult, BlockHeight, jsonBigIntStringReplacer } from '@cerc-io/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -122,7 +122,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
         return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
       },
 
-      getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
+      getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
         log('getState', blockHash, contractAddress, kind);
 
         const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);

packages/mobymask-watcher/src/resolvers.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { ValueResult, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';
+import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -131,7 +131,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
         return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
       },
 
-      getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
+      getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
         log('getState', blockHash, contractAddress, kind);
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('getState').inc(1);