Refactor state creation code (#204)

* Remove support for pushing state to IPFS

* Move job handlers for state creation to util

* Rename state creation related methods and objects

* Update mock indexer used in graph-node testing

* Fetch and merge diffs in batches while creating a state checkpoint

* Fix timing logs while for state checkpoint creation

* Refactor method to get state query result to util

* Accept contracts for state verification in compare CLI config

* Make method to update state status map synchronous

packages/graph-node/README.md

@@ -69,13 +69,13 @@
       ```
 
       The queries will be fired if the corresponding entities are updated.
-    
+
     * Run the CLI:
 
       ```bash
       ./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10
       ```
-    
+
     * For comparing entities after fetching updated entity ids from watcher database:
 
       * Set the watcher config file path and entities directory.
@@ -90,13 +90,13 @@
           [queries.names]
             author = "Author"
             blog = "Blog"
-        
+
         [watcher]
           configPath = "../../graph-test-watcher/environments/local.toml"
           entitiesDir = "../../graph-test-watcher/dist/entity/*"
         ```
-      
-      * To verify diff IPLD state generated at each block, set the watcher endpoint and `verifyState` flag to true
+
+      * To verify `diff` State state generated at each block, set the watcher endpoint and `verifyState` flag to true
 
         ```toml
         [watcher]
@@ -105,7 +105,7 @@
           endpoint = "gqlEndpoint2"
           verifyState = true
         ```
-      
+
       * Run the CLI with `fetch-ids` flag set to true:\
 
         ```bash

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -12,7 +12,17 @@ import _ from 'lodash';
 import { getConfig as getWatcherConfig, wait } from '@cerc-io/util';
 import { GraphQLClient } from '@cerc-io/ipld-eth-client';
 
-import { checkGQLEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig, checkGQLEntitiesInIPLDState } from './utils';
+import {
+  checkGQLEntityInState,
+  compareQuery,
+  Config,
+  getStatesByBlock,
+  checkStateMetaData,
+  combineState,
+  getClients,
+  getConfig,
+  checkGQLEntitiesInState
+} from './utils';
 import { Database } from '../../database';
 import { getSubgraphConfig } from '../../utils';
 
@@ -117,8 +127,14 @@ export const main = async (): Promise<void> => {
     await db.init();
 
     if (config.watcher.verifyState) {
-      const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath);
-      subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address);
+      // Use provided contracts if available; else read from subraph config.
+      if (config.watcher.contracts) {
+        subgraphContracts = config.watcher.contracts;
+      } else {
+        const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath);
+        subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address);
+      }
+
       const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string;
       subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint });
     }
@@ -134,7 +150,7 @@ export const main = async (): Promise<void> => {
       const block = { number: blockNumber };
       const updatedEntityIds: { [entityName: string]: string[] } = {};
       const updatedEntities: Set<string> = new Set();
-      let ipldStateByBlock = {};
+      let stateByBlock = {};
       assert(db);
       console.time(`time:compare-block-${blockNumber}`);
 
@@ -166,18 +182,18 @@ export const main = async (): Promise<void> => {
         assert(db);
         const [block] = await db.getBlocksAtHeight(blockNumber, false);
         assert(subgraphGQLClient);
-        const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
+        const contractStatesByBlock = await getStatesByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
 
-        // Check meta data for each IPLD block found
-        contractIPLDsByBlock.flat().forEach(contractIPLD => {
-          const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
-          if (ipldMetaDataDiff) {
-            log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
+        // Check meta data for each State entry found
+        contractStatesByBlock.flat().forEach(contractStateEntry => {
+          const stateMetaDataDiff = checkStateMetaData(contractStateEntry, contractLatestStateCIDMap, rawJson);
+          if (stateMetaDataDiff) {
+            log('Results mismatch for State meta data:', stateMetaDataDiff);
             diffFound = true;
           }
         });
 
-        ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
+        stateByBlock = combineState(contractStatesByBlock.flat());
       }
 
       await blockDelay;
@@ -205,10 +221,10 @@ export const main = async (): Promise<void> => {
               );
 
               if (config.watcher.verifyState) {
-                const ipldDiff = await checkGQLEntityInIPLDState(ipldStateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields);
+                const stateDiff = await checkGQLEntityInState(stateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields);
 
-                if (ipldDiff) {
-                  log('Results mismatch for IPLD state:', ipldDiff);
+                if (stateDiff) {
+                  log('Results mismatch for State:', stateDiff);
                   diffFound = true;
                 }
               }
@@ -236,10 +252,10 @@ export const main = async (): Promise<void> => {
                 ));
 
                 if (config.watcher.verifyState) {
-                  const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
+                  const stateDiff = await checkGQLEntitiesInState(stateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
 
-                  if (ipldDiff) {
-                    log('Results mismatch for IPLD state:', ipldDiff);
+                  if (stateDiff) {
+                    log('Results mismatch for State:', stateDiff);
                     diffFound = true;
                   }
                 }

packages/graph-node/src/cli/compare/utils.ts

@@ -21,7 +21,7 @@ import { DEFAULT_LIMIT } from '../../database';
 
 const log = debug('vulcanize:compare-utils');
 
-const IPLD_STATE_QUERY = `
+const STATE_QUERY = `
 query getState($blockHash: String!, $contractAddress: String!, $kind: String){
   getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){
     block {
@@ -63,7 +63,8 @@ export interface Config {
     entitiesDir: string;
     verifyState: boolean;
     endpoint: keyof EndpointConfig;
-    skipFields: EntitySkipFields[]
+    skipFields: EntitySkipFields[];
+    contracts: string[];
   }
   cache: {
     endpoint: keyof EndpointConfig;
@@ -169,24 +170,24 @@ export const getClients = async (config: Config, timeDiff: boolean, queryDir?: s
   };
 };
 
-export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
-  // Fetch IPLD states for all contracts
+export const getStatesByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
+  // Fetch States for all contracts
   return Promise.all(contracts.map(async contract => {
     const { getState } = await client.query(
-      gql(IPLD_STATE_QUERY),
+      gql(STATE_QUERY),
       {
         blockHash,
         contractAddress: contract
       }
     );
 
-    const stateIPLDs = [];
+    const states = [];
 
     // If 'checkpoint' is found at the same block, fetch 'diff' as well
     if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) {
       // Check if 'init' present at the same block
       const { getState: getInitState } = await client.query(
-        gql(IPLD_STATE_QUERY),
+        gql(STATE_QUERY),
         {
           blockHash,
           contractAddress: contract,
@@ -195,13 +196,13 @@ export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[]
       );
 
       if (getInitState && getInitState.block.hash === blockHash) {
-        // Append the 'init' IPLD block to the result
-        stateIPLDs.push(getInitState);
+        // Append the 'init' state to the result
+        states.push(getInitState);
       }
 
-      // Check if 'diff' present at the same block
+      // Check if 'diff' state present at the same block
       const { getState: getDiffState } = await client.query(
-        gql(IPLD_STATE_QUERY),
+        gql(STATE_QUERY),
         {
           blockHash,
           contractAddress: contract,
@@ -210,25 +211,25 @@ export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[]
       );
 
       if (getDiffState && getDiffState.block.hash === blockHash) {
-        // Append the 'diff' IPLD block to the result
-        stateIPLDs.push(getDiffState);
+        // Append the 'diff' state to the result
+        states.push(getDiffState);
       }
     }
 
-    // Append the IPLD block to the result
-    stateIPLDs.push(getState);
+    // Append the state to the result
+    states.push(getState);
 
-    return stateIPLDs;
+    return states;
   }));
 };
 
-export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map<string, { diff: string, checkpoint: string }>, rawJson: boolean) => {
-  // Return if IPLD for a contract not found
-  if (!contractIPLD) {
+export const checkStateMetaData = (contractState: {[key: string]: any}, contractLatestStateCIDMap: Map<string, { diff: string, checkpoint: string }>, rawJson: boolean) => {
+  // Return if State for a contract not found
+  if (!contractState) {
     return;
   }
 
-  const { contractAddress, cid, kind, block } = contractIPLD;
+  const { contractAddress, cid, kind, block } = contractState;
 
   const parentCIDs = contractLatestStateCIDMap.get(contractAddress);
   assert(parentCIDs);
@@ -246,7 +247,7 @@ export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLa
   contractLatestStateCIDMap.set(contractAddress, nextParentCIDs);
 
   // Actual meta data from the GQL result
-  const data = JSON.parse(contractIPLD.data);
+  const data = JSON.parse(contractState.data);
 
   // If parentCID not initialized (is empty at start)
   // Take the expected parentCID from the actual data itself
@@ -279,13 +280,13 @@ export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLa
   return compareObjects(expectedMetaData, data.meta, rawJson);
 };
 
-export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => {
-  const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => {
-    if (!contractIPLD) {
+export const combineState = (contractStateEntries: {[key: string]: any}[]): {[key: string]: any} => {
+  const contractStates: {[key: string]: any}[] = contractStateEntries.map(contractStateEntry => {
+    if (!contractStateEntry) {
       return {};
     }
 
-    const data = JSON.parse(contractIPLD.data);
+    const data = JSON.parse(contractStateEntry.data);
 
     // Apply default limit and sort by id on array type relation fields.
     Object.values(data.state)
@@ -309,18 +310,18 @@ export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key:
     return data.state;
   });
 
-  return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
+  return contractStates.reduce((acc, state) => _.merge(acc, state));
 };
 
-export const checkGQLEntityInIPLDState = async (
-  ipldState: {[key: string]: any},
+export const checkGQLEntityInState = async (
+  state: {[key: string]: any},
   entityName: string,
   entityResult: {[key: string]: any},
   id: string,
   rawJson: boolean,
   skipFields: EntitySkipFields[] = []
 ): Promise<string> => {
-  const ipldEntity = ipldState[entityName][id];
+  const stateEntity = state[entityName][id];
 
   // Filter __typename key in GQL result.
   entityResult = omitDeep(entityResult, '__typename');
@@ -329,24 +330,24 @@ export const checkGQLEntityInIPLDState = async (
   skipFields.forEach(({ entity, fields }) => {
     if (entityName === entity) {
       omitDeep(entityResult, fields);
-      omitDeep(ipldEntity, fields);
+      omitDeep(stateEntity, fields);
     }
   });
 
-  const diff = compareObjects(entityResult, ipldEntity, rawJson);
+  const diff = compareObjects(entityResult, stateEntity, rawJson);
 
   return diff;
 };
 
-export const checkGQLEntitiesInIPLDState = async (
-  ipldState: {[key: string]: any},
+export const checkGQLEntitiesInState = async (
+  state: {[key: string]: any},
   entityName: string,
   entitiesResult: any[],
   rawJson: boolean,
   skipFields: EntitySkipFields[] = []
 ): Promise<string> => {
   // Form entities from state to compare with GQL result
-  const stateEntities = ipldState[entityName];
+  const stateEntities = state[entityName];
 
   for (const entityResult of entitiesResult) {
     const stateEntity = stateEntities[entityResult.id];

packages/graph-node/src/database.ts

@@ -31,8 +31,6 @@ import { Block, fromEntityValue, fromStateEntityValues, toEntityValue } from './
 
 export const DEFAULT_LIMIT = 100;
 
-const log = debug('vulcanize:graph-node-database');
-
 interface CachedEntities {
   frothyBlocks: Map<
     string,
@@ -641,7 +639,7 @@ export class Database {
     }, {});
   }
 
-  fromIPLDState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any {
+  fromState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any {
     const repo = this._conn.getRepository(entity);
     const entityFields = repo.metadata.columns;
 
@@ -678,7 +676,7 @@ export class Database {
     }, {});
   }
 
-  cacheUpdatedEntity<Entity> (entityName: string, entity: any, pruned = false): void {
+  cacheUpdatedEntity (entityName: string, entity: any, pruned = false): void {
     const repo = this._conn.getRepository(entityName);
     const tableName = repo.metadata.tableName;
 

packages/graph-node/src/watcher.ts

@@ -12,7 +12,7 @@ import { SelectionNode } from 'graphql';
 
 import { ResultObject } from '@vulcanize/assemblyscript/lib/loader';
 import { EthClient } from '@cerc-io/ipld-eth-client';
-import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util';
+import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, StateInterface, IndexerInterface, BlockProgressInterface, cachePrunedEntitiesCount } from '@cerc-io/util';
 
 import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils';
 import { Context, GraphData, instantiate } from './loader';
@@ -349,9 +349,9 @@ export class GraphWatcher {
     }
   }
 
-  async updateEntitiesFromIPLDState (ipldBlock: IPLDBlockInterface) {
+  async updateEntitiesFromState (state: StateInterface) {
     assert(this._indexer);
-    const data = this._indexer.getIPLDData(ipldBlock);
+    const data = this._indexer.getStateData(state);
 
     for (const [entityName, entities] of Object.entries(data.state)) {
       // Get relations for subgraph entity
@@ -363,13 +363,13 @@ export class GraphWatcher {
 
       const relations = result ? result[1] : {};
 
-      log(`Updating entities from IPLD state for entity ${entityName}`);
-      console.time(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`);
+      log(`Updating entities from State for entity ${entityName}`);
+      console.time(`time:watcher#GraphWatcher-updateEntitiesFromState-update-entity-${entityName}`);
       for (const [id, entityData] of Object.entries(entities as any)) {
-        const dbData = this._database.fromIPLDState(ipldBlock.block, entityName, entityData, relations);
+        const dbData = this._database.fromState(state.block, entityName, entityData, relations);
         await this._database.saveEntity(entityName, dbData);
       }
-      console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`);
+      console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromState-update-entity-${entityName}`);
     }
   }
 

packages/graph-node/test/utils/indexer.ts

@@ -9,8 +9,9 @@ import {
   ServerConfig as ServerConfigInterface,
   ValueResult,
   ContractInterface,
-  IpldStatus as IpldStatusInterface,
-  IPLDBlockInterface
+  StateStatus,
+  StateSyncStatusInterface,
+  StateInterface
 } from '@cerc-io/util';
 import { EthClient } from '@cerc-io/ipld-eth-client';
 import { GetStorageAt, getStorageValue, MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
@@ -107,7 +108,7 @@ export class Indexer implements IndexerInterface {
     assert(blockHash);
     assert(blockNumber);
 
-    return new SyncStatus();
+    return {} as SyncStatusInterface;
   }
 
   async updateSyncStatusIndexedBlock (blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface> {
@@ -115,7 +116,7 @@ export class Indexer implements IndexerInterface {
     assert(blockHash);
     assert(force);
 
-    return new SyncStatus();
+    return {} as SyncStatusInterface;
   }
 
   async updateSyncStatusCanonicalBlock (blockHash: string, blockNumber: number, force?: boolean): Promise<SyncStatusInterface> {
@@ -123,7 +124,7 @@ export class Indexer implements IndexerInterface {
     assert(blockHash);
     assert(force);
 
-    return new SyncStatus();
+    return {} as SyncStatusInterface;
   }
 
   async markBlocksAsPruned (blocks: BlockProgressInterface[]): Promise<void> {
@@ -157,6 +158,22 @@ export class Indexer implements IndexerInterface {
     assert(event);
   }
 
+  async getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined> {
+    return undefined;
+  }
+
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatusInterface> {
+    return {} as StateSyncStatusInterface;
+  }
+
+  async updateStateSyncStatusCheckpointBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatusInterface> {
+    return {} as StateSyncStatusInterface;
+  }
+
+  async getLatestCanonicalBlock (): Promise<BlockProgressInterface> {
+    return {} as BlockProgressInterface;
+  }
+
   isWatchedContract (address : string): ContractInterface | undefined {
     return undefined;
   }
@@ -165,36 +182,20 @@ export class Indexer implements IndexerInterface {
     return undefined;
   }
 
-  getIPLDData (ipldBlock: IPLDBlockInterface): any {
+  async processCanonicalBlock (blockHash: string, blockNumber: number): Promise<void> {
     return undefined;
   }
 
-  async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
+  async processCheckpoint (blockHash: string): Promise<void> {
     return undefined;
   }
-}
 
-class SyncStatus implements SyncStatusInterface {
-  id: number;
-  chainHeadBlockHash: string;
-  chainHeadBlockNumber: number;
-  latestIndexedBlockHash: string;
-  latestIndexedBlockNumber: number;
-  latestCanonicalBlockHash: string;
-  latestCanonicalBlockNumber: number;
-  initialIndexedBlockHash: string;
-  initialIndexedBlockNumber: number;
+  getStateData (state: StateInterface): any {
+    return undefined;
+  }
 
-  constructor () {
-    this.id = 0;
-    this.chainHeadBlockHash = '0';
-    this.chainHeadBlockNumber = 0;
-    this.latestIndexedBlockHash = '0';
-    this.latestIndexedBlockNumber = 0;
-    this.latestCanonicalBlockHash = '0';
-    this.latestCanonicalBlockNumber = 0;
-    this.initialIndexedBlockHash = '0';
-    this.initialIndexedBlockNumber = 0;
+  updateStateStatusMap (address: string, stateStatus: StateStatus): void {
+    return undefined;
   }
 }
 
@@ -205,7 +206,6 @@ class ServerConfig implements ServerConfigInterface {
   kind: string;
   checkpointing: boolean;
   checkpointInterval: number;
-  ipfsApiAddr: string;
   subgraphPath: string;
   disableSubgraphState: boolean;
   wasmRestartBlocksInterval: number;
@@ -220,7 +220,6 @@ class ServerConfig implements ServerConfigInterface {
     this.kind = '';
     this.checkpointing = false;
     this.checkpointInterval = 0;
-    this.ipfsApiAddr = '';
     this.subgraphPath = '';
     this.disableSubgraphState = false;
     this.wasmRestartBlocksInterval = 0;