Update watcher import CLI to create entities from checkpoint (#186)

* Import entities from checkpoint

* Fix IPLD state when updating subgraph entity

* Changes in codegen and other watchers

* Update IPLD state with all Block entities

* Add verify and create sub commands to checkpoint

* Add option for specifying snapshot block in export state CLI

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -182,7 +182,7 @@ export const main = async (): Promise<void> => {
         }
       } catch (err: any) {
         log('Error:', err.message);
-        log('Error:', err);
+        log('Error:', JSON.stringify(err, null, 2));
       }
     }
 

packages/graph-node/src/cli/compare/utils.ts

@@ -276,7 +276,7 @@ export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key:
 
     const data = JSON.parse(contractIPLD.data);
 
-    // Apply default limit on array type relation fields.
+    // Apply default limit and sort by id on array type relation fields.
     Object.values(data.state)
       .forEach((idEntityMap: any) => {
         Object.values(idEntityMap)
@@ -288,6 +288,7 @@ export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key:
                   fieldValue.length &&
                   fieldValue[0].id
                 ) {
+                  fieldValue.sort((a: any, b: any) => a.id.localeCompare(b.id));
                   fieldValue.splice(DEFAULT_LIMIT);
                 }
               });
@@ -323,7 +324,7 @@ export const checkEntityInIPLDState = async (
     }
   });
 
-  const diff = compareObjects(ipldEntity, resultEntity, rawJson);
+  const diff = compareObjects(resultEntity, ipldEntity, rawJson);
 
   return diff;
 };

packages/graph-node/src/database.ts

@@ -21,7 +21,7 @@ import {
   Where
 } from '@cerc-io/util';
 
-import { Block, fromEntityValue, toEntityValue } from './utils';
+import { Block, fromEntityValue, fromStateEntityValues, toEntityValue } from './utils';
 
 export const DEFAULT_LIMIT = 100;
 
@@ -508,6 +508,43 @@ export class Database {
     }, {});
   }
 
+  fromIPLDState (block: BlockProgressInterface, entity: string, stateEntity: any, relations: { [key: string]: any } = {}): any {
+    const repo = this._conn.getRepository(entity);
+    const entityFields = repo.metadata.columns;
+
+    return this.getStateEntityValues(block, stateEntity, entityFields, relations);
+  }
+
+  getStateEntityValues (block: BlockProgressInterface, stateEntity: any, entityFields: any, relations: { [key: string]: any } = {}): { [key: string]: any } {
+    const entityValues = entityFields.map((field: any) => {
+      const { propertyName } = field;
+
+      // Get blockHash property for db entry from block instance.
+      if (propertyName === 'blockHash') {
+        return block.blockHash;
+      }
+
+      // Get blockNumber property for db entry from block instance.
+      if (propertyName === 'blockNumber') {
+        return block.blockNumber;
+      }
+
+      // Get blockNumber as _blockNumber and blockHash as _blockHash from the entityInstance (wasm).
+      if (['_blockNumber', '_blockHash'].includes(propertyName)) {
+        return fromStateEntityValues(stateEntity, propertyName.slice(1), relations);
+      }
+
+      return fromStateEntityValues(stateEntity, propertyName, relations);
+    }, {});
+
+    return entityFields.reduce((acc: { [key: string]: any }, field: any, index: number) => {
+      const { propertyName } = field;
+      acc[propertyName] = entityValues[index];
+
+      return acc;
+    }, {});
+  }
+
   async getBlocksAtHeight (height: number, isPruned: boolean) {
     const repo: Repository<BlockProgressInterface> = this._conn.getRepository('block_progress');
 

packages/graph-node/src/utils.ts

@@ -821,9 +821,7 @@ export const prepareEntityState = (updatedEntity: any, entityName: string, relat
       }
 
       if (isArray) {
-        updatedEntity[relation] = updatedEntity[relation]
-          .map((id: string) => ({ id }))
-          .sort((a: any, b: any) => a.id.localeCompare(b.id));
+        updatedEntity[relation] = updatedEntity[relation].map((id: string) => ({ id }));
       } else {
         updatedEntity[relation] = { id: updatedEntity[relation] };
       }
@@ -841,3 +839,20 @@ export const prepareEntityState = (updatedEntity: any, entityName: string, relat
 
   return diffData;
 };
+
+export const fromStateEntityValues = (stateEntity: any, propertyName: string, relations: { [key: string]: any } = {}): any => {
+  // Parse DB data value from state entity data.
+  if (relations) {
+    const relation = relations[propertyName];
+
+    if (relation) {
+      if (relation.isArray) {
+        return stateEntity[propertyName].map((relatedEntity: { id: string }) => relatedEntity.id);
+      } else {
+        return stateEntity[propertyName]?.id;
+      }
+    }
+  }
+
+  return stateEntity[propertyName];
+};

packages/graph-node/src/watcher.ts

@@ -11,7 +11,7 @@ import { ContractInterface, utils, providers } from 'ethers';
 
 import { ResultObject } from '@vulcanize/assemblyscript/lib/loader';
 import { EthClient } from '@cerc-io/ipld-eth-client';
-import { IndexerInterface, getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions } from '@cerc-io/util';
+import { getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions, IPLDBlockInterface, IPLDIndexerInterface } from '@cerc-io/util';
 
 import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils';
 import { Context, GraphData, instantiate } from './loader';
@@ -27,7 +27,7 @@ interface DataSource {
 
 export class GraphWatcher {
   _database: Database;
-  _indexer?: IndexerInterface;
+  _indexer?: IPLDIndexerInterface;
   _ethClient: EthClient;
   _ethProvider: providers.BaseProvider;
   _subgraphPath: string;
@@ -253,7 +253,7 @@ export class GraphWatcher {
     }
   }
 
-  setIndexer (indexer: IndexerInterface): void {
+  setIndexer (indexer: IPLDIndexerInterface): void {
     this._indexer = indexer;
   }
 
@@ -326,6 +326,30 @@ export class GraphWatcher {
     }
   }
 
+  async updateEntitiesFromIPLDState (ipldBlock: IPLDBlockInterface) {
+    assert(this._indexer);
+    const data = this._indexer.getIPLDData(ipldBlock);
+
+    for (const [entityName, entities] of Object.entries(data.state)) {
+      // Get relations for subgraph entity
+      assert(this._indexer.getRelationsMap);
+      const relationsMap = this._indexer.getRelationsMap();
+
+      const result = Array.from(relationsMap.entries())
+        .find(([key]) => key.name === entityName);
+
+      const relations = result ? result[1] : {};
+
+      log(`Updating entities from IPLD state for entity ${entityName}`);
+      console.time(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`);
+      for (const [id, entityData] of Object.entries(entities as any)) {
+        const dbData = this._database.fromIPLDState(ipldBlock.block, entityName, entityData, relations);
+        await this._database.saveEntity(entityName, dbData);
+      }
+      console.timeEnd(`time:watcher#GraphWatcher-updateEntitiesFromIPLDState-IPLD-update-entity-${entityName}`);
+    }
+  }
+
   /**
    * Method to reinstantiate WASM instance for specified dataSource.
    * @param dataSourceName