Add a CLI in eden-watcher to fill state for a given range (#176)

* Add a CLI to fill state for a given range

* Refactor code

* Add a CLI to reset IPLD state

* Replace ORDER BY clause in the query to get latest IPLD block

* Optimize delete query in CLI to reset IPLD state

* Add an option to decouple subgraph state creation from mapping code

* Use a raw SQL query to delete IPLD blocks in a block range

* Accomodate changes in codegen

packages/graph-node/src/database.ts

@@ -82,6 +82,18 @@ export class Database {
     }
   }
 
+  async getEntitiesForBlock (blockHash: string, tableName: string): Promise<any[]> {
+    const repo = this._conn.getRepository(tableName);
+
+    const entities = await repo.find({
+      where: {
+        blockHash
+      }
+    });
+
+    return entities;
+  }
+
   async getEntityIdsAtBlockNumber (blockNumber: number, tableName: string): Promise<string[]> {
     const repo = this._conn.getRepository(tableName);
 

packages/graph-node/src/index.ts

@@ -1,2 +1,3 @@
 export * from './watcher';
 export * from './database';
+export { prepareEntityState } from './utils';

packages/graph-node/src/loader.ts

@@ -22,10 +22,10 @@ import {
   Block,
   fromEthereumValue,
   toEthereumValue,
-  resolveEntityFieldConflicts,
   getEthereumTypes,
   jsonFromBytes,
-  getStorageValueType
+  getStorageValueType,
+  prepareEntityState
 } from './utils';
 import { Database } from './database';
 
@@ -94,53 +94,19 @@ export const instantiate = async (
         const entityInstance = await Entity.wrap(data);
 
         assert(context.block);
-        let dbData = await database.fromGraphEntity(instanceExports, context.block, entityName, entityInstance);
+        const dbData = await database.fromGraphEntity(instanceExports, context.block, entityName, entityInstance);
         await database.saveEntity(entityName, dbData);
 
-        // Resolve any field name conflicts in the dbData for auto-diff.
-        dbData = resolveEntityFieldConflicts(dbData);
+        // Update the in-memory subgraph state if not disabled.
+        if (!indexer.serverConfig.disableSubgraphState) {
+          // Prepare diff data for the entity update
+          assert(indexer.getRelationsMap);
+          const diffData = prepareEntityState(dbData, entityName, indexer.getRelationsMap());
 
-        // Prepare the diff data.
-        const diffData: any = { state: {} };
-        assert(indexer.getRelationsMap);
-
-        const result = Array.from(indexer.getRelationsMap().entries())
-          .find(([key]) => key.name === entityName);
-
-        if (result) {
-          // Update dbData if relations exist.
-          const [_, relations] = result;
-
-          // Update relation fields for diff data to be similar to GQL query entities.
-          Object.entries(relations).forEach(([relation, { isArray, isDerived }]) => {
-            if (isDerived || !dbData[relation]) {
-              // Field is not present in dbData for derived relations
-              return;
-            }
-
-            if (isArray) {
-              dbData[relation] = dbData[relation]
-                .map((id: string) => ({ id }))
-                .sort((a: any, b: any) => a.id.localeCompare(b.id));
-            } else {
-              dbData[relation] = { id: dbData[relation] };
-            }
-          });
+          assert(indexer.updateSubgraphState);
+          assert(context.contractAddress);
+          indexer.updateSubgraphState(context.contractAddress, diffData);
         }
-
-        // JSON stringify and parse data for handling unknown types when encoding.
-        // For example, decimal.js values are converted to string in the diff data.
-        diffData.state[entityName] = {
-          // Using custom replacer to store bigints as string values to be encoded by IPLD dag-cbor.
-          // TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode.
-          // https://github.com/rvagg/cborg#type-encoders
-          [dbData.id]: JSON.parse(JSON.stringify(dbData, jsonBigIntStringReplacer))
-        };
-
-        // Update the in-memory subgraph state.
-        assert(indexer.updateSubgraphState);
-        assert(context.contractAddress);
-        indexer.updateSubgraphState(context.contractAddress, diffData);
       },
 
       'log.log': (level: number, msg: number) => {

packages/graph-node/src/utils.ts

@@ -6,7 +6,7 @@ import yaml from 'js-yaml';
 import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
 import assert from 'assert';
 
-import { GraphDecimal } from '@vulcanize/util';
+import { GraphDecimal, jsonBigIntStringReplacer } from '@vulcanize/util';
 
 import { TypeId, EthereumValueKind, ValueKind } from './types';
 import { MappingKey, StorageLayout } from '@vulcanize/solidity-mapper';
@@ -798,3 +798,46 @@ const getEthereumType = (storageTypes: StorageLayout['types'], type: string, map
 
   return utils.ParamType.from(label);
 };
+
+export const prepareEntityState = (updatedEntity: any, entityName: string, relationsMap: Map<any, { [key: string]: any }>): any => {
+  // Resolve any field name conflicts in the dbData for auto-diff.
+  updatedEntity = resolveEntityFieldConflicts(updatedEntity);
+
+  // Prepare the diff data.
+  const diffData: any = { state: {} };
+
+  const result = Array.from(relationsMap.entries())
+    .find(([key]) => key.name === entityName);
+
+  if (result) {
+    // Update entity data if relations exist.
+    const [_, relations] = result;
+
+    // Update relation fields for diff data to be similar to GQL query entities.
+    Object.entries(relations).forEach(([relation, { isArray, isDerived }]) => {
+      if (isDerived || !updatedEntity[relation]) {
+        // Field is not present in dbData for derived relations
+        return;
+      }
+
+      if (isArray) {
+        updatedEntity[relation] = updatedEntity[relation]
+          .map((id: string) => ({ id }))
+          .sort((a: any, b: any) => a.id.localeCompare(b.id));
+      } else {
+        updatedEntity[relation] = { id: updatedEntity[relation] };
+      }
+    });
+  }
+
+  // JSON stringify and parse data for handling unknown types when encoding.
+  // For example, decimal.js values are converted to string in the diff data.
+  diffData.state[entityName] = {
+    // Using custom replacer to store bigints as string values to be encoded by IPLD dag-cbor.
+    // TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode.
+    // https://github.com/rvagg/cborg#type-encoders
+    [updatedEntity.id]: JSON.parse(JSON.stringify(updatedEntity, jsonBigIntStringReplacer))
+  };
+
+  return diffData;
+};

packages/graph-node/src/watcher.ts

@@ -101,6 +101,10 @@ export class GraphWatcher {
     }, {});
   }
 
+  get dataSources (): any[] {
+    return this._dataSources;
+  }
+
   async addContracts () {
     assert(this._indexer);
     assert(this._indexer.watchContract);