Add flag to load relations sequentially or concurrently (#225)

* Add flag to load relations sequentially or concurrently

* Fix database init in graph-node test

* Fix graph-node watcher query method and add changes to codegen

packages/codegen/src/templates/checkpoint-create-template.handlebars

@@ -41,7 +41,7 @@ export const handler = async (argv: any): Promise<void> => {
 
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/checkpoint-verify-template.handlebars

@@ -34,7 +34,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/export-state-template.handlebars

@@ -59,7 +59,7 @@ const main = async (): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/fill-template.handlebars

@@ -72,7 +72,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/import-state-template.handlebars

@@ -50,7 +50,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/index-block-template.handlebars

@@ -44,7 +44,7 @@ const main = async (): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/inspect-cid-template.handlebars

@@ -45,7 +45,7 @@ const main = async (): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/job-runner-template.handlebars

@@ -97,7 +97,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/reset-watcher-template.handlebars

@@ -41,7 +41,7 @@ export const handler = async (argv: any): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/server-template.handlebars

@@ -46,7 +46,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/watch-contract-template.handlebars

@@ -61,7 +61,7 @@ const main = async (): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/checkpoint-cmds/create.ts

@@ -37,7 +37,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts

@@ -33,7 +33,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/export-state.ts

@@ -47,7 +47,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/import-state.ts

@@ -47,7 +47,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/index-block.ts

@@ -41,7 +41,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/inspect-cid.ts

@@ -42,7 +42,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/reset-cmds/watcher.ts

@@ -32,7 +32,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/cli/watch-contract.ts

@@ -58,7 +58,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/fill.ts

@@ -65,7 +65,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/job-runner.ts

@@ -94,7 +94,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/eden-watcher/src/server.ts

@@ -43,7 +43,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-node/src/cli/compare/compare-blocks.ts

@@ -127,7 +127,7 @@ export const main = async (): Promise<void> => {
     const baseDatabase = new BaseDatabase({ ...watcherConfig.database, entities: [entitiesDir] });
     await baseDatabase.init();
 
-    db = new Database(baseDatabase);
+    db = new Database(watcherConfig.server, baseDatabase);
     await db.init();
 
     if (config.watcher.verifyState) {

packages/graph-node/src/database.ts

@@ -28,6 +28,7 @@ import {
   eventProcessingLoadEntityCount,
   eventProcessingLoadEntityDBQueryDuration,
   QueryOptions,
+  ServerConfig,
   Where
 } from '@cerc-io/util';
 
@@ -61,6 +62,7 @@ interface CachedEntities {
 }
 
 export class Database {
+  _serverConfig: ServerConfig
   _conn!: Connection
   _baseDatabase: BaseDatabase
   _entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE>
@@ -70,7 +72,8 @@ export class Database {
     latestPrunedEntities: new Map()
   }
 
-  constructor (baseDatabase: BaseDatabase, entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE> = new Map()) {
+  constructor (serverConfig: ServerConfig, baseDatabase: BaseDatabase, entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE> = new Map()) {
+    this._serverConfig = serverConfig;
     this._baseDatabase = baseDatabase;
     this._entityQueryTypeMap = entityQueryTypeMap;
   }
@@ -815,7 +818,29 @@ export class Database {
 
     const relationSelections = selections.filter((selection) => selection.kind === 'Field' && Boolean(relations[selection.name.value]));
 
+    if (this._serverConfig.loadRelationsSequential) {
       for (const selection of relationSelections) {
+        await this.loadRelation(queryRunner, block, relationsMap, relations, entities, selection);
+      }
+    } else {
+      const loadRelationPromises = relationSelections.map(async selection => {
+        await this.loadRelation(queryRunner, block, relationsMap, relations, entities, selection);
+      });
+
+      await Promise.all(loadRelationPromises);
+    }
+
+    return entities;
+  }
+
+  async loadRelation<Entity> (
+    queryRunner: QueryRunner,
+    block: BlockHeight,
+    relationsMap: Map<any, { [key: string]: any }>,
+    relations: { [key: string]: any },
+    entities: Entity[],
+    selection: SelectionNode
+  ): Promise<void> {
     assert(selection.kind === 'Field');
     const field = selection.name.value;
     const { entity: relationEntity, isArray, isDerived, field: foreignKey } = relations[field];
@@ -866,7 +891,7 @@ export class Database {
         }
       });
 
-        continue;
+      return;
     }
 
     if (isArray) {
@@ -910,7 +935,7 @@ export class Database {
         });
       });
 
-        continue;
+      return;
     }
 
     // field is neither an array nor derivedFrom
@@ -921,7 +946,7 @@ export class Database {
         entity[field] = { id: entity[field] };
       });
 
-        continue;
+      return;
     }
 
     const where: Where = {
@@ -955,9 +980,6 @@ export class Database {
     });
   }
 
-    return entities;
-  }
-
   async saveEntity (entity: string, data: any): Promise<any> {
     const repo = this._conn.getRepository(entity);
 

packages/graph-node/src/watcher.ts

@@ -338,6 +338,8 @@ export class GraphWatcher {
       // Get entities from the database.
       const entities = await this._database.getEntities(dbTx, entity, relationsMap, block, where, queryOptions, selections);
       await dbTx.commitTransaction();
+
+      return entities;
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/graph-node/test/utils/index.ts

@@ -9,7 +9,7 @@ import { StorageLayout } from '@cerc-io/solidity-mapper';
 
 import { EventData } from '../../src/utils';
 import { Database } from '../../src/database';
-import { Indexer } from './indexer';
+import { Indexer, ServerConfig } from './indexer';
 
 const NETWORK_URL = 'http://127.0.0.1:8081';
 const IPLD_ETH_SERVER_GQL_URL = 'http://127.0.0.1:8082/graphql';
@@ -72,7 +72,8 @@ export const getDummyGraphData = (): any => {
 
 export const getTestDatabase = (): Database => {
   const baseDatabase = new BaseDatabase({ type: 'postgres' });
-  return new Database(baseDatabase);
+  const serverConfig = new ServerConfig();
+  return new Database(serverConfig, baseDatabase);
 };
 
 export const getTestIndexer = (storageLayout?: Map<string, StorageLayout>): Indexer => {

packages/graph-node/test/utils/indexer.ts

@@ -203,7 +203,7 @@ export class Indexer implements IndexerInterface {
   }
 }
 
-class ServerConfig implements ServerConfigInterface {
+export class ServerConfig implements ServerConfigInterface {
   host: string;
   port: number;
   mode: string;
@@ -217,6 +217,7 @@ class ServerConfig implements ServerConfigInterface {
   maxEventsBlockRange: number;
   clearEntitiesCacheInterval: number;
   skipStateFieldsUpdate: boolean;
+  loadRelationsSequential: boolean;
 
   constructor () {
     this.host = '';
@@ -232,5 +233,6 @@ class ServerConfig implements ServerConfigInterface {
     this.maxEventsBlockRange = 0;
     this.clearEntitiesCacheInterval = 0;
     this.skipStateFieldsUpdate = false;
+    this.loadRelationsSequential = false;
   }
 }

packages/graph-test-watcher/src/cli/checkpoint-cmds/create.ts

@@ -37,7 +37,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts

@@ -33,7 +33,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/export-state.ts

@@ -47,7 +47,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/import-state.ts

@@ -47,7 +47,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/index-block.ts

@@ -41,7 +41,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/inspect-cid.ts

@@ -42,7 +42,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts

@@ -32,7 +32,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/cli/watch-contract.ts

@@ -58,7 +58,7 @@ const main = async (): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/fill.ts

@@ -59,7 +59,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/job-runner.ts

@@ -94,7 +94,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/graph-test-watcher/src/server.ts

@@ -43,7 +43,7 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/util/src/config.ts

@@ -51,6 +51,9 @@ export interface ServerConfig {
 
   // Max GQL API requests in queue until reject (defaults to -1, means do not reject).
   maxRequestQueueLimit?: number;
+
+  // Boolean to load GQL query nested entity relations sequentially.
+  loadRelationsSequential: boolean;
 }
 
 export interface UpstreamConfig {