Add flag to load relations sequentially or concurrently (#225)

* Add flag to load relations sequentially or concurrently * Fix database init in graph-node test * Fix graph-node watcher query method and add changes to codegen
2025-07-30 19:52:07 +00:00 · 2022-11-14 14:23:46 +05:30 · 2022-11-14 14:23:46 +05:30 · 6f8ededd52
commit 6f8ededd52
parent 13edff143b
39 changed files with 178 additions and 148 deletions
--- a/packages/codegen/src/templates/checkpoint-create-template.handlebars
+++ b/packages/codegen/src/templates/checkpoint-create-template.handlebars
@ -41,7 +41,7 @@ export const handler = async (argv: any): Promise<void> => {

  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/checkpoint-verify-template.handlebars
+++ b/packages/codegen/src/templates/checkpoint-verify-template.handlebars
@ -34,7 +34,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/export-state-template.handlebars
+++ b/packages/codegen/src/templates/export-state-template.handlebars
@ -59,7 +59,7 @@ const main = async (): Promise<void> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/fill-template.handlebars
+++ b/packages/codegen/src/templates/fill-template.handlebars
@ -72,7 +72,7 @@ export const main = async (): Promise<any> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/import-state-template.handlebars
+++ b/packages/codegen/src/templates/import-state-template.handlebars
@ -50,7 +50,7 @@ export const main = async (): Promise<any> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/index-block-template.handlebars
+++ b/packages/codegen/src/templates/index-block-template.handlebars
@ -44,7 +44,7 @@ const main = async (): Promise<void> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/inspect-cid-template.handlebars
+++ b/packages/codegen/src/templates/inspect-cid-template.handlebars
@ -45,7 +45,7 @@ const main = async (): Promise<void> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/job-runner-template.handlebars
+++ b/packages/codegen/src/templates/job-runner-template.handlebars
@ -97,7 +97,7 @@ export const main = async (): Promise<any> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/reset-watcher-template.handlebars
+++ b/packages/codegen/src/templates/reset-watcher-template.handlebars
@ -41,7 +41,7 @@ export const handler = async (argv: any): Promise<void> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/server-template.handlebars
+++ b/packages/codegen/src/templates/server-template.handlebars
@ -46,7 +46,7 @@ export const main = async (): Promise<any> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/codegen/src/templates/watch-contract-template.handlebars
+++ b/packages/codegen/src/templates/watch-contract-template.handlebars
@ -61,7 +61,7 @@ const main = async (): Promise<void> => {
  await db.init();
  {{#if (subgraphPath)}}

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/checkpoint-cmds/create.ts
+++ b/packages/eden-watcher/src/cli/checkpoint-cmds/create.ts
@ -37,7 +37,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts
+++ b/packages/eden-watcher/src/cli/checkpoint-cmds/verify.ts
@ -33,7 +33,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/export-state.ts
+++ b/packages/eden-watcher/src/cli/export-state.ts
@ -47,7 +47,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/import-state.ts
+++ b/packages/eden-watcher/src/cli/import-state.ts
@ -47,7 +47,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/index-block.ts
+++ b/packages/eden-watcher/src/cli/index-block.ts
@ -41,7 +41,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/inspect-cid.ts
+++ b/packages/eden-watcher/src/cli/inspect-cid.ts
@ -42,7 +42,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/reset-cmds/watcher.ts
+++ b/packages/eden-watcher/src/cli/reset-cmds/watcher.ts
@ -32,7 +32,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/cli/watch-contract.ts
+++ b/packages/eden-watcher/src/cli/watch-contract.ts
@ -58,7 +58,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/fill.ts
+++ b/packages/eden-watcher/src/fill.ts
@ -65,7 +65,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/job-runner.ts
+++ b/packages/eden-watcher/src/job-runner.ts
@ -94,7 +94,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/eden-watcher/src/server.ts
+++ b/packages/eden-watcher/src/server.ts
@ -43,7 +43,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-node/src/cli/compare/compare-blocks.ts
+++ b/packages/graph-node/src/cli/compare/compare-blocks.ts
@ -127,7 +127,7 @@ export const main = async (): Promise<void> => {
    const baseDatabase = new BaseDatabase({ ...watcherConfig.database, entities: [entitiesDir] });
    await baseDatabase.init();

-    db = new Database(baseDatabase);
+    db = new Database(watcherConfig.server, baseDatabase);
    await db.init();

    if (config.watcher.verifyState) {
--- a/packages/graph-node/src/database.ts
+++ b/packages/graph-node/src/database.ts
@ -28,6 +28,7 @@ import {
  eventProcessingLoadEntityCount,
  eventProcessingLoadEntityDBQueryDuration,
  QueryOptions,
+  ServerConfig,
  Where
 } from '@cerc-io/util';

@ -61,6 +62,7 @@ interface CachedEntities {
 }

 export class Database {
+  _serverConfig: ServerConfig
  _conn!: Connection
  _baseDatabase: BaseDatabase
  _entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE>
@ -70,7 +72,8 @@ export class Database {
    latestPrunedEntities: new Map()
  }

-  constructor (baseDatabase: BaseDatabase, entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE> = new Map()) {
+  constructor (serverConfig: ServerConfig, baseDatabase: BaseDatabase, entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE> = new Map()) {
+    this._serverConfig = serverConfig;
    this._baseDatabase = baseDatabase;
    this._entityQueryTypeMap = entityQueryTypeMap;
  }
@ -815,7 +818,29 @@ export class Database {

    const relationSelections = selections.filter((selection) => selection.kind === 'Field' && Boolean(relations[selection.name.value]));

+    if (this._serverConfig.loadRelationsSequential) {
      for (const selection of relationSelections) {
+        await this.loadRelation(queryRunner, block, relationsMap, relations, entities, selection);
+      }
+    } else {
+      const loadRelationPromises = relationSelections.map(async selection => {
+        await this.loadRelation(queryRunner, block, relationsMap, relations, entities, selection);
+      });
+
+      await Promise.all(loadRelationPromises);
+    }
+
+    return entities;
+  }
+
+  async loadRelation<Entity> (
+    queryRunner: QueryRunner,
+    block: BlockHeight,
+    relationsMap: Map<any, { [key: string]: any }>,
+    relations: { [key: string]: any },
+    entities: Entity[],
+    selection: SelectionNode
+  ): Promise<void> {
    assert(selection.kind === 'Field');
    const field = selection.name.value;
    const { entity: relationEntity, isArray, isDerived, field: foreignKey } = relations[field];
@ -866,7 +891,7 @@ export class Database {
        }
      });

-        continue;
+      return;
    }

    if (isArray) {
@ -910,7 +935,7 @@ export class Database {
        });
      });

-        continue;
+      return;
    }

    // field is neither an array nor derivedFrom
@ -921,7 +946,7 @@ export class Database {
        entity[field] = { id: entity[field] };
      });

-        continue;
+      return;
    }

    const where: Where = {
@ -955,9 +980,6 @@ export class Database {
    });
  }

-    return entities;
-  }
-
  async saveEntity (entity: string, data: any): Promise<any> {
    const repo = this._conn.getRepository(entity);

--- a/packages/graph-node/src/watcher.ts
+++ b/packages/graph-node/src/watcher.ts
@ -338,6 +338,8 @@ export class GraphWatcher {
      // Get entities from the database.
      const entities = await this._database.getEntities(dbTx, entity, relationsMap, block, where, queryOptions, selections);
      await dbTx.commitTransaction();
+
+      return entities;
    } catch (error) {
      await dbTx.rollbackTransaction();
      throw error;
--- a/packages/graph-node/test/utils/index.ts
+++ b/packages/graph-node/test/utils/index.ts
@ -9,7 +9,7 @@ import { StorageLayout } from '@cerc-io/solidity-mapper';

 import { EventData } from '../../src/utils';
 import { Database } from '../../src/database';
-import { Indexer } from './indexer';
+import { Indexer, ServerConfig } from './indexer';

 const NETWORK_URL = 'http://127.0.0.1:8081';
 const IPLD_ETH_SERVER_GQL_URL = 'http://127.0.0.1:8082/graphql';
@ -72,7 +72,8 @@ export const getDummyGraphData = (): any => {

 export const getTestDatabase = (): Database => {
  const baseDatabase = new BaseDatabase({ type: 'postgres' });
-  return new Database(baseDatabase);
+  const serverConfig = new ServerConfig();
+  return new Database(serverConfig, baseDatabase);
 };

 export const getTestIndexer = (storageLayout?: Map<string, StorageLayout>): Indexer => {
--- a/packages/graph-node/test/utils/indexer.ts
+++ b/packages/graph-node/test/utils/indexer.ts
@ -203,7 +203,7 @@ export class Indexer implements IndexerInterface {
  }
 }

-class ServerConfig implements ServerConfigInterface {
+export class ServerConfig implements ServerConfigInterface {
  host: string;
  port: number;
  mode: string;
@ -217,6 +217,7 @@ class ServerConfig implements ServerConfigInterface {
  maxEventsBlockRange: number;
  clearEntitiesCacheInterval: number;
  skipStateFieldsUpdate: boolean;
+  loadRelationsSequential: boolean;

  constructor () {
    this.host = '';
@ -232,5 +233,6 @@ class ServerConfig implements ServerConfigInterface {
    this.maxEventsBlockRange = 0;
    this.clearEntitiesCacheInterval = 0;
    this.skipStateFieldsUpdate = false;
+    this.loadRelationsSequential = false;
  }
 }
--- a/packages/graph-test-watcher/src/cli/checkpoint-cmds/create.ts
+++ b/packages/graph-test-watcher/src/cli/checkpoint-cmds/create.ts
@ -37,7 +37,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts
+++ b/packages/graph-test-watcher/src/cli/checkpoint-cmds/verify.ts
@ -33,7 +33,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/export-state.ts
+++ b/packages/graph-test-watcher/src/cli/export-state.ts
@ -47,7 +47,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/import-state.ts
+++ b/packages/graph-test-watcher/src/cli/import-state.ts
@ -47,7 +47,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/index-block.ts
+++ b/packages/graph-test-watcher/src/cli/index-block.ts
@ -41,7 +41,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/inspect-cid.ts
+++ b/packages/graph-test-watcher/src/cli/inspect-cid.ts
@ -42,7 +42,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts
+++ b/packages/graph-test-watcher/src/cli/reset-cmds/watcher.ts
@ -32,7 +32,7 @@ export const handler = async (argv: any): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/cli/watch-contract.ts
+++ b/packages/graph-test-watcher/src/cli/watch-contract.ts
@ -58,7 +58,7 @@ const main = async (): Promise<void> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/fill.ts
+++ b/packages/graph-test-watcher/src/fill.ts
@ -59,7 +59,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/job-runner.ts
+++ b/packages/graph-test-watcher/src/job-runner.ts
@ -94,7 +94,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/graph-test-watcher/src/server.ts
+++ b/packages/graph-test-watcher/src/server.ts
@ -43,7 +43,7 @@ export const main = async (): Promise<any> => {
  const db = new Database(config.database);
  await db.init();

-  const graphDb = new GraphDatabase(db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
  await graphDb.init();

  const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
--- a/packages/util/src/config.ts
+++ b/packages/util/src/config.ts
@ -51,6 +51,9 @@ export interface ServerConfig {

  // Max GQL API requests in queue until reject (defaults to -1, means do not reject).
  maxRequestQueueLimit?: number;
+
+  // Boolean to load GQL query nested entity relations sequentially.
+  loadRelationsSequential: boolean;
 }

 export interface UpstreamConfig {