Latest entity queries to optimize frontend app GQL requests (#232)

* Add latest entity table query pattern

* Add subscriber config to typeorm

packages/eden-watcher/src/database.ts

@@ -44,7 +44,8 @@ export class Database implements DatabaseInterface {
 
     this._config = {
       ...config,
-      entities: [path.join(__dirname, 'entity/*')]
+      entities: [path.join(__dirname, 'entity/*')],
+      subscribers: [path.join(__dirname, 'entity/Subscriber.*')]
     };
 
     this._baseDatabase = new BaseDatabase(this._config);

packages/eden-watcher/src/entity/Subscriber.ts

@@ -4,9 +4,10 @@
 
 import { EventSubscriber, EntitySubscriberInterface, InsertEvent, UpdateEvent } from 'typeorm';
 
+import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node';
+
 import { FrothyEntity } from './FrothyEntity';
 import { ENTITIES } from '../database';
-import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node';
 
 @EventSubscriber()
 export class EntitySubscriber implements EntitySubscriberInterface {

packages/graph-node/src/database.ts

@@ -68,16 +68,23 @@ export class Database {
   _conn!: Connection
   _baseDatabase: BaseDatabase
   _entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE>
+  _entityToLatestEntityMap: Map<new () => any, new () => any> = new Map()
 
   _cachedEntities: CachedEntities = {
     frothyBlocks: new Map(),
     latestPrunedEntities: new Map()
   }
 
-  constructor (serverConfig: ServerConfig, baseDatabase: BaseDatabase, entityQueryTypeMap: Map<new() => any, ENTITY_QUERY_TYPE> = new Map()) {
+  constructor (
+    serverConfig: ServerConfig,
+    baseDatabase: BaseDatabase,
+    entityQueryTypeMap: Map<new () => any, ENTITY_QUERY_TYPE> = new Map(),
+    entityToLatestEntityMap: Map<new () => any, new () => any> = new Map()
+  ) {
     this._serverConfig = serverConfig;
     this._baseDatabase = baseDatabase;
     this._entityQueryTypeMap = entityQueryTypeMap;
+    this._entityToLatestEntityMap = entityToLatestEntityMap;
   }
 
   get cachedEntities () {
@@ -367,40 +374,54 @@ export class Database {
     queryOptions: QueryOptions = {},
     selections: ReadonlyArray<SelectionNode> = []
   ): Promise<Entity[]> {
-    let entities: Entity[];
+    let entities: Entity[] = [];
+    const latestEntity = this._entityToLatestEntityMap.get(entity);
 
-    // Use different suitable query patterns based on entities.
-    switch (this._entityQueryTypeMap.get(entity)) {
-      case ENTITY_QUERY_TYPE.SINGULAR:
-        entities = await this.getEntitiesSingular(queryRunner, entity, block, where);
-        break;
+    if (latestEntity) {
+      if (!Object.keys(block).length) {
+        // Use latest entity tables if block height not passed.
+        entities = await this.getEntitiesLatest(
+          queryRunner,
+          entity,
+          latestEntity,
+          where,
+          queryOptions
+        );
+      }
+    } else {
+      // Use different suitable query patterns based on entities.
+      switch (this._entityQueryTypeMap.get(entity)) {
+        case ENTITY_QUERY_TYPE.SINGULAR:
+          entities = await this.getEntitiesSingular(queryRunner, entity, block, where);
+          break;
 
-      case ENTITY_QUERY_TYPE.UNIQUE:
-        entities = await this.getEntitiesUnique(queryRunner, entity, block, where, queryOptions);
-        break;
+        case ENTITY_QUERY_TYPE.UNIQUE:
+          entities = await this.getEntitiesUnique(queryRunner, entity, block, where, queryOptions);
+          break;
 
-      case ENTITY_QUERY_TYPE.UNIQUE_WITHOUT_PRUNED:
-        entities = await this.getEntitiesUniqueWithoutPruned(queryRunner, entity, block, where, queryOptions);
-        break;
+        case ENTITY_QUERY_TYPE.UNIQUE_WITHOUT_PRUNED:
+          entities = await this.getEntitiesUniqueWithoutPruned(queryRunner, entity, block, where, queryOptions);
+          break;
 
-      case ENTITY_QUERY_TYPE.DISTINCT_ON:
-        entities = await this.getEntitiesDistinctOn(queryRunner, entity, block, where, queryOptions);
-        break;
+        case ENTITY_QUERY_TYPE.DISTINCT_ON:
+          entities = await this.getEntitiesDistinctOn(queryRunner, entity, block, where, queryOptions);
+          break;
 
-      case ENTITY_QUERY_TYPE.DISTINCT_ON_WITHOUT_PRUNED:
-        entities = await this.getEntitiesDistinctOnWithoutPruned(queryRunner, entity, block, where, queryOptions);
-        break;
+        case ENTITY_QUERY_TYPE.DISTINCT_ON_WITHOUT_PRUNED:
+          entities = await this.getEntitiesDistinctOnWithoutPruned(queryRunner, entity, block, where, queryOptions);
+          break;
 
-      case ENTITY_QUERY_TYPE.GROUP_BY_WITHOUT_PRUNED:
-        // Use group by query if entity query type is not specified in map.
-        entities = await this.getEntitiesGroupByWithoutPruned(queryRunner, entity, block, where, queryOptions);
-        break;
+        case ENTITY_QUERY_TYPE.GROUP_BY_WITHOUT_PRUNED:
+          // Use group by query if entity query type is not specified in map.
+          entities = await this.getEntitiesGroupByWithoutPruned(queryRunner, entity, block, where, queryOptions);
+          break;
 
-      case ENTITY_QUERY_TYPE.GROUP_BY:
-      default:
-        // Use group by query if entity query type is not specified in map.
-        entities = await this.getEntitiesGroupBy(queryRunner, entity, block, where, queryOptions);
-        break;
+        case ENTITY_QUERY_TYPE.GROUP_BY:
+        default:
+          // Use group by query if entity query type is not specified in map.
+          entities = await this.getEntitiesGroupBy(queryRunner, entity, block, where, queryOptions);
+          break;
+      }
     }
 
     if (!entities.length) {
@@ -805,6 +826,42 @@ export class Database {
     return entities as Entity[];
   }
 
+  async getEntitiesLatest<Entity> (
+    queryRunner: QueryRunner,
+    entity: new () => Entity,
+    latestEntity: new () => any,
+    where: Where = {},
+    queryOptions: QueryOptions = {}
+  ): Promise<Entity[]> {
+    const repo = queryRunner.manager.getRepository(entity);
+    const { tableName } = repo.metadata;
+
+    let selectQueryBuilder = repo.createQueryBuilder(tableName)
+      .innerJoin(
+        latestEntity,
+        'latest',
+        `latest.id = ${tableName}.id AND latest.blockHash = ${tableName}.blockHash`
+      );
+
+    selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where, 'latest');
+
+    if (queryOptions.orderBy) {
+      selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, '', 'latest');
+    }
+
+    selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, '', 'latest');
+
+    if (queryOptions.skip) {
+      selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip);
+    }
+
+    if (queryOptions.limit) {
+      selectQueryBuilder = selectQueryBuilder.limit(queryOptions.limit);
+    }
+
+    return selectQueryBuilder.getMany();
+  }
+
   async loadEntitiesRelations<Entity> (
     queryRunner: QueryRunner,
     block: BlockHeight,

packages/graph-node/src/utils.ts

@@ -899,10 +899,18 @@ export const updateEntitiesFromState = async (database: Database, indexer: Index
   }
 };
 
-export const afterEntityInsertOrUpdate = async<Entity> (frothyEntityType: EntityTarget<Entity>, entities: Set<any>, event: InsertEvent<any> | UpdateEvent<any>): Promise<void> => {
+export const afterEntityInsertOrUpdate = async<Entity> (
+  frothyEntityType: EntityTarget<Entity>,
+  entities: Set<any>,
+  event: InsertEvent<any> | UpdateEvent<any>,
+  entityToLatestEntityMap: Map<new () => any, new () => any> = new Map()
+): Promise<void> => {
   const entity = event.entity;
 
-  // TODO: Check and return if entity is being pruned (is_pruned flag update)
+  // Return if the entity is being pruned
+  if (entity.isPruned) {
+    return;
+  }
 
   // Insert the entity details in FrothyEntity table
   if (entities.has(entity.constructor)) {
@@ -922,5 +930,26 @@ export const afterEntityInsertOrUpdate = async<Entity> (frothyEntityType: Entity
       .execute();
   }
 
-  // TOOD: Update latest entity tables
+  // Get latest entity's type
+  const entityTarget = entityToLatestEntityMap.get(entity.constructor);
+
+  if (!entityTarget) {
+    return;
+  }
+
+  // Get latest entity's fields to be updated
+  const latestEntityRepo = event.manager.getRepository(entityTarget);
+  const latestEntityFields = latestEntityRepo.metadata.columns.map(column => column.propertyName);
+  const fieldsToUpdate = latestEntityRepo.metadata.columns.map(column => column.databaseName).filter(val => val !== 'id');
+
+  // Create a latest entity instance and upsert in the db
+  const latestEntity = event.manager.create(entityTarget, _.pick(entity, latestEntityFields));
+  await event.manager.createQueryBuilder()
+    .insert()
+    .into(entityTarget)
+    .values(latestEntity)
+    .orUpdate(
+      { conflict_target: ['id'], overwrite: fieldsToUpdate }
+    )
+    .execute();
 };

packages/util/src/database.ts

@@ -788,20 +788,29 @@ export class Database {
     return repo.save(entity);
   }
 
-  buildQuery<Entity> (repo: Repository<Entity>, selectQueryBuilder: SelectQueryBuilder<Entity>, where: Where = {}): SelectQueryBuilder<Entity> {
+  buildQuery<Entity> (
+    repo: Repository<Entity>,
+    selectQueryBuilder: SelectQueryBuilder<Entity>,
+    where: Where = {},
+    alias?: string
+  ): SelectQueryBuilder<Entity> {
+    if (!alias) {
+      alias = selectQueryBuilder.alias;
+    }
+
     Object.entries(where).forEach(([field, filters]) => {
       filters.forEach((filter, index) => {
         // Form the where clause.
         let { not, operator, value } = filter;
         const columnMetadata = repo.metadata.findColumnWithPropertyName(field);
         assert(columnMetadata);
-        let whereClause = `"${selectQueryBuilder.alias}"."${columnMetadata.databaseName}" `;
+        let whereClause = `"${alias}"."${columnMetadata.databaseName}" `;
 
         if (columnMetadata.relationMetadata) {
           // For relation fields, use the id column.
           const idColumn = columnMetadata.relationMetadata.joinColumns.find(column => column.referencedColumn?.propertyName === 'id');
           assert(idColumn);
-          whereClause = `"${selectQueryBuilder.alias}"."${idColumn.databaseName}" `;
+          whereClause = `"${alias}"."${idColumn.databaseName}" `;
         }
 
         if (not) {
@@ -853,8 +862,13 @@ export class Database {
     repo: Repository<Entity>,
     selectQueryBuilder: SelectQueryBuilder<Entity>,
     orderOptions: { orderBy?: string, orderDirection?: string },
-    columnPrefix = ''
+    columnPrefix = '',
+    alias?: string
   ): SelectQueryBuilder<Entity> {
+    if (!alias) {
+      alias = selectQueryBuilder.alias;
+    }
+
     const { orderBy, orderDirection } = orderOptions;
     assert(orderBy);
 
@@ -862,7 +876,7 @@ export class Database {
     assert(columnMetadata);
 
     return selectQueryBuilder.addOrderBy(
-      `"${selectQueryBuilder.alias}"."${columnPrefix}${columnMetadata.databaseName}"`,
+      `"${alias}"."${columnPrefix}${columnMetadata.databaseName}"`,
       orderDirection === 'desc' ? 'DESC' : 'ASC'
     );
   }