From a084b4e40c298a62f18f6244a973dfed20ab1baf Mon Sep 17 00:00:00 2001 From: nikugogoi Date: Wed, 16 Nov 2022 18:11:27 +0530 Subject: [PATCH] Latest entity queries to optimize frontend app GQL requests (#232) * Add latest entity table query pattern * Add subscriber config to typeorm --- packages/eden-watcher/src/database.ts | 3 +- .../eden-watcher/src/entity/Subscriber.ts | 3 +- packages/graph-node/src/database.ts | 113 +++++++++++++----- packages/graph-node/src/utils.ts | 35 +++++- packages/util/src/database.ts | 24 +++- 5 files changed, 140 insertions(+), 38 deletions(-) diff --git a/packages/eden-watcher/src/database.ts b/packages/eden-watcher/src/database.ts index 814898a0..8f2542a7 100644 --- a/packages/eden-watcher/src/database.ts +++ b/packages/eden-watcher/src/database.ts @@ -44,7 +44,8 @@ export class Database implements DatabaseInterface { this._config = { ...config, - entities: [path.join(__dirname, 'entity/*')] + entities: [path.join(__dirname, 'entity/*')], + subscribers: [path.join(__dirname, 'entity/Subscriber.*')] }; this._baseDatabase = new BaseDatabase(this._config); diff --git a/packages/eden-watcher/src/entity/Subscriber.ts b/packages/eden-watcher/src/entity/Subscriber.ts index fc1127d5..93c77fe4 100644 --- a/packages/eden-watcher/src/entity/Subscriber.ts +++ b/packages/eden-watcher/src/entity/Subscriber.ts @@ -4,9 +4,10 @@ import { EventSubscriber, EntitySubscriberInterface, InsertEvent, UpdateEvent } from 'typeorm'; +import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node'; + import { FrothyEntity } from './FrothyEntity'; import { ENTITIES } from '../database'; -import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node'; @EventSubscriber() export class EntitySubscriber implements EntitySubscriberInterface { diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index 16ed8d99..3ccbb4ff 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -68,16 +68,23 @@ export class Database { _conn!: Connection _baseDatabase: BaseDatabase _entityQueryTypeMap: Map any, ENTITY_QUERY_TYPE> + _entityToLatestEntityMap: Map any, new () => any> = new Map() _cachedEntities: CachedEntities = { frothyBlocks: new Map(), latestPrunedEntities: new Map() } - constructor (serverConfig: ServerConfig, baseDatabase: BaseDatabase, entityQueryTypeMap: Map any, ENTITY_QUERY_TYPE> = new Map()) { + constructor ( + serverConfig: ServerConfig, + baseDatabase: BaseDatabase, + entityQueryTypeMap: Map any, ENTITY_QUERY_TYPE> = new Map(), + entityToLatestEntityMap: Map any, new () => any> = new Map() + ) { this._serverConfig = serverConfig; this._baseDatabase = baseDatabase; this._entityQueryTypeMap = entityQueryTypeMap; + this._entityToLatestEntityMap = entityToLatestEntityMap; } get cachedEntities () { @@ -367,40 +374,54 @@ export class Database { queryOptions: QueryOptions = {}, selections: ReadonlyArray = [] ): Promise { - let entities: Entity[]; + let entities: Entity[] = []; + const latestEntity = this._entityToLatestEntityMap.get(entity); - // Use different suitable query patterns based on entities. - switch (this._entityQueryTypeMap.get(entity)) { - case ENTITY_QUERY_TYPE.SINGULAR: - entities = await this.getEntitiesSingular(queryRunner, entity, block, where); - break; + if (latestEntity) { + if (!Object.keys(block).length) { + // Use latest entity tables if block height not passed. + entities = await this.getEntitiesLatest( + queryRunner, + entity, + latestEntity, + where, + queryOptions + ); + } + } else { + // Use different suitable query patterns based on entities. + switch (this._entityQueryTypeMap.get(entity)) { + case ENTITY_QUERY_TYPE.SINGULAR: + entities = await this.getEntitiesSingular(queryRunner, entity, block, where); + break; - case ENTITY_QUERY_TYPE.UNIQUE: - entities = await this.getEntitiesUnique(queryRunner, entity, block, where, queryOptions); - break; + case ENTITY_QUERY_TYPE.UNIQUE: + entities = await this.getEntitiesUnique(queryRunner, entity, block, where, queryOptions); + break; - case ENTITY_QUERY_TYPE.UNIQUE_WITHOUT_PRUNED: - entities = await this.getEntitiesUniqueWithoutPruned(queryRunner, entity, block, where, queryOptions); - break; + case ENTITY_QUERY_TYPE.UNIQUE_WITHOUT_PRUNED: + entities = await this.getEntitiesUniqueWithoutPruned(queryRunner, entity, block, where, queryOptions); + break; - case ENTITY_QUERY_TYPE.DISTINCT_ON: - entities = await this.getEntitiesDistinctOn(queryRunner, entity, block, where, queryOptions); - break; + case ENTITY_QUERY_TYPE.DISTINCT_ON: + entities = await this.getEntitiesDistinctOn(queryRunner, entity, block, where, queryOptions); + break; - case ENTITY_QUERY_TYPE.DISTINCT_ON_WITHOUT_PRUNED: - entities = await this.getEntitiesDistinctOnWithoutPruned(queryRunner, entity, block, where, queryOptions); - break; + case ENTITY_QUERY_TYPE.DISTINCT_ON_WITHOUT_PRUNED: + entities = await this.getEntitiesDistinctOnWithoutPruned(queryRunner, entity, block, where, queryOptions); + break; - case ENTITY_QUERY_TYPE.GROUP_BY_WITHOUT_PRUNED: - // Use group by query if entity query type is not specified in map. - entities = await this.getEntitiesGroupByWithoutPruned(queryRunner, entity, block, where, queryOptions); - break; + case ENTITY_QUERY_TYPE.GROUP_BY_WITHOUT_PRUNED: + // Use group by query if entity query type is not specified in map. + entities = await this.getEntitiesGroupByWithoutPruned(queryRunner, entity, block, where, queryOptions); + break; - case ENTITY_QUERY_TYPE.GROUP_BY: - default: - // Use group by query if entity query type is not specified in map. - entities = await this.getEntitiesGroupBy(queryRunner, entity, block, where, queryOptions); - break; + case ENTITY_QUERY_TYPE.GROUP_BY: + default: + // Use group by query if entity query type is not specified in map. + entities = await this.getEntitiesGroupBy(queryRunner, entity, block, where, queryOptions); + break; + } } if (!entities.length) { @@ -805,6 +826,42 @@ export class Database { return entities as Entity[]; } + async getEntitiesLatest ( + queryRunner: QueryRunner, + entity: new () => Entity, + latestEntity: new () => any, + where: Where = {}, + queryOptions: QueryOptions = {} + ): Promise { + const repo = queryRunner.manager.getRepository(entity); + const { tableName } = repo.metadata; + + let selectQueryBuilder = repo.createQueryBuilder(tableName) + .innerJoin( + latestEntity, + 'latest', + `latest.id = ${tableName}.id AND latest.blockHash = ${tableName}.blockHash` + ); + + selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where, 'latest'); + + if (queryOptions.orderBy) { + selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, '', 'latest'); + } + + selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, '', 'latest'); + + if (queryOptions.skip) { + selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip); + } + + if (queryOptions.limit) { + selectQueryBuilder = selectQueryBuilder.limit(queryOptions.limit); + } + + return selectQueryBuilder.getMany(); + } + async loadEntitiesRelations ( queryRunner: QueryRunner, block: BlockHeight, diff --git a/packages/graph-node/src/utils.ts b/packages/graph-node/src/utils.ts index 52434c3a..8c04bc61 100644 --- a/packages/graph-node/src/utils.ts +++ b/packages/graph-node/src/utils.ts @@ -899,10 +899,18 @@ export const updateEntitiesFromState = async (database: Database, indexer: Index } }; -export const afterEntityInsertOrUpdate = async (frothyEntityType: EntityTarget, entities: Set, event: InsertEvent | UpdateEvent): Promise => { +export const afterEntityInsertOrUpdate = async ( + frothyEntityType: EntityTarget, + entities: Set, + event: InsertEvent | UpdateEvent, + entityToLatestEntityMap: Map any, new () => any> = new Map() +): Promise => { const entity = event.entity; - // TODO: Check and return if entity is being pruned (is_pruned flag update) + // Return if the entity is being pruned + if (entity.isPruned) { + return; + } // Insert the entity details in FrothyEntity table if (entities.has(entity.constructor)) { @@ -922,5 +930,26 @@ export const afterEntityInsertOrUpdate = async (frothyEntityType: Entity .execute(); } - // TOOD: Update latest entity tables + // Get latest entity's type + const entityTarget = entityToLatestEntityMap.get(entity.constructor); + + if (!entityTarget) { + return; + } + + // Get latest entity's fields to be updated + const latestEntityRepo = event.manager.getRepository(entityTarget); + const latestEntityFields = latestEntityRepo.metadata.columns.map(column => column.propertyName); + const fieldsToUpdate = latestEntityRepo.metadata.columns.map(column => column.databaseName).filter(val => val !== 'id'); + + // Create a latest entity instance and upsert in the db + const latestEntity = event.manager.create(entityTarget, _.pick(entity, latestEntityFields)); + await event.manager.createQueryBuilder() + .insert() + .into(entityTarget) + .values(latestEntity) + .orUpdate( + { conflict_target: ['id'], overwrite: fieldsToUpdate } + ) + .execute(); }; diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 863e5b60..cfce1392 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -788,20 +788,29 @@ export class Database { return repo.save(entity); } - buildQuery (repo: Repository, selectQueryBuilder: SelectQueryBuilder, where: Where = {}): SelectQueryBuilder { + buildQuery ( + repo: Repository, + selectQueryBuilder: SelectQueryBuilder, + where: Where = {}, + alias?: string + ): SelectQueryBuilder { + if (!alias) { + alias = selectQueryBuilder.alias; + } + Object.entries(where).forEach(([field, filters]) => { filters.forEach((filter, index) => { // Form the where clause. let { not, operator, value } = filter; const columnMetadata = repo.metadata.findColumnWithPropertyName(field); assert(columnMetadata); - let whereClause = `"${selectQueryBuilder.alias}"."${columnMetadata.databaseName}" `; + let whereClause = `"${alias}"."${columnMetadata.databaseName}" `; if (columnMetadata.relationMetadata) { // For relation fields, use the id column. const idColumn = columnMetadata.relationMetadata.joinColumns.find(column => column.referencedColumn?.propertyName === 'id'); assert(idColumn); - whereClause = `"${selectQueryBuilder.alias}"."${idColumn.databaseName}" `; + whereClause = `"${alias}"."${idColumn.databaseName}" `; } if (not) { @@ -853,8 +862,13 @@ export class Database { repo: Repository, selectQueryBuilder: SelectQueryBuilder, orderOptions: { orderBy?: string, orderDirection?: string }, - columnPrefix = '' + columnPrefix = '', + alias?: string ): SelectQueryBuilder { + if (!alias) { + alias = selectQueryBuilder.alias; + } + const { orderBy, orderDirection } = orderOptions; assert(orderBy); @@ -862,7 +876,7 @@ export class Database { assert(columnMetadata); return selectQueryBuilder.addOrderBy( - `"${selectQueryBuilder.alias}"."${columnPrefix}${columnMetadata.databaseName}"`, + `"${alias}"."${columnPrefix}${columnMetadata.databaseName}"`, orderDirection === 'desc' ? 'DESC' : 'ASC' ); }