Load relations for single entity GQL query with default limit 100 (#182)

* Check endBlock greater than chainHeadBlock in fill blocks CLI

* Load relations for single entity with limit 100

* Use single tx query runner for entity GQL query

* Remove assert for CLI input checks
This commit is contained in:
nikugogoi 2022-09-13 17:24:14 +05:30 committed by GitHub
parent 8960f67f1b
commit 5a7dcbd20f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 230 additions and 127 deletions

View File

@ -9,6 +9,7 @@ import {
ConnectionOptions,
FindOneOptions,
LessThanOrEqual,
QueryRunner,
Repository
} from 'typeorm';
@ -49,6 +50,10 @@ export class Database {
return this._baseDatabase.close();
}
async createTransactionRunner (): Promise<QueryRunner> {
return this._baseDatabase.createTransactionRunner();
}
async getEntity<Entity> (entity: (new () => Entity) | string, id: string, blockHash?: string): Promise<Entity | undefined> {
const queryRunner = this._conn.createQueryRunner();
@ -107,13 +112,9 @@ export class Database {
return entities.map((entity: any) => entity.id);
}
async getEntityWithRelations<Entity> (entity: (new () => Entity), id: string, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight = {}): Promise<Entity | undefined> {
const queryRunner = this._conn.createQueryRunner();
async getEntityWithRelations<Entity> (queryRunner: QueryRunner, entity: (new () => Entity), id: string, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight = {}, depth = 1): Promise<Entity | undefined> {
let { hash: blockHash, number: blockNumber } = block;
try {
const repo = queryRunner.manager.getRepository(entity);
const whereOptions: any = { id };
if (blockNumber) {
@ -141,18 +142,94 @@ export class Database {
// Get relational fields
if (entityData) {
[entityData] = await this.loadRelations(block, relationsMap, entity, [entityData], 1);
entityData = await this.loadEntityRelations(queryRunner, block, relationsMap, entity, entityData, depth);
}
return entityData;
} finally {
await queryRunner.release();
}
}
async getEntities<Entity> (entity: new () => Entity, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight, where: Where = {}, queryOptions: QueryOptions = {}, depth = 1): Promise<Entity[]> {
const queryRunner = this._conn.createQueryRunner();
try {
async loadEntityRelations<Entity> (queryRunner: QueryRunner, block: BlockHeight, relationsMap: Map<any, { [key: string]: any }>, entity: new () => Entity, entityData: any, depth: number): Promise<Entity> {
// Only support two-level nesting of relations
if (depth > 2) {
return entityData;
}
const relations = relationsMap.get(entity);
if (relations === undefined) {
return entityData;
}
const relationPromises = Object.entries(relations)
.map(async ([field, data]) => {
const { entity: relationEntity, isArray, isDerived, field: foreignKey } = data;
if (isDerived) {
const where: Where = {
[foreignKey]: [{
value: entityData.id,
not: false,
operator: 'equals'
}]
};
const relatedEntities = await this.getEntities(
queryRunner,
relationEntity,
relationsMap,
block,
where,
{ limit: DEFAULT_LIMIT },
depth + 1
);
entityData[field] = relatedEntities;
return;
}
if (isArray) {
const where: Where = {
id: [{
value: entityData[field],
not: false,
operator: 'in'
}]
};
const relatedEntities = await this.getEntities(
queryRunner,
relationEntity,
relationsMap,
block,
where,
{ limit: DEFAULT_LIMIT },
depth + 1
);
entityData[field] = relatedEntities;
return;
}
// field is neither an array nor derivedFrom
const relatedEntity = await this.getEntityWithRelations(
queryRunner,
relationEntity,
entityData[field],
relationsMap,
block,
depth + 1
);
entityData[field] = relatedEntity;
});
await Promise.all(relationPromises);
return entityData;
}
async getEntities<Entity> (queryRunner: QueryRunner, entity: new () => Entity, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight, where: Where = {}, queryOptions: QueryOptions = {}, depth = 1): Promise<Entity[]> {
const repo = queryRunner.manager.getRepository(entity);
const { tableName } = repo.metadata;
@ -208,13 +285,10 @@ export class Database {
return [];
}
return this.loadRelations(block, relationsMap, entity, entities, depth);
} finally {
await queryRunner.release();
}
return this.loadEntitiesRelations(queryRunner, block, relationsMap, entity, entities, depth);
}
async loadRelations<Entity> (block: BlockHeight, relationsMap: Map<any, { [key: string]: any }>, entity: new () => Entity, entities: Entity[], depth: number): Promise<Entity[]> {
async loadEntitiesRelations<Entity> (queryRunner: QueryRunner, block: BlockHeight, relationsMap: Map<any, { [key: string]: any }>, entity: new () => Entity, entities: Entity[], depth: number): Promise<Entity[]> {
// Only support two-level nesting of relations
if (depth > 2) {
return entities;
@ -238,6 +312,7 @@ export class Database {
};
const relatedEntities = await this.getEntities(
queryRunner,
relationEntity,
relationsMap,
block,
@ -288,6 +363,7 @@ export class Database {
};
const relatedEntities = await this.getEntities(
queryRunner,
relationEntity,
relationsMap,
block,
@ -325,6 +401,7 @@ export class Database {
};
const relatedEntities = await this.getEntities(
queryRunner,
relationEntity,
relationsMap,
block,

View File

@ -258,14 +258,27 @@ export class GraphWatcher {
}
async getEntity<Entity> (entity: new () => Entity, id: string, relationsMap: Map<any, { [key: string]: any }>, block?: BlockHeight): Promise<any> {
const dbTx = await this._database.createTransactionRunner();
try {
// Get entity from the database.
const result = await this._database.getEntityWithRelations(entity, id, relationsMap, block);
const result = await this._database.getEntityWithRelations(dbTx, entity, id, relationsMap, block);
await dbTx.commitTransaction();
// Resolve any field name conflicts in the entity result.
return resolveEntityFieldConflicts(result);
} catch (error) {
await dbTx.rollbackTransaction();
throw error;
} finally {
await dbTx.release();
}
}
async getEntities<Entity> (entity: new () => Entity, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight, where: { [key: string]: any } = {}, queryOptions: QueryOptions): Promise<any> {
const dbTx = await this._database.createTransactionRunner();
try {
where = Object.entries(where).reduce((acc: { [key: string]: any }, [fieldWithSuffix, value]) => {
const [field, ...suffix] = fieldWithSuffix.split('_');
@ -300,10 +313,17 @@ export class GraphWatcher {
}
// Get entities from the database.
const entities = await this._database.getEntities(entity, relationsMap, block, where, queryOptions);
const entities = await this._database.getEntities(dbTx, entity, relationsMap, block, where, queryOptions);
await dbTx.commitTransaction();
// Resolve any field name conflicts in the entity result.
return entities.map(entity => resolveEntityFieldConflicts(entity));
} catch (error) {
await dbTx.rollbackTransaction();
throw error;
} finally {
await dbTx.release();
}
}
/**

View File

@ -2,7 +2,6 @@
// Copyright 2021 Vulcanize, Inc.
//
import assert from 'assert';
import debug from 'debug';
import { JobQueue } from './job-queue';
@ -27,7 +26,10 @@ export const fillBlocks = async (
}
): Promise<any> => {
let { startBlock, endBlock, prefetch = false, batchBlocks = DEFAULT_PREFETCH_BATCH_SIZE } = argv;
assert(startBlock <= endBlock, 'endBlock should be greater than or equal to startBlock');
if (startBlock > endBlock) {
throw new Error(`endBlock ${endBlock} should be greater than or equal to startBlock ${startBlock}`);
}
const syncStatus = await indexer.getSyncStatus();
@ -45,6 +47,10 @@ export const fillBlocks = async (
throw new Error(`Missing blocks between startBlock ${startBlock} and chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`);
}
if (endBlock <= syncStatus.chainHeadBlockNumber) {
throw new Error(`endBlock ${endBlock} should be greater than chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`);
}
startBlock = syncStatus.chainHeadBlockNumber + 1;
}