Add DB queries without checking pruned block (#217)

packages/graph-node/src/database.ts

@@ -41,8 +41,10 @@ const DEFAULT_CLEAR_ENTITIES_CACHE_INTERVAL = 1000;
 export enum ENTITY_QUERY_TYPE {
   SINGULAR,
   DISTINCT_ON,
+  DISTINCT_ON_WITHOUT_PRUNED,
   GROUP_BY,
-  UNIQUE
+  GROUP_BY_WITHOUT_PRUNED,
+  UNIQUE,
 }
 
 interface CachedEntities {
@@ -375,14 +377,24 @@ export class Database {
         entities = await this.getEntitiesSingular(queryRunner, entity, block, where);
         break;
 
-      case ENTITY_QUERY_TYPE.DISTINCT_ON:
-        entities = await this.getEntitiesDistinctOn(queryRunner, entity, block, where, queryOptions);
-        break;
-
       case ENTITY_QUERY_TYPE.UNIQUE:
         entities = await this.getEntitiesUnique(queryRunner, entity, block, where, queryOptions);
         break;
 
+      case ENTITY_QUERY_TYPE.DISTINCT_ON:
+        entities = await this.getEntitiesDistinctOn(queryRunner, entity, block, where, queryOptions);
+        break;
+
+      case ENTITY_QUERY_TYPE.DISTINCT_ON_WITHOUT_PRUNED:
+        entities = await this.getEntitiesDistinctOnWithoutPruned(queryRunner, entity, block, where, queryOptions);
+        break;
+
+      case ENTITY_QUERY_TYPE.GROUP_BY_WITHOUT_PRUNED:
+        // Use group by query if entity query type is not specified in map.
+        entities = await this.getEntitiesGroupByWithoutPruned(queryRunner, entity, block, where, queryOptions);
+        break;
+
+      case ENTITY_QUERY_TYPE.GROUP_BY:
       default:
         // Use group by query if entity query type is not specified in map.
         entities = await this.getEntitiesGroupBy(queryRunner, entity, block, where, queryOptions);
@@ -532,6 +544,132 @@ export class Database {
     return entities as Entity[];
   }
 
+  async getEntitiesGroupByWithoutPruned<Entity> (
+    queryRunner: QueryRunner,
+    entity: new () => Entity,
+    block: BlockHeight,
+    where: Where = {},
+    queryOptions: QueryOptions = {}
+  ): Promise<Entity[]> {
+    const repo = queryRunner.manager.getRepository(entity);
+    const { tableName } = repo.metadata;
+
+    let subQuery = repo.createQueryBuilder('subTable')
+      .select('subTable.id', 'id')
+      .addSelect('MAX(subTable.block_number)', 'block_number')
+      .groupBy('subTable.id');
+
+    if (where.id) {
+      subQuery = this._baseDatabase.buildQuery(repo, subQuery, { id: where.id });
+      delete where.id;
+    }
+
+    if (block.hash) {
+      const { canonicalBlockNumber, blockHashes } = await this._baseDatabase.getFrothyRegion(queryRunner, block.hash);
+
+      subQuery = subQuery
+        .andWhere(new Brackets(qb => {
+          qb.where('subTable.block_hash IN (:...blockHashes)', { blockHashes })
+            .orWhere('subTable.block_number <= :canonicalBlockNumber', { canonicalBlockNumber });
+        }));
+    }
+
+    if (block.number) {
+      subQuery = subQuery.andWhere('subTable.block_number <= :blockNumber', { blockNumber: block.number });
+    }
+
+    let selectQueryBuilder = repo.createQueryBuilder(tableName)
+      .innerJoin(
+        `(${subQuery.getQuery()})`,
+        'latestEntities',
+        `${tableName}.id = "latestEntities"."id" AND ${tableName}.block_number = "latestEntities"."block_number"`
+      )
+      .setParameters(subQuery.getParameters());
+
+    selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where);
+
+    if (queryOptions.orderBy) {
+      selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions);
+    }
+
+    selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' });
+
+    if (queryOptions.skip) {
+      selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip);
+    }
+
+    if (queryOptions.limit) {
+      selectQueryBuilder = selectQueryBuilder.limit(queryOptions.limit);
+    }
+
+    const entities = await selectQueryBuilder.getMany();
+
+    return entities;
+  }
+
+  async getEntitiesDistinctOnWithoutPruned<Entity> (
+    queryRunner: QueryRunner,
+    entity: new () => Entity,
+    block: BlockHeight,
+    where: Where = {},
+    queryOptions: QueryOptions = {}
+  ): Promise<Entity[]> {
+    const repo = queryRunner.manager.getRepository(entity);
+
+    let subQuery = repo.createQueryBuilder('subTable')
+      .distinctOn(['subTable.id'])
+      .addOrderBy('subTable.id', 'ASC')
+      .addOrderBy('subTable.block_number', 'DESC');
+
+    if (where.id) {
+      subQuery = this._baseDatabase.buildQuery(repo, subQuery, { id: where.id });
+      delete where.id;
+    }
+
+    if (block.hash) {
+      const { canonicalBlockNumber, blockHashes } = await this._baseDatabase.getFrothyRegion(queryRunner, block.hash);
+
+      subQuery = subQuery
+        .andWhere(new Brackets(qb => {
+          qb.where('subTable.block_hash IN (:...blockHashes)', { blockHashes })
+            .orWhere('subTable.block_number <= :canonicalBlockNumber', { canonicalBlockNumber });
+        }));
+    }
+
+    if (block.number) {
+      subQuery = subQuery.andWhere('subTable.block_number <= :blockNumber', { blockNumber: block.number });
+    }
+
+    subQuery = this._baseDatabase.buildQuery(repo, subQuery, where);
+
+    let selectQueryBuilder = queryRunner.manager.createQueryBuilder()
+      .from(
+        `(${subQuery.getQuery()})`,
+        'latestEntities'
+      )
+      .setParameters(subQuery.getParameters());
+
+    if (queryOptions.orderBy) {
+      selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions, 'subTable_');
+      if (queryOptions.orderBy !== 'id') {
+        selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }, 'subTable_');
+      }
+    }
+
+    if (queryOptions.skip) {
+      selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip);
+    }
+
+    if (queryOptions.limit) {
+      selectQueryBuilder = selectQueryBuilder.limit(queryOptions.limit);
+    }
+
+    let entities = await selectQueryBuilder.getRawMany();
+    entities = await this._transformResults(queryRunner, repo.createQueryBuilder('subTable'), entities);
+
+    return entities as Entity[];
+  }
+
   async getEntitiesSingular<Entity> (
     queryRunner: QueryRunner,
     entity: new () => Entity,
@@ -733,8 +871,9 @@ export class Database {
       }
 
       // field is neither an array nor derivedFrom
+
+      // Avoid loading relation if selections only has id field.
       if (childSelections.length === 1 && childSelections[0].kind === 'Field' && childSelections[0].name.value === 'id') {
-        // Avoid loading relation if selections only has id field.
         entities.forEach((entity: any) => {
           entity[field] = { id: entity[field] };
         });

packages/graph-node/src/index.ts

@@ -2,5 +2,6 @@ export * from './watcher';
 export * from './database';
 export {
   prepareEntityState,
-  updateEntitiesFromState
+  updateEntitiesFromState,
+  resolveEntityFieldConflicts
 } from './utils';

packages/util/src/database.ts

@@ -26,7 +26,7 @@ import { BlockProgressInterface, ContractInterface, EventInterface, StateInterfa
 import { MAX_REORG_DEPTH, UNKNOWN_EVENT_NAME } from './constants';
 import { blockProgressCount, eventCount } from './metrics';
 
-const OPERATOR_MAP = {
+export const OPERATOR_MAP = {
   equals: '=',
   gt: '>',
   lt: '<',