Accommodate GQL optimizations in codegen (#254)

* Prune optional methods in indexer and database interfaces

* Implement GQL optimization changes in codegen

* Fix graph-node test indexer

* Add demos to codegen package

packages/codegen/src/entity.ts

@@ -253,6 +253,20 @@ export class Entity {
       // Add subgraph entity specific columns.
       entityObject = this._addSubgraphColumns(subgraphTypeDefs, entityObject, def);
 
+      // Add is_pruned column.
+      entityObject.columns.push({
+        name: 'isPruned',
+        pgType: 'boolean',
+        tsType: 'boolean',
+        columnType: 'Column',
+        columnOptions: [
+          {
+            option: 'default',
+            value: false
+          }
+        ]
+      });
+
       // Add decimalTransformer column option if required.
       this._addDecimalTransformerOption(entityObject);
 

packages/codegen/src/templates/checkpoint-create-template.handlebars

@@ -40,13 +40,12 @@ export const handler = async (argv: any): Promise<void> => {
   await db.init();
 
   {{#if (subgraphPath)}}
-
   const graphDb = new GraphDatabase(config.server, db.baseDatabase);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
-  {{/if}}
 
+  {{/if}}
   const jobQueueConfig = config.jobQueue;
   assert(jobQueueConfig, 'Missing job queue config');
 

packages/codegen/src/templates/checkpoint-verify-template.handlebars

@@ -9,7 +9,7 @@ import assert from 'assert';
 import { getConfig, initClients, JobQueue, Config, verifyCheckpointData } from '@cerc-io/util';
 import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 
-import { Database } from '../../database';
+import { Database, ENTITY_TO_LATEST_ENTITY_MAP, ENTITY_QUERY_TYPE_MAP } from '../../database';
 import { Indexer } from '../../indexer';
 
 const log = debug('vulcanize:checkpoint-verify');
@@ -34,7 +34,7 @@ export const handler = async (argv: any): Promise<void> => {
   const db = new Database(config.database);
   await db.init();
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/database-template.handlebars

@@ -7,6 +7,9 @@ import { Connection, ConnectionOptions, DeepPartial, FindConditions, QueryRunner
 import path from 'path';
 
 import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, Where } from '@cerc-io/util';
+{{#if (subgraphPath)}}
+import { ENTITY_QUERY_TYPE } from '@cerc-io/graph-node';
+{{/if}}
 
 import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';
@@ -21,11 +24,23 @@ import { {{query.entityName}} } from './entity/{{query.entityName}}';
 import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.className}}';
 {{/each}}
 
-export const ENTITIES = [
-  {{~#each queries as | query |}}{{query.entityName}}, {{/each}}
+{{#if (subgraphPath)}}
+export const SUBGRAPH_ENTITIES = new Set([
   {{~#each subgraphEntities as | subgraphEntity |}}{{subgraphEntity.className}}
   {{~#unless @last}}, {{/unless}}
+  {{~/each}}]);
+{{/if}}
+export const ENTITIES = [
+  {{~#if (subgraphPath)}}...SUBGRAPH_ENTITIES, {{/if}}
+  {{~#each queries as | query |}}{{query.entityName}}
+  {{~#unless @last}}, {{/unless}}
   {{~/each}}];
+{{#if (subgraphPath)}}
+// Map: Entity to suitable query type.
+export const ENTITY_QUERY_TYPE_MAP = new Map<new() => any, ENTITY_QUERY_TYPE>([]);
+
+export const ENTITY_TO_LATEST_ENTITY_MAP: Map<any, any> = new Map();
+{{/if}}
 
 export class Database implements DatabaseInterface {
   _config: ConnectionOptions;
@@ -38,6 +53,9 @@ export class Database implements DatabaseInterface {
 
     this._config = {
       ...config,
+      {{#if (subgraphPath)}}
+      subscribers: [path.join(__dirname, 'entity/Subscriber.*')],
+      {{/if}}
       entities: [path.join(__dirname, 'entity/*')]
     };
 

packages/codegen/src/templates/export-state-template.handlebars

@@ -24,7 +24,7 @@ import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 {{/if}}
 import * as codec from '@ipld/dag-cbor';
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:export-state');
@@ -59,7 +59,7 @@ const main = async (): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/fill-template.handlebars

@@ -7,14 +7,14 @@ import 'reflect-metadata';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
- import { PubSub } from 'graphql-subscriptions';
+import { PubSub } from 'graphql-subscriptions';
 
 import { Config, getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, initClients } from '@cerc-io/util';
 {{#if (subgraphPath)}}
 import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 {{/if}}
 
-import { Database } from './database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database';
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
 {{#if (subgraphPath)}}
@@ -72,7 +72,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/import-state-template.handlebars

@@ -17,7 +17,7 @@ import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 {{/if}}
 import * as codec from '@ipld/dag-cbor';
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 import { EventWatcher } from '../events';
 import { State } from '../entity/State';
@@ -50,7 +50,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/index-block-template.handlebars

@@ -12,7 +12,7 @@ import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, indexBlo
 import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 {{/if}}
 
-import { Database } from '../database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from '../database';
 import { Indexer } from '../indexer';
 
 const log = debug('vulcanize:index-block');
@@ -44,7 +44,7 @@ const main = async (): Promise<void> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/indexer-template.handlebars

@@ -41,7 +41,7 @@ import { GraphWatcher } from '@cerc-io/graph-node';
 {{#each contracts as | contract |}}
 import {{contract.contractName}}Artifacts from './artifacts/{{contract.contractName}}.json';
 {{/each}}
-import { Database, ENTITIES } from './database';
+import { Database, ENTITIES{{#if (subgraphPath)}}, SUBGRAPH_ENTITIES{{/if}} } from './database';
 import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint } from './hooks';
 import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';
@@ -104,17 +104,17 @@ export class Indexer implements IndexerInterface {
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
-
     {{#each contracts as | contract |}}
+
     const {
       abi: {{contract.contractName}}ABI,
       {{#if contract.contractStorageLayout}}
       storageLayout: {{contract.contractName}}StorageLayout
       {{/if}}
     } = {{contract.contractName}}Artifacts;
-
     {{/each}}
     {{#each contracts as | contract |}}
+
     assert({{contract.contractName}}ABI);
     this._abiMap.set(KIND_{{capitalize contract.contractName}}, {{contract.contractName}}ABI);
     {{#if contract.contractStorageLayout}}
@@ -122,9 +122,9 @@ export class Indexer implements IndexerInterface {
     this._storageLayoutMap.set(KIND_{{capitalize contract.contractName}}, {{contract.contractName}}StorageLayout);
     {{/if}}
     this._contractMap.set(KIND_{{capitalize contract.contractName}}, new ethers.utils.Interface({{contract.contractName}}ABI));
-
     {{/each}}
     {{#if (subgraphPath)}}
+
     this._entityTypesMap = new Map();
     this._populateEntityTypesMap();
 
@@ -259,8 +259,10 @@ export class Indexer implements IndexerInterface {
   }
 
   async processCanonicalBlock (blockHash: string, blockNumber: number): Promise<void> {
+    console.time('time:indexer#processCanonicalBlock-finalize_auto_diffs');
     // Finalize staged diff blocks if any.
     await this._baseIndexer.finalizeDiffStaged(blockHash);
+    console.timeEnd('time:indexer#processCanonicalBlock-finalize_auto_diffs');
 
     // Call custom stateDiff hook.
     await createStateDiff(this, blockHash);
@@ -275,7 +277,9 @@ export class Indexer implements IndexerInterface {
     const checkpointInterval = this._serverConfig.checkpointInterval;
     if (checkpointInterval <= 0) return;
 
+    console.time('time:indexer#processCheckpoint-checkpoint');
     await this._baseIndexer.processCheckpoint(this, blockHash, checkpointInterval);
+    console.timeEnd('time:indexer#processCheckpoint-checkpoint');
   }
 
   async processCLICheckpoint (contractAddress: string, blockHash?: string): Promise<string | undefined> {
@@ -308,7 +312,9 @@ export class Indexer implements IndexerInterface {
 
   // Method used to create auto diffs (diff_staged).
   async createDiffStaged (contractAddress: string, blockHash: string, data: any): Promise<void> {
+    console.time('time:indexer#createDiffStaged-auto_diff');
     await this._baseIndexer.createDiffStaged(contractAddress, blockHash, data);
+    console.timeEnd('time:indexer#createDiffStaged-auto_diff');
   }
 
   // Method to be used by createStateDiff hook.
@@ -361,13 +367,25 @@ export class Indexer implements IndexerInterface {
     return data;
   }
 
+  async getSubgraphEntities<Entity> (
+    entity: new () => Entity,
+    block: BlockHeight,
+    where: { [key: string]: any } = {},
+    queryOptions: QueryOptions = {},
+    selections: ReadonlyArray<SelectionNode> = []
+  ): Promise<any[]> {
+    return this._graphWatcher.getEntities(entity, this._relationsMap, block, where, queryOptions, selections);
+  }
+
   {{/if}}
   async triggerIndexingOnEvent (event: Event): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     {{#if (subgraphPath)}}
+    console.time('time:indexer#processEvent-mapping_code');
     // Call subgraph handler for event.
     await this._graphWatcher.handleEvent(resultEvent);
+    console.timeEnd('time:indexer#processEvent-mapping_code');
 
     {{/if}}
     // Call custom hook function for indexing on event.
@@ -380,8 +398,10 @@ export class Indexer implements IndexerInterface {
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
+    console.time('time:indexer#processBlock-init_state');
     // Call a function to create initial state for contracts.
     await this._baseIndexer.createInit(this, blockProgress.blockHash, blockProgress.blockNumber);
+    console.timeEnd('time:indexer#processBlock-init_state');
     {{#if (subgraphPath)}}
 
     this._graphWatcher.updateEntityCacheFrothyBlocks(blockProgress);
@@ -390,11 +410,15 @@ export class Indexer implements IndexerInterface {
 
   {{#if (subgraphPath)}}
   async processBlockAfterEvents (blockHash: string, blockNumber: number): Promise<void> {
+    console.time('time:indexer#processBlockAfterEvents-mapping_code');
     // Call subgraph handler for block.
     await this._graphWatcher.handleBlock(blockHash, blockNumber);
+    console.timeEnd('time:indexer#processBlockAfterEvents-mapping_code');
 
+    console.time('time:indexer#processBlockAfterEvents-dump_subgraph_state');
     // Persist subgraph state to the DB.
     await this.dumpSubgraphState(blockHash);
+    console.timeEnd('time:indexer#processBlockAfterEvents-dump_subgraph_state');
   }
 
   {{/if}}
@@ -557,15 +581,23 @@ export class Indexer implements IndexerInterface {
   }
 
   async markBlocksAsPruned (blocks: BlockProgress[]): Promise<void> {
-    return this._baseIndexer.markBlocksAsPruned(blocks);
+    await this._baseIndexer.markBlocksAsPruned(blocks);
+    {{#if (subgraphPath)}}
+
+    await this._graphWatcher.pruneEntities(FrothyEntity, blocks, SUBGRAPH_ENTITIES);
+    {{/if}}
   }
 
   {{#if (subgraphPath)}}
   async pruneFrothyEntities (blockNumber: number): Promise<void> {
     await this._graphWatcher.pruneFrothyEntities(FrothyEntity, blockNumber);
   }
-  {{/if}}
 
+  async resetLatestEntities (blockNumber: number): Promise<void> {
+    await this._graphWatcher.resetLatestEntities(blockNumber);
+  }
+
+  {{/if}}
   async updateBlockProgress (block: BlockProgress, lastProcessedEventIndex: number): Promise<BlockProgress> {
     return this._baseIndexer.updateBlockProgress(block, lastProcessedEventIndex);
   }
@@ -581,9 +613,13 @@ export class Indexer implements IndexerInterface {
     const entities = [...ENTITIES];
     {{/if}}
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
-  }
+    {{#if (subgraphPath)}}
 
+    await this.resetLatestEntities(blockNumber);
+    {{/if}}
+  }
   {{#if (subgraphPath)}}
+
   getEntityTypesMap (): Map<string, { [key: string]: string }> {
     return this._entityTypesMap;
   }

packages/codegen/src/templates/job-runner-template.handlebars

@@ -28,7 +28,7 @@ import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 {{/if}}
 
 import { Indexer } from './indexer';
-import { Database } from './database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database';
 
 const log = debug('vulcanize:job-runner');
 
@@ -97,7 +97,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/resolvers-template.handlebars

@@ -12,10 +12,12 @@ import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigInt
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
+{{#if (subgraphPath)}}
 
 {{#each subgraphQueries as | query |}}
 import { {{query.entityName}} } from './entity/{{query.entityName}}';
 {{/each}}
+{{/if}}
 
 const log = debug('vulcanize:resolver');
 

packages/codegen/src/templates/server-template.handlebars

@@ -20,7 +20,7 @@ import { GraphWatcher, Database as GraphDatabase } from '@cerc-io/graph-node';
 
 import { createResolvers } from './resolvers';
 import { Indexer } from './indexer';
-import { Database } from './database';
+import { Database{{#if (subgraphPath)}}, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP{{/if}} } from './database';
 import { EventWatcher } from './events';
 
 const log = debug('vulcanize:server');
@@ -45,7 +45,7 @@ export const main = async (): Promise<any> => {
   await db.init();
   {{#if (subgraphPath)}}
 
-  const graphDb = new GraphDatabase(config.server, db.baseDatabase);
+  const graphDb = new GraphDatabase(config.server, db.baseDatabase, ENTITY_QUERY_TYPE_MAP, ENTITY_TO_LATEST_ENTITY_MAP);
   await graphDb.init();
 
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);

packages/codegen/src/templates/subscriber-template.handlebars

@@ -5,16 +5,16 @@
 import { EventSubscriber, EntitySubscriberInterface, InsertEvent, UpdateEvent } from 'typeorm';
 
 import { FrothyEntity } from './FrothyEntity';
-import { ENTITIES } from '../database';
+import { ENTITY_TO_LATEST_ENTITY_MAP, SUBGRAPH_ENTITIES } from '../database';
 import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node';
 
 @EventSubscriber()
 export class EntitySubscriber implements EntitySubscriberInterface {
   async afterInsert (event: InsertEvent<any>): Promise<void> {
-    await afterEntityInsertOrUpdate(FrothyEntity, ENTITIES, event);
+    await afterEntityInsertOrUpdate(FrothyEntity, SUBGRAPH_ENTITIES, event, ENTITY_TO_LATEST_ENTITY_MAP);
   }
 
   async afterUpdate (event: UpdateEvent<any>): Promise<void> {
-    await afterEntityInsertOrUpdate(FrothyEntity, ENTITIES, event);
+    await afterEntityInsertOrUpdate(FrothyEntity, SUBGRAPH_ENTITIES, event, ENTITY_TO_LATEST_ENTITY_MAP);
   }
 }

packages/codegen/src/templates/types-template.handlebars

@@ -1,8 +1,8 @@
 //
 // Copyright 2021 Vulcanize, Inc.
 //
-
 {{#each types as | type |}}
+
 export enum {{type.name}} {
   {{#each type.values as | value |}}
   {{value}} = '{{value}}',