Add a table for entites in frothy region for subgraph watchers (#231)

* Add a table for entites in frothy region and update it in a subscriber

* Accommodate changes to other watchers and codegen

packages/codegen/src/data/entities/FrothyEntity.yaml

@@ -0,0 +1,31 @@
+className: FrothyEntity
+indexOn:
+  - columns:
+      - blockNumber
+columns:
+  - name: id
+    pgType: varchar
+    tsType: string
+    columnType: PrimaryColumn
+  - name: name
+    pgType: varchar
+    tsType: string
+    columnType: PrimaryColumn
+  - name: blockHash
+    pgType: varchar
+    tsType: string
+    columnType: PrimaryColumn
+    columnOptions:
+      - option: length
+        value: 66
+  - name: blockNumber
+    pgType: integer
+    tsType: number
+    columnType: Column
+imports:
+  - toImport:
+      - Entity
+      - PrimaryColumn
+      - Column
+      - Index
+    from: typeorm

packages/codegen/src/database.ts

@@ -16,10 +16,12 @@ const TEMPLATE_FILE = './templates/database-template.handlebars';
 
 export class Database {
   _queries: Array<any>;
+  _subgraphEntities: Array<any>;
   _templateString: string;
 
   constructor () {
     this._queries = [];
+    this._subgraphEntities = [];
     this._templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString();
   }
 
@@ -72,6 +74,23 @@ export class Database {
     this._queries.push(queryObject);
   }
 
+  addSubgraphEntities (subgraphSchemaDocument: any): void {
+    // Add subgraph entities for adding them to the entities list.
+    const subgraphTypeDefs = subgraphSchemaDocument.definitions;
+
+    subgraphTypeDefs.forEach((def: any) => {
+      if (def.kind !== 'ObjectTypeDefinition') {
+        return;
+      }
+
+      const entityObject: any = {
+        className: def.name.value
+      };
+
+      this._subgraphEntities.push(entityObject);
+    });
+  }
+
   /**
    * Writes the database file generated from a template to a stream.
    * @param outStream A writable output stream to write the database file to.
@@ -79,7 +98,8 @@ export class Database {
   exportDatabase (outStream: Writable): void {
     const template = Handlebars.compile(this._templateString);
     const obj = {
-      queries: this._queries
+      queries: this._queries,
+      subgraphEntities: this._subgraphEntities
     };
     const database = template(obj);
     outStream.write(database);

packages/codegen/src/entity.ts

@@ -173,7 +173,7 @@ export class Entity {
    * Writes the generated entity files in the given directory.
    * @param entityDir Directory to write the entities to.
    */
-  exportEntities (entityDir: string): void {
+  exportEntities (entityDir: string, subgraphPath: string): void {
     this._addEventEntity();
     this._addSyncStatusEntity();
     this._addContractEntity();
@@ -181,6 +181,11 @@ export class Entity {
     this._addStateEntity();
     this._addStateSyncStatusEntity();
 
+    // Add FrothyEntity table only for subgraph watchers
+    if (subgraphPath) {
+      this._addFrothyEntity();
+    }
+
     const template = Handlebars.compile(this._templateString);
     this._entities.forEach(entityObj => {
       const entity = template(entityObj);
@@ -288,6 +293,11 @@ export class Entity {
     this._entities.push(entity);
   }
 
+  _addFrothyEntity (): void {
+    const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'FrothyEntity.yaml'), 'utf8'));
+    this._entities.push(entity);
+  }
+
   _addBigIntTransformerOption (entityObject: any): void {
     let importObject = entityObject.imports.find((element: any) => {
       return element.from === '@cerc-io/util';

packages/codegen/src/generate-code.ts

@@ -37,6 +37,7 @@ import { importState } from './import-state';
 import { exportInspectCID } from './inspect-cid';
 import { getSubgraphConfig } from './utils/subgraph';
 import { exportIndexBlock } from './index-block';
+import { exportSubscriber } from './subscriber';
 
 const main = async (): Promise<void> => {
   const argv = await yargs(hideBin(process.argv))
@@ -217,7 +218,7 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
   const entityDir = outputDir
     ? path.join(outputDir, 'src/entity')
     : '';
-  visitor.exportEntities(entityDir);
+  visitor.exportEntities(entityDir, config.subgraphPath);
 
   outStream = outputDir
     ? fs.createWriteStream(path.join(outputDir, 'README.md'))
@@ -323,6 +324,13 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
     ? fs.createWriteStream(path.join(outputDir, 'src/cli/index-block.ts'))
     : process.stdout;
   exportIndexBlock(outStream);
+
+  if (config.subgraphPath) {
+    outStream = outputDir
+      ? fs.createWriteStream(path.join(outputDir, 'src/entity/Subscriber.ts'))
+      : process.stdout;
+    exportSubscriber(outStream);
+  }
 }
 
 function getConfig (configFile: string): any {

packages/codegen/src/subscriber.ts

@@ -0,0 +1,21 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import fs from 'fs';
+import path from 'path';
+import Handlebars from 'handlebars';
+import { Writable } from 'stream';
+
+const SUBSCRIBER_TEMPLATE_FILE = './templates/subscriber-template.handlebars';
+
+/**
+ * Writes the subscriber file generated from template to a stream.
+ * @param outStream A writable output stream to write the subscriber file to.
+ */
+export function exportSubscriber (subscriberOutStream: Writable): void {
+  const subscriberTemplateString = fs.readFileSync(path.resolve(__dirname, SUBSCRIBER_TEMPLATE_FILE)).toString();
+  const subscriberTemplate = Handlebars.compile(subscriberTemplateString);
+  const subscriber = subscriberTemplate({});
+  subscriberOutStream.write(subscriber);
+}

packages/codegen/src/templates/database-template.handlebars

@@ -17,6 +17,15 @@ import { State } from './entity/State';
 {{#each queries as | query |}}
 import { {{query.entityName}} } from './entity/{{query.entityName}}';
 {{/each}}
+{{#each subgraphEntities as | subgraphEntity |}}
+import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.className}}';
+{{/each}}
+
+export const ENTITIES = [
+  {{~#each queries as | query |}}{{query.entityName}}, {{/each}}
+  {{~#each subgraphEntities as | subgraphEntity |}}{{subgraphEntity.className}}
+  {{~#unless @last}}, {{/unless}}
+  {{~/each}}];
 
 export class Database implements DatabaseInterface {
   _config: ConnectionOptions;

packages/codegen/src/templates/indexer-template.handlebars

@@ -41,7 +41,7 @@ import { GraphWatcher } from '@cerc-io/graph-node';
 {{#each contracts as | contract |}}
 import {{contract.contractName}}Artifacts from './artifacts/{{contract.contractName}}.json';
 {{/each}}
-import { Database } from './database';
+import { Database, ENTITIES } from './database';
 import { createInitialState, handleEvent, createStateDiff, createStateCheckpoint } from './hooks';
 import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';
@@ -49,13 +49,12 @@ import { SyncStatus } from './entity/SyncStatus';
 import { StateSyncStatus } from './entity/StateSyncStatus';
 import { BlockProgress } from './entity/BlockProgress';
 import { State } from './entity/State';
-
-{{#each queries as | query |}}
-import { {{query.entityName}} } from './entity/{{query.entityName}}';
-{{/each}}
 {{#each subgraphEntities as | subgraphEntity |}}
 import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.className}}';
 {{/each}}
+{{#if (subgraphPath)}}
+import { FrothyEntity } from './entity/FrothyEntity';
+{{/if}}
 
 const log = debug('vulcanize:indexer');
 const JSONbigNative = JSONbig({ useNativeBigInt: true });
@@ -521,7 +520,12 @@ export class Indexer implements IndexerInterface {
   }
 
   async updateSyncStatusCanonicalBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
-    return this._baseIndexer.updateSyncStatusCanonicalBlock(blockHash, blockNumber, force);
+    const syncStatus = this._baseIndexer.updateSyncStatusCanonicalBlock(blockHash, blockNumber, force);
+    {{#if (subgraphPath)}}
+    await this.pruneFrothyEntities(blockNumber);
+    {{/if}}
+
+    return syncStatus;
   }
 
   async getEvent (id: string): Promise<Event | undefined> {
@@ -556,6 +560,12 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.markBlocksAsPruned(blocks);
   }
 
+  {{#if (subgraphPath)}}
+  async pruneFrothyEntities (blockNumber: number): Promise<void> {
+    await this._graphWatcher.pruneFrothyEntities(FrothyEntity, blockNumber);
+  }
+  {{/if}}
+
   async updateBlockProgress (block: BlockProgress, lastProcessedEventIndex: number): Promise<BlockProgress> {
     return this._baseIndexer.updateBlockProgress(block, lastProcessedEventIndex);
   }
@@ -565,14 +575,11 @@ export class Indexer implements IndexerInterface {
   }
 
   async resetWatcherToBlock (blockNumber: number): Promise<void> {
-    const entities = [
-      {{#each queries as | query |}}
-      {{query.entityName}},
-      {{/each}}
-      {{#each subgraphEntities as | subgraphEntity |}}
-      {{subgraphEntity.className}},
-      {{/each}}
-    ];
+    {{#if (subgraphPath)}}
+    const entities = [...ENTITIES, FrothyEntity];
+    {{else}}
+    const entities = [...ENTITIES];
+    {{/if}}
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 

packages/codegen/src/templates/subscriber-template.handlebars

@@ -0,0 +1,20 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { EventSubscriber, EntitySubscriberInterface, InsertEvent, UpdateEvent } from 'typeorm';
+
+import { FrothyEntity } from './FrothyEntity';
+import { ENTITIES } from '../database';
+import { afterEntityInsertOrUpdate } from '@cerc-io/graph-node';
+
+@EventSubscriber()
+export class EntitySubscriber implements EntitySubscriberInterface {
+  async afterInsert (event: InsertEvent<any>): Promise<void> {
+    await afterEntityInsertOrUpdate(FrothyEntity, ENTITIES, event);
+  }
+
+  async afterUpdate (event: UpdateEvent<any>): Promise<void> {
+    await afterEntityInsertOrUpdate(FrothyEntity, ENTITIES, event);
+  }
+}

packages/codegen/src/templates/tsconfig-template.handlebars

@@ -21,7 +21,7 @@
     // "removeComments": true,                      /* Do not emit comments to output. */
     // "noEmit": true,                              /* Do not emit outputs. */
     // "importHelpers": true,                       /* Import emit helpers from 'tslib'. */
-    // "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
     // "isolatedModules": true,                     /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
 
     /* Strict Type-Checking Options */

packages/codegen/src/visitor.ts

@@ -155,6 +155,7 @@ export class Visitor {
     this._resolvers.addSubgraphResolvers(subgraphSchemaDocument);
     this._reset.addSubgraphEntities(subgraphSchemaDocument);
     this._indexer.addSubgraphEntities(subgraphSchemaDocument);
+    this._database.addSubgraphEntities(subgraphSchemaDocument);
   }
 
   /**
@@ -187,8 +188,8 @@ export class Visitor {
    * Writes the generated entity files in the given directory.
    * @param entityDir Directory to write the entities to.
    */
-  exportEntities (entityDir: string): void {
-    this._entity.exportEntities(entityDir);
+  exportEntities (entityDir: string, subgraphPath: string): void {
+    this._entity.exportEntities(entityDir, subgraphPath);
   }
 
   /**