Update codegen with changes implemented in mobymask watcher (#148)

* Update codegen with index-block CLI and remove graph-node

* Add filter logs by contract flag

* Skip generating GQL API for immutable variables

* Add config for maxEventsBlockRange

* Add new flags in existing watchers

packages/mobymask-watcher/src/cli/index-block.ts

@@ -7,16 +7,12 @@ import 'reflect-metadata';
 import debug from 'debug';
 import assert from 'assert';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, OrderDirection, UNKNOWN_EVENT_NAME } from '@vulcanize/util';
+import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, indexBlock } from '@vulcanize/util';
 
 import { Database } from '../database';
 import { Indexer } from '../indexer';
-import { BlockProgress } from '../entity/BlockProgress';
-import { Event } from '../entity/Event';
 
-const DEFAULT_EVENTS_IN_BATCH = 50;
-
-const log = debug('vulcanize:watch-contract');
+const log = debug('vulcanize:index-block');
 
 const main = async (): Promise<void> => {
   const argv = await yargs.parserConfiguration({
@@ -55,70 +51,7 @@ const main = async (): Promise<void> => {
   const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
   await indexer.init();
 
-  let blockProgressEntities: Partial<BlockProgress>[] = await indexer.getBlocksAtHeight(argv.block, false);
-
-  if (!blockProgressEntities.length) {
-    console.time('time:index-block#getBlocks-ipld-eth-server');
-    const blocks = await indexer.getBlocks({ blockNumber: argv.block });
-
-    blockProgressEntities = blocks.map((block: any): Partial<BlockProgress> => {
-      block.blockTimestamp = block.timestamp;
-
-      return block;
-    });
-
-    console.timeEnd('time:index-block#getBlocks-ipld-eth-server');
-  }
-
-  assert(blockProgressEntities.length, `No blocks fetched for block number ${argv.block}.`);
-
-  for (let blockProgress of blockProgressEntities) {
-    // Check if blockProgress fetched from database.
-    if (!blockProgress.id) {
-      blockProgress = await indexer.fetchBlockEvents(blockProgress);
-    }
-
-    assert(blockProgress instanceof BlockProgress);
-    assert(indexer.processBlock);
-    await indexer.processBlock(blockProgress.blockHash, blockProgress.blockNumber);
-
-    // Check if block has unprocessed events.
-    if (blockProgress.numProcessedEvents < blockProgress.numEvents) {
-      while (!blockProgress.isComplete) {
-        console.time('time:index-block#fetching_events_batch');
-
-        // Fetch events in batches
-        const events = await indexer.getBlockEvents(
-          blockProgress.blockHash,
-          {
-            index: [
-              { value: blockProgress.lastProcessedEventIndex + 1, operator: 'gte', not: false }
-            ]
-          },
-          {
-            limit: jobQueueConfig.eventsInBatch || DEFAULT_EVENTS_IN_BATCH,
-            orderBy: 'index',
-            orderDirection: OrderDirection.asc
-          }
-        );
-
-        console.timeEnd('time:index-block#fetching_events_batch');
-
-        if (events.length) {
-          log(`Processing events batch from index ${events[0].index} to ${events[0].index + events.length - 1}`);
-        }
-
-        console.time('time:index-block#processEvents-processing_events_batch');
-
-        for (const event of events) {
-          // Process events in loop
-          await processEvent(indexer, blockProgress, event);
-        }
-
-        console.timeEnd('time:index-block#processEvents-processing_events_batch');
-      }
-    }
-  }
+  await indexBlock(indexer, jobQueueConfig.eventsInBatch, argv);
 
   await db.close();
 };
@@ -128,57 +61,3 @@ main().catch(err => {
 }).finally(() => {
   process.exit(0);
 });
-
-/**
- * Process individual event from database.
- * @param indexer
- * @param block
- * @param event
- */
-const processEvent = async (indexer: Indexer, block: BlockProgress, event: Event) => {
-  const eventIndex = event.index;
-
-  // Check that events are processed in order.
-  if (eventIndex <= block.lastProcessedEventIndex) {
-    throw new Error(`Events received out of order for block number ${block.blockNumber} hash ${block.blockHash}, got event index ${eventIndex} and lastProcessedEventIndex ${block.lastProcessedEventIndex}, aborting`);
-  }
-
-  // Check if previous event in block has been processed exactly before this and abort if not.
-  // Skip check if logs fetched are filtered by contract address.
-  if (!indexer.serverConfig.filterLogs) {
-    const prevIndex = eventIndex - 1;
-
-    if (prevIndex !== block.lastProcessedEventIndex) {
-      throw new Error(`Events received out of order for block number ${block.blockNumber} hash ${block.blockHash},` +
-      ` prev event index ${prevIndex}, got event index ${event.index} and lastProcessedEventIndex ${block.lastProcessedEventIndex}, aborting`);
-    }
-  }
-
-  let watchedContract;
-
-  if (!indexer.isWatchedContract) {
-    watchedContract = true;
-  } else {
-    watchedContract = await indexer.isWatchedContract(event.contract);
-  }
-
-  if (watchedContract) {
-    // We might not have parsed this event yet. This can happen if the contract was added
-    // as a result of a previous event in the same block.
-    if (event.eventName === UNKNOWN_EVENT_NAME) {
-      const logObj = JSON.parse(event.extraInfo);
-
-      assert(indexer.parseEventNameAndArgs);
-      assert(typeof watchedContract !== 'boolean');
-      const { eventName, eventInfo } = indexer.parseEventNameAndArgs(watchedContract.kind, logObj);
-
-      event.eventName = eventName;
-      event.eventInfo = JSON.stringify(eventInfo);
-      event = await indexer.saveEventEntity(event);
-    }
-
-    await indexer.processEvent(event);
-  }
-
-  block = await indexer.updateBlockProgress(block, event.index);
-};

packages/mobymask-watcher/src/cli/reset-cmds/state.ts

@@ -12,7 +12,6 @@ import { Database } from '../../database';
 import { Indexer } from '../../indexer';
 import { BlockProgress } from '../../entity/BlockProgress';
 
-import { DomainHash } from '../../entity/DomainHash';
 import { MultiNonce } from '../../entity/MultiNonce';
 import { _Owner } from '../../entity/_Owner';
 import { IsRevoked } from '../../entity/IsRevoked';
@@ -60,7 +59,7 @@ export const handler = async (argv: any): Promise<void> => {
   const dbTx = await db.createTransactionRunner();
 
   try {
-    const entities = [BlockProgress, DomainHash, MultiNonce, _Owner, IsRevoked, IsPhisher, IsMember];
+    const entities = [BlockProgress, MultiNonce, _Owner, IsRevoked, IsPhisher, IsMember];
 
     const removeEntitiesPromise = entities.map(async entityClass => {
       return db.removeEntities<any>(dbTx, entityClass, { blockNumber: MoreThan(argv.blockNumber) });

packages/mobymask-watcher/src/client.ts

@@ -17,15 +17,6 @@ export class Client {
     this._client = new GraphQLClient(config);
   }
 
-  async getDomainHash (blockHash: string, contractAddress: string): Promise<any> {
-    const { domainHash } = await this._client.query(
-      gql(queries.domainHash),
-      { blockHash, contractAddress }
-    );
-
-    return domainHash;
-  }
-
   async getMultiNonce (blockHash: string, contractAddress: string, key0: string, key1: bigint): Promise<any> {
     const { multiNonce } = await this._client.query(
       gql(queries.multiNonce),

packages/mobymask-watcher/src/database.ts

@@ -14,7 +14,6 @@ import { SyncStatus } from './entity/SyncStatus';
 import { IpldStatus } from './entity/IpldStatus';
 import { BlockProgress } from './entity/BlockProgress';
 import { IPLDBlock } from './entity/IPLDBlock';
-import { DomainHash } from './entity/DomainHash';
 import { MultiNonce } from './entity/MultiNonce';
 import { _Owner } from './entity/_Owner';
 import { IsRevoked } from './entity/IsRevoked';
@@ -48,14 +47,6 @@ export class Database implements IPLDDatabaseInterface {
     return this._baseDatabase.close();
   }
 
-  async getDomainHash ({ blockHash, contractAddress }: { blockHash: string, contractAddress: string }): Promise<DomainHash | undefined> {
-    return this._conn.getRepository(DomainHash)
-      .findOne({
-        blockHash,
-        contractAddress
-      });
-  }
-
   async getMultiNonce ({ blockHash, contractAddress, key0, key1 }: { blockHash: string, contractAddress: string, key0: string, key1: bigint }): Promise<MultiNonce | undefined> {
     return this._conn.getRepository(MultiNonce)
       .findOne({
@@ -111,12 +102,6 @@ export class Database implements IPLDDatabaseInterface {
       });
   }
 
-  async saveDomainHash ({ blockHash, blockNumber, contractAddress, value, proof }: DeepPartial<DomainHash>): Promise<DomainHash> {
-    const repo = this._conn.getRepository(DomainHash);
-    const entity = repo.create({ blockHash, blockNumber, contractAddress, value, proof });
-    return repo.save(entity);
-  }
-
   async saveMultiNonce ({ blockHash, blockNumber, contractAddress, key0, key1, value, proof }: DeepPartial<MultiNonce>): Promise<MultiNonce> {
     const repo = this._conn.getRepository(MultiNonce);
     const entity = repo.create({ blockHash, blockNumber, contractAddress, key0, key1, value, proof });
@@ -332,7 +317,6 @@ export class Database implements IPLDDatabaseInterface {
   }
 
   _setPropColMaps (): void {
-    this._propColMaps.DomainHash = this._getPropertyColumnMapForEntity('DomainHash');
     this._propColMaps.MultiNonce = this._getPropertyColumnMapForEntity('MultiNonce');
     this._propColMaps._Owner = this._getPropertyColumnMapForEntity('_Owner');
     this._propColMaps.IsRevoked = this._getPropertyColumnMapForEntity('IsRevoked');

packages/mobymask-watcher/src/entity/DomainHash.ts

@@ -1,27 +0,0 @@
-//
-// Copyright 2021 Vulcanize, Inc.
-//
-
-import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
-
-@Entity()
-@Index(['blockHash', 'contractAddress'], { unique: true })
-export class DomainHash {
-  @PrimaryGeneratedColumn()
-  id!: number;
-
-  @Column('varchar', { length: 66 })
-  blockHash!: string;
-
-  @Column('integer')
-  blockNumber!: number;
-
-  @Column('varchar', { length: 42 })
-  contractAddress!: string;
-
-  @Column('varchar')
-  value!: string;
-
-  @Column('text', { nullable: true })
-  proof!: string;
-}

packages/mobymask-watcher/src/gql/queries/domainHash.gql

@@ -1,8 +0,0 @@
-query domainHash($blockHash: String!, $contractAddress: String!){
-    domainHash(blockHash: $blockHash, contractAddress: $contractAddress){
-        value
-        proof{
-            data
-        }
-    }
-}

packages/mobymask-watcher/src/gql/queries/index.ts

@@ -3,7 +3,6 @@ import path from 'path';
 
 export const events = fs.readFileSync(path.join(__dirname, 'events.gql'), 'utf8');
 export const eventsInRange = fs.readFileSync(path.join(__dirname, 'eventsInRange.gql'), 'utf8');
-export const domainHash = fs.readFileSync(path.join(__dirname, 'domainHash.gql'), 'utf8');
 export const multiNonce = fs.readFileSync(path.join(__dirname, 'multiNonce.gql'), 'utf8');
 export const _owner = fs.readFileSync(path.join(__dirname, '_owner.gql'), 'utf8');
 export const isRevoked = fs.readFileSync(path.join(__dirname, 'isRevoked.gql'), 'utf8');

packages/mobymask-watcher/src/indexer.ts

@@ -45,7 +45,6 @@ import { IsPhisher } from './entity/IsPhisher';
 import { IsRevoked } from './entity/IsRevoked';
 import { _Owner } from './entity/_Owner';
 import { MultiNonce } from './entity/MultiNonce';
-import { DomainHash } from './entity/DomainHash';
 
 const log = debug('vulcanize:indexer');
 
@@ -56,8 +55,6 @@ const MEMBERSTATUSUPDATED_EVENT = 'MemberStatusUpdated';
 const OWNERSHIPTRANSFERRED_EVENT = 'OwnershipTransferred';
 const PHISHERSTATUSUPDATED_EVENT = 'PhisherStatusUpdated';
 
-const MAX_EVENTS_BLOCK_RANGE = -1;
-
 export type ResultEvent = {
   block: {
     cid: string;
@@ -109,9 +106,6 @@ export class Indexer implements IPLDIndexerInterface {
 
   _ipfsClient: IPFSClient
 
-  _entityTypesMap: Map<string, { [key: string]: string }>
-  _relationsMap: Map<any, { [key: string]: any }>
-
   constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, ethProvider: JsonRpcProvider, jobQueue: JobQueue) {
     assert(db);
     assert(ethClient);
@@ -137,13 +131,9 @@ export class Indexer implements IPLDIndexerInterface {
     assert(PhisherRegistryStorageLayout);
     this._storageLayoutMap.set(KIND_PHISHERREGISTRY, PhisherRegistryStorageLayout);
     this._contractMap.set(KIND_PHISHERREGISTRY, new ethers.utils.Interface(PhisherRegistryABI));
-
-    this._entityTypesMap = new Map();
-
-    this._relationsMap = new Map();
   }
 
-  get serverConfig () {
+  get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
@@ -207,37 +197,6 @@ export class Indexer implements IPLDIndexerInterface {
     };
   }
 
-  async domainHash (blockHash: string, contractAddress: string, diff = false): Promise<ValueResult> {
-    let entity = await this._db.getDomainHash({ blockHash, contractAddress });
-
-    if (entity) {
-      log('domainHash: db hit.');
-    } else {
-      log('domainHash: db miss, fetching from upstream server');
-
-      entity = await this._getStorageEntity(
-        blockHash,
-        contractAddress,
-        DomainHash,
-        'domainHash',
-        {},
-        ''
-      );
-
-      await this._db.saveDomainHash(entity);
-
-      if (diff) {
-        const stateUpdate = updateStateForElementaryType({}, 'domainHash', entity.value.toString());
-        await this.createDiffStaged(contractAddress, blockHash, stateUpdate);
-      }
-    }
-
-    return {
-      value: entity.value,
-      proof: JSON.parse(entity.proof)
-    };
-  }
-
   async multiNonce (blockHash: string, contractAddress: string, key0: string, key1: bigint, diff = false): Promise<ValueResult> {
     let entity = await this._db.getMultiNonce({ blockHash, contractAddress, key0, key1 });
 
@@ -740,7 +699,7 @@ export class Indexer implements IPLDIndexerInterface {
   }
 
   async getEventsInRange (fromBlockNumber: number, toBlockNumber: number): Promise<Array<Event>> {
-    return this._baseIndexer.getEventsInRange(fromBlockNumber, toBlockNumber, MAX_EVENTS_BLOCK_RANGE);
+    return this._baseIndexer.getEventsInRange(fromBlockNumber, toBlockNumber, this._serverConfig.maxEventsBlockRange);
   }
 
   async getSyncStatus (): Promise<SyncStatus | undefined> {
@@ -820,49 +779,48 @@ export class Indexer implements IPLDIndexerInterface {
     return this._contractMap.get(kind);
   }
 
-  getEntityTypesMap (): Map<string, { [key: string]: string }> {
-    return this._entityTypesMap;
-  }
-
   async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial<BlockProgress>): Promise<BlockProgress> {
     assert(blockHash);
-    let block: any, logs: any[];
+    const transactionsPromise = this._ethClient.getBlockWithTransactions({ blockHash });
+    const blockPromise = this._ethClient.getBlockByHash(blockHash);
+    let logs: any[];
 
     if (this._serverConfig.filterLogs) {
       const watchedContracts = this._baseIndexer.getWatchedContracts();
 
       // TODO: Query logs by multiple contracts.
-      const contractlogsWithBlockPromises = watchedContracts.map((watchedContract): Promise<any> => this._ethClient.getLogs({
+      const contractlogsPromises = watchedContracts.map((watchedContract): Promise<any> => this._ethClient.getLogs({
         blockHash,
         contract: watchedContract.address
       }));
 
-      const contractlogsWithBlock = await Promise.all(contractlogsWithBlockPromises);
+      const contractlogs = await Promise.all(contractlogsPromises);
 
       // Flatten logs by contract and sort by index.
-      logs = contractlogsWithBlock.map(data => {
+      logs = contractlogs.map(data => {
         return data.logs;
       }).flat()
         .sort((a, b) => {
           return a.index - b.index;
         });
-
-      ({ block } = await this._ethClient.getBlockByHash(blockHash));
     } else {
-      ({ block, logs } = await this._ethClient.getLogs({ blockHash }));
+      ({ logs } = await this._ethClient.getLogs({ blockHash }));
     }
 
-    const {
-      allEthHeaderCids: {
-        nodes: [
-          {
-            ethTransactionCidsByHeaderId: {
-              nodes: transactions
+    let [
+      { block },
+      {
+        allEthHeaderCids: {
+          nodes: [
+            {
+              ethTransactionCidsByHeaderId: {
+                nodes: transactions
+              }
             }
-          }
-        ]
+          ]
+        }
       }
-    } = await this._ethClient.getBlockWithTransactions({ blockHash });
+    ] = await Promise.all([blockPromise, transactionsPromise]);
 
     const transactionMap = transactions.reduce((acc: {[key: string]: any}, transaction: {[key: string]: any}) => {
       acc[transaction.txHash] = transaction;

packages/mobymask-watcher/src/resolvers.ts

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { ValueResult, BlockHeight, StateKind } from '@vulcanize/util';
+import { ValueResult, StateKind } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -58,11 +58,6 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
     },
 
     Query: {
-      domainHash: (_: any, { blockHash, contractAddress }: { blockHash: string, contractAddress: string }): Promise<ValueResult> => {
-        log('domainHash', blockHash, contractAddress);
-        return indexer.domainHash(blockHash, contractAddress);
-      },
-
       multiNonce: (_: any, { blockHash, contractAddress, key0, key1 }: { blockHash: string, contractAddress: string, key0: string, key1: bigint }): Promise<ValueResult> => {
         log('multiNonce', blockHash, contractAddress, key0, key1);
         return indexer.multiNonce(blockHash, contractAddress, key0, key1);

packages/mobymask-watcher/src/schema.gql

@@ -90,7 +90,6 @@ type ResultIPLDBlock {
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
-  domainHash(blockHash: String!, contractAddress: String!): ResultString!
   multiNonce(blockHash: String!, contractAddress: String!, key0: String!, key1: BigInt!): ResultBigInt!
   _owner(blockHash: String!, contractAddress: String!): ResultString!
   isRevoked(blockHash: String!, contractAddress: String!, key0: String!): ResultBoolean!