Changes to use packages in uniswap-watcher (#196)

* Changes to use util from uniswap-watcher

* Refactor ResultIPLDBlock to util

* Verify state in compare CLI for uniswap multiple queries

* Prepare watcher-ts for publishing packages

* Fix verify state in compare CLI for multiple entities query

* Fix codegen util imports

packages/erc20-watcher/src/database.ts

@@ -6,7 +6,7 @@ import assert from 'assert';
 import { Connection, ConnectionOptions, DeepPartial, FindConditions, FindManyOptions, QueryRunner } from 'typeorm';
 import path from 'path';
 
-import { Database as BaseDatabase, QueryOptions, Where } from '@cerc-io/util';
+import { Database as BaseDatabase, DatabaseInterface, QueryOptions, StateKind, Where } from '@cerc-io/util';
 
 import { Allowance } from './entity/Allowance';
 import { Balance } from './entity/Balance';
@@ -14,8 +14,10 @@ import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';
 import { SyncStatus } from './entity/SyncStatus';
 import { BlockProgress } from './entity/BlockProgress';
+import { IPLDBlock } from './entity/IPLDBlock';
+import { IpldStatus } from './entity/IpldStatus';
 
-export class Database {
+export class Database implements DatabaseInterface {
   _config: ConnectionOptions
   _conn!: Connection
   _baseDatabase: BaseDatabase;
@@ -39,6 +41,47 @@ export class Database {
     return this._baseDatabase.close();
   }
 
+  getNewIPLDBlock (): IPLDBlock {
+    return new IPLDBlock();
+  }
+
+  async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
+    const repo = this._conn.getRepository(IPLDBlock);
+
+    return this._baseDatabase.getIPLDBlocks(repo, where);
+  }
+
+  async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
+    const repo = this._conn.getRepository(IPLDBlock);
+
+    return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
+  }
+
+  // Fetch all diff IPLDBlocks after the specified block number.
+  async getDiffIPLDBlocksInRange (contractAddress: string, startblock: number, endBlock: number): Promise<IPLDBlock[]> {
+    const repo = this._conn.getRepository(IPLDBlock);
+
+    return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startblock, endBlock);
+  }
+
+  async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
+    const repo = dbTx.manager.getRepository(IPLDBlock);
+
+    return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
+  }
+
+  async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
+    const repo = dbTx.manager.getRepository(IPLDBlock);
+
+    await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
+  }
+
+  async getIPLDStatus (): Promise<IpldStatus | undefined> {
+    const repo = this._conn.getRepository(IpldStatus);
+
+    return this._baseDatabase.getIPLDStatus(repo);
+  }
+
   async getBalance ({ blockHash, token, owner }: { blockHash: string, token: string, owner: string }): Promise<Balance | undefined> {
     return this._conn.getRepository(Balance)
       .createQueryBuilder('balance')
@@ -107,11 +150,17 @@ export class Database {
     return this._baseDatabase.getBlockEvents(repo, blockHash, where, queryOptions);
   }
 
-  async saveEvents (queryRunner: QueryRunner, block: DeepPartial<BlockProgress>, events: DeepPartial<Event>[]): Promise<BlockProgress> {
+  async saveBlockWithEvents (queryRunner: QueryRunner, block: DeepPartial<BlockProgress>, events: DeepPartial<Event>[]): Promise<BlockProgress> {
     const blockRepo = queryRunner.manager.getRepository(BlockProgress);
     const eventRepo = queryRunner.manager.getRepository(Event);
 
-    return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events);
+    return this._baseDatabase.saveBlockWithEvents(blockRepo, eventRepo, block, events);
+  }
+
+  async saveEvents (queryRunner: QueryRunner, events: Event[]): Promise<void> {
+    const eventRepo = queryRunner.manager.getRepository(Event);
+
+    return this._baseDatabase.saveEvents(eventRepo, events);
   }
 
   async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
@@ -173,6 +222,12 @@ export class Database {
     return this._baseDatabase.getBlockProgressEntities(repo, where, options);
   }
 
+  async saveBlockProgress (queryRunner: QueryRunner, block: DeepPartial<BlockProgress>): Promise<BlockProgress> {
+    const repo = queryRunner.manager.getRepository(BlockProgress);
+
+    return this._baseDatabase.saveBlockProgress(repo, block);
+  }
+
   async updateBlockProgress (queryRunner: QueryRunner, block: BlockProgress, lastProcessedEventIndex: number): Promise<BlockProgress> {
     const repo = queryRunner.manager.getRepository(BlockProgress);
 

packages/erc20-watcher/src/entity/IPLDBlock.ts

@@ -0,0 +1,36 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm';
+
+import { StateKind } from '@cerc-io/util';
+
+import { BlockProgress } from './BlockProgress';
+
+@Entity()
+@Index(['cid'], { unique: true })
+@Index(['block', 'contractAddress'])
+@Index(['block', 'contractAddress', 'kind'], { unique: true })
+export class IPLDBlock {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @ManyToOne(() => BlockProgress, { onDelete: 'CASCADE' })
+  block!: BlockProgress;
+
+  @Column('varchar', { length: 42 })
+  contractAddress!: string;
+
+  @Column('varchar')
+  cid!: string;
+
+  @Column({
+    type: 'enum',
+    enum: StateKind
+  })
+  kind!: StateKind;
+
+  @Column('bytea')
+  data!: Buffer;
+}

packages/erc20-watcher/src/entity/IpldStatus.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
+
+@Entity()
+export class IpldStatus {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @Column('integer')
+  latestHooksBlockNumber!: number;
+
+  @Column('integer', { nullable: true })
+  latestCheckpointBlockNumber!: number;
+
+  @Column('integer', { nullable: true })
+  latestIPFSBlockNumber!: number;
+}

packages/erc20-watcher/src/indexer.ts

@@ -12,7 +12,7 @@ import { BaseProvider } from '@ethersproject/providers';
 
 import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
-import { IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, JobQueue, Where, QueryOptions, ServerConfig } from '@cerc-io/util';
+import { IndexerInterface, Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, JobQueue, Where, QueryOptions, ServerConfig, IPFSClient, IpldStatus as IpldStatusInterface } from '@cerc-io/util';
 
 import { Database } from './database';
 import { Event } from './entity/Event';
@@ -21,6 +21,7 @@ import { SyncStatus } from './entity/SyncStatus';
 import artifacts from './artifacts/ERC20.json';
 import { BlockProgress } from './entity/BlockProgress';
 import { Contract } from './entity/Contract';
+import { IPLDBlock } from './entity/IPLDBlock';
 
 const log = debug('vulcanize:indexer');
 const JSONbigNative = JSONbig({ useNativeBigInt: true });
@@ -63,7 +64,8 @@ export class Indexer implements IndexerInterface {
     this._ethProvider = ethProvider;
     this._serverConfig = serverConfig;
     this._serverMode = serverConfig.mode;
-    this._baseIndexer = new BaseIndexer(this._db, this._ethClient, this._ethProvider, jobQueue);
+    const ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr);
+    this._baseIndexer = new BaseIndexer(serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue, ipfsClient);
 
     const { abi, storageLayout } = artifacts;
 
@@ -248,6 +250,10 @@ export class Indexer implements IndexerInterface {
     );
   }
 
+  getIPLDData (ipldBlock: IPLDBlock): any {
+    return this._baseIndexer.getIPLDData(ipldBlock);
+  }
+
   async triggerIndexingOnEvent (event: Event): Promise<void> {
     const { eventName, eventInfo, contract: token, block: { blockHash } } = event;
     const eventFields = JSON.parse(eventInfo);
@@ -278,6 +284,11 @@ export class Indexer implements IndexerInterface {
     await this.triggerIndexingOnEvent(event);
   }
 
+  async processBlock (blockProgress: BlockProgress): Promise<void> {
+    // Call a function to create initial state for contracts.
+    await this._baseIndexer.createInit(this, blockProgress.blockHash, blockProgress.blockNumber);
+  }
+
   parseEventNameAndArgs (kind: string, logObj: any): any {
     const { topics, data } = logObj;
     const logDescription = this._contract.parseLog({ data, topics });
@@ -291,7 +302,7 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
 
-  async isWatchedContract (address : string): Promise<Contract | undefined> {
+  isWatchedContract (address : string): Contract | undefined {
     return this._baseIndexer.isWatchedContract(address);
   }
 
@@ -299,6 +310,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
   }
 
+  async updateIPLDStatusMap (address: string, ipldStatus: IpldStatusInterface): Promise<void> {
+    await this._baseIndexer.updateIPLDStatusMap(address, ipldStatus);
+  }
+
   cacheContract (contract: Contract): void {
     return this._baseIndexer.cacheContract(contract);
   }
@@ -351,8 +366,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async fetchBlockEvents (block: DeepPartial<BlockProgress>): Promise<BlockProgress> {
-    return this._baseIndexer.fetchBlockEvents(block, this._fetchAndSaveEvents.bind(this));
+  async fetchBlockWithEvents (block: DeepPartial<BlockProgress>): Promise<BlockProgress> {
+    return this._baseIndexer.fetchBlockWithEvents(block, this._fetchAndSaveEvents.bind(this));
   }
 
   async getBlockEvents (blockHash: string, where: Where, queryOptions: QueryOptions): Promise<Array<Event>> {
@@ -451,7 +466,7 @@ export class Indexer implements IndexerInterface {
       };
 
       console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
-      const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
+      const blockProgress = await this._db.saveBlockWithEvents(dbTx, block, dbEvents);
       await dbTx.commitTransaction();
       console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');