Indexing to get traces given address (#79)

* Build reverse index from address to traces.

* Create reverse index from address to traces.

packages/address-watcher/environments/local.toml

@@ -10,7 +10,7 @@
   username = "postgres"
   password = "postgres"
   synchronize = true
-  logging = false
+  logging = true
 
   entities = [ "src/entity/**/*.ts" ]
   migrations = [ "src/migration/**/*.ts" ]
@@ -24,7 +24,7 @@
 [upstream]
   gqlEndpoint = "http://127.0.0.1:8083/graphql"
   gqlSubscriptionEndpoint = "http://127.0.0.1:5000/graphql"
-  traceProviderEndpoint = "http://127.0.0.1:8545"
+  traceProviderEndpoint = "http://127.0.0.1:9545"
 
   [upstream.cache]
     name = "requests"

packages/address-watcher/src/cli/watch-address.ts

@@ -40,6 +40,6 @@ import { Database } from '../database';
   // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
   const address = ethers.utils.getAddress(argv.address);
 
-  await db.saveAddress(address, argv.startingBlock);
+  await db.saveAccount(address, argv.startingBlock);
   await db.close();
 })();

packages/address-watcher/src/database.ts

@@ -2,7 +2,7 @@ import assert from 'assert';
 import { Connection, ConnectionOptions, createConnection, DeepPartial } from 'typeorm';
 import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
 
-import { Address } from './entity/Address';
+import { Account } from './entity/Account';
 import { Trace } from './entity/Trace';
 
 export class Database {
@@ -28,7 +28,7 @@ export class Database {
   }
 
   async isWatchedAddress (address: string): Promise<boolean> {
-    const numRows = await this._conn.getRepository(Address)
+    const numRows = await this._conn.getRepository(Account)
       .createQueryBuilder()
       .where('address = :address', { address })
       .getCount();
@@ -36,9 +36,9 @@ export class Database {
     return numRows > 0;
   }
 
-  async saveAddress (address: string, startingBlock: number): Promise<void> {
+  async saveAccount (address: string, startingBlock: number): Promise<void> {
     await this._conn.transaction(async (tx) => {
-      const repo = tx.getRepository(Address);
+      const repo = tx.getRepository(Account);
 
       const numRows = await repo
         .createQueryBuilder()
@@ -46,12 +46,19 @@ export class Database {
         .getCount();
 
       if (numRows === 0) {
-        const entity = repo.create({ address, startingBlock: BigInt(startingBlock) });
+        const entity = repo.create({ address, startingBlock });
         await repo.save(entity);
       }
     });
   }
 
+  async getAccount (address: string): Promise<Account | undefined> {
+    return this._conn.getRepository(Account)
+      .createQueryBuilder()
+      .where('address = :address', { address })
+      .getOne();
+  }
+
   async getTrace (txHash: string): Promise<Trace | undefined> {
     return this._conn.getRepository(Trace)
       .createQueryBuilder('trace')
@@ -64,4 +71,18 @@ export class Database {
     const entity = repo.create({ txHash, blockNumber, blockHash, trace });
     return repo.save(entity);
   }
+
+  async saveTraceEntity (trace: Trace): Promise<Trace> {
+    const repo = this._conn.getRepository(Trace);
+    return repo.save(trace);
+  }
+
+  async getAppearances (address: string, fromBlockNumber: number, toBlockNumber: number): Promise<Trace[]> {
+    return this._conn.getRepository(Trace)
+      .createQueryBuilder('trace')
+      .leftJoinAndSelect('trace.accounts', 'account')
+      .where('address = :address AND block_number >= :fromBlockNumber AND block_number <= :toBlockNumber', { address, fromBlockNumber, toBlockNumber })
+      .orderBy({ block_number: 'ASC' })
+      .getMany();
+  }
 }

packages/address-watcher/src/entity/Account.ts

@@ -0,0 +1,15 @@
+import { Entity, PrimaryColumn, Column, ManyToMany, JoinTable } from 'typeorm';
+import { Trace } from './Trace';
+
+@Entity()
+export class Account {
+  @PrimaryColumn('varchar', { length: 42 })
+  address!: string;
+
+  @Column('numeric')
+  startingBlock!: number;
+
+  @ManyToMany(() => Trace, trace => trace.accounts)
+  @JoinTable()
+  appearances: Trace[];
+}

packages/address-watcher/src/entity/Address.ts

@@ -1,14 +0,0 @@
-import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
-
-@Entity()
-@Index(['address'], { unique: true })
-export class Address {
-  @PrimaryGeneratedColumn()
-  id!: number;
-
-  @Column('varchar', { length: 42 })
-  address!: string;
-
-  @Column('numeric')
-  startingBlock!: bigint;
-}

packages/address-watcher/src/entity/Trace.ts

@@ -1,12 +1,12 @@
-import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index, ManyToMany } from 'typeorm';
+
+import { Account } from './Account';
 
 @Entity()
 @Index(['txHash'], { unique: true })
+@Index(['blockNumber'])
 export class Trace {
-  @PrimaryGeneratedColumn()
+  @PrimaryColumn('varchar', { length: 66 })
-  id!: number;
-
-  @Column('varchar', { length: 66 })
   txHash!: string;
 
   @Column('numeric')
@@ -17,4 +17,7 @@ export class Trace {
 
   @Column('text')
   trace!: string;
+
+  @ManyToMany(() => Account, account => account.appearances, { cascade: ['insert'] })
+  accounts: Account[]
 }

packages/address-watcher/src/indexer.ts

@@ -2,15 +2,30 @@ import assert from 'assert';
 import debug from 'debug';
 import { ethers } from 'ethers';
 import { PubSub } from 'apollo-server-express';
+import _ from 'lodash';
 
 import { EthClient } from '@vulcanize/ipld-eth-client';
 import { GetStorageAt } from '@vulcanize/solidity-mapper';
 import { TracingClient } from '@vulcanize/tracing-client';
 
 import { Database } from './database';
+import { Trace } from './entity/Trace';
+import { Account } from './entity/Account';
 
 const log = debug('vulcanize:indexer');
 
+const addressesIn = (obj: any): any => {
+  if (!obj) {
+    return [];
+  }
+
+  if (_.isArray(obj)) {
+    return _.map(obj, addressesIn);
+  }
+
+  return [obj.from, obj.to, ...addressesIn(obj.calls)];
+};
+
 export class Indexer {
   _db: Database
   _ethClient: EthClient
@@ -43,7 +58,7 @@ export class Indexer {
 
   async watchAddress (address: string, startingBlock: number): Promise<boolean> {
     // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
-    await this._db.saveAddress(ethers.utils.getAddress(address), startingBlock);
+    await this._db.saveAccount(ethers.utils.getAddress(address), startingBlock);
 
     return true;
   }
@@ -64,6 +79,8 @@ export class Indexer {
         blockHash: tx.blockHash,
         trace: JSON.stringify(trace)
       });
+
+      await this.indexAppearances(entity);
     }
 
     return {
@@ -73,4 +90,23 @@ export class Indexer {
       trace: entity.trace
     };
   }
+
+  async getAppearances (address: string, fromBlockNumber: number, toBlockNumber: number): Promise<Trace[]> {
+    return this._db.getAppearances(address, fromBlockNumber, toBlockNumber);
+  }
+
+  async indexAppearances (trace: Trace): Promise<Trace> {
+    const traceObj = JSON.parse(trace.trace);
+    const addresses = _.uniq(_.compact(_.flattenDeep(addressesIn(traceObj)))).sort();
+
+    trace.accounts = _.map(addresses, address => {
+      const account = new Account();
+      account.address = address || '';
+      account.startingBlock = trace.blockNumber;
+
+      return account;
+    });
+
+    return await this._db.saveTraceEntity(trace);
+  }
 }

packages/address-watcher/src/resolvers.ts

@@ -5,6 +5,17 @@ import { Indexer } from './indexer';
 
 const log = debug('vulcanize:resolver');
 
+interface WatchAddressParams {
+  address: string,
+  startingBlock: number
+}
+
+interface AppearanceParams {
+  address: string,
+  fromBlockNumber: number,
+  toBlockNumber: number
+}
+
 export const createResolvers = async (indexer: Indexer): Promise<any> => {
   assert(indexer);
 
@@ -16,13 +27,18 @@ export const createResolvers = async (indexer: Indexer): Promise<any> => {
     },
 
     Mutation: {
-      watchAddress: (_: any, { address, startingBlock = 1 }: { address: string, startingBlock: number }): Promise<boolean> => {
+      watchAddress: (_: any, { address, startingBlock = 1 }: WatchAddressParams): Promise<boolean> => {
         log('watchAddress', address, startingBlock);
         return indexer.watchAddress(address, startingBlock);
       }
     },
 
     Query: {
+      appearances: async (_: any, { address, fromBlockNumber, toBlockNumber }: AppearanceParams): Promise<any> => {
+        log('appearances', address, fromBlockNumber, toBlockNumber);
+        return indexer.getAppearances(address, fromBlockNumber, toBlockNumber);
+      },
+
       traceTx: async (_: any, { txHash }: { txHash: string }): Promise<any> => {
         log('traceTx', txHash);
         return indexer.traceTx(txHash);

packages/address-watcher/src/types/common/main.d.ts

@@ -1,2 +1,3 @@
 // https://medium.com/@steveruiz/using-a-javascript-library-without-type-declarations-in-a-typescript-project-3643490015f3
 declare module 'canonical-json'
+declare module 'lodash-contrib';

packages/address-watcher/tsconfig.json

@@ -30,7 +30,7 @@
     // "strictNullChecks": true,                    /* Enable strict null checks. */
     // "strictFunctionTypes": true,                 /* Enable strict checking of function types. */
     // "strictBindCallApply": true,                 /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
-    // "strictPropertyInitialization": true,        /* Enable strict checking of property initialization in classes. */
+    "strictPropertyInitialization": false,          /* Enable strict checking of property initialization in classes. */
     // "noImplicitThis": true,                      /* Raise error on 'this' expressions with an implied 'any' type. */
     // "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
 

packages/erc20-watcher/src/database.ts

@@ -151,7 +151,7 @@ export class Database {
         .getCount();
 
       if (numRows === 0) {
-        const entity = repo.create({ address, startingBlock: BigInt(startingBlock) });
+        const entity = repo.create({ address, startingBlock });
         await repo.save(entity);
       }
     });

packages/erc20-watcher/src/entity/Contract.ts

@@ -10,5 +10,5 @@ export class Contract {
   address!: string;
 
   @Column('numeric')
-  startingBlock!: bigint;
+  startingBlock!: number;
 }