Indexing to get traces given address (#79)

* Build reverse index from address to traces.

* Create reverse index from address to traces.
This commit is contained in:
Ashwin Phatak 2021-06-21 15:38:36 +05:30 committed by GitHub
parent c9bf002675
commit eea69fe4d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 110 additions and 32 deletions

View File

@ -10,7 +10,7 @@
username = "postgres"
password = "postgres"
synchronize = true
logging = false
logging = true
entities = [ "src/entity/**/*.ts" ]
migrations = [ "src/migration/**/*.ts" ]
@ -24,7 +24,7 @@
[upstream]
gqlEndpoint = "http://127.0.0.1:8083/graphql"
gqlSubscriptionEndpoint = "http://127.0.0.1:5000/graphql"
traceProviderEndpoint = "http://127.0.0.1:8545"
traceProviderEndpoint = "http://127.0.0.1:9545"
[upstream.cache]
name = "requests"

View File

@ -40,6 +40,6 @@ import { Database } from '../database';
// Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
const address = ethers.utils.getAddress(argv.address);
await db.saveAddress(address, argv.startingBlock);
await db.saveAccount(address, argv.startingBlock);
await db.close();
})();

View File

@ -2,7 +2,7 @@ import assert from 'assert';
import { Connection, ConnectionOptions, createConnection, DeepPartial } from 'typeorm';
import { SnakeNamingStrategy } from 'typeorm-naming-strategies';
import { Address } from './entity/Address';
import { Account } from './entity/Account';
import { Trace } from './entity/Trace';
export class Database {
@ -28,7 +28,7 @@ export class Database {
}
async isWatchedAddress (address: string): Promise<boolean> {
const numRows = await this._conn.getRepository(Address)
const numRows = await this._conn.getRepository(Account)
.createQueryBuilder()
.where('address = :address', { address })
.getCount();
@ -36,9 +36,9 @@ export class Database {
return numRows > 0;
}
async saveAddress (address: string, startingBlock: number): Promise<void> {
async saveAccount (address: string, startingBlock: number): Promise<void> {
await this._conn.transaction(async (tx) => {
const repo = tx.getRepository(Address);
const repo = tx.getRepository(Account);
const numRows = await repo
.createQueryBuilder()
@ -46,12 +46,19 @@ export class Database {
.getCount();
if (numRows === 0) {
const entity = repo.create({ address, startingBlock: BigInt(startingBlock) });
const entity = repo.create({ address, startingBlock });
await repo.save(entity);
}
});
}
async getAccount (address: string): Promise<Account | undefined> {
return this._conn.getRepository(Account)
.createQueryBuilder()
.where('address = :address', { address })
.getOne();
}
async getTrace (txHash: string): Promise<Trace | undefined> {
return this._conn.getRepository(Trace)
.createQueryBuilder('trace')
@ -64,4 +71,18 @@ export class Database {
const entity = repo.create({ txHash, blockNumber, blockHash, trace });
return repo.save(entity);
}
async saveTraceEntity (trace: Trace): Promise<Trace> {
const repo = this._conn.getRepository(Trace);
return repo.save(trace);
}
async getAppearances (address: string, fromBlockNumber: number, toBlockNumber: number): Promise<Trace[]> {
return this._conn.getRepository(Trace)
.createQueryBuilder('trace')
.leftJoinAndSelect('trace.accounts', 'account')
.where('address = :address AND block_number >= :fromBlockNumber AND block_number <= :toBlockNumber', { address, fromBlockNumber, toBlockNumber })
.orderBy({ block_number: 'ASC' })
.getMany();
}
}

View File

@ -0,0 +1,15 @@
import { Entity, PrimaryColumn, Column, ManyToMany, JoinTable } from 'typeorm';
import { Trace } from './Trace';
@Entity()
export class Account {
@PrimaryColumn('varchar', { length: 42 })
address!: string;
@Column('numeric')
startingBlock!: number;
@ManyToMany(() => Trace, trace => trace.accounts)
@JoinTable()
appearances: Trace[];
}

View File

@ -1,14 +0,0 @@
import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
@Entity()
@Index(['address'], { unique: true })
export class Address {
@PrimaryGeneratedColumn()
id!: number;
@Column('varchar', { length: 42 })
address!: string;
@Column('numeric')
startingBlock!: bigint;
}

View File

@ -1,12 +1,12 @@
import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index, ManyToMany } from 'typeorm';
import { Account } from './Account';
@Entity()
@Index(['txHash'], { unique: true })
@Index(['blockNumber'])
export class Trace {
@PrimaryGeneratedColumn()
id!: number;
@Column('varchar', { length: 66 })
@PrimaryColumn('varchar', { length: 66 })
txHash!: string;
@Column('numeric')
@ -17,4 +17,7 @@ export class Trace {
@Column('text')
trace!: string;
@ManyToMany(() => Account, account => account.appearances, { cascade: ['insert'] })
accounts: Account[]
}

View File

@ -2,15 +2,30 @@ import assert from 'assert';
import debug from 'debug';
import { ethers } from 'ethers';
import { PubSub } from 'apollo-server-express';
import _ from 'lodash';
import { EthClient } from '@vulcanize/ipld-eth-client';
import { GetStorageAt } from '@vulcanize/solidity-mapper';
import { TracingClient } from '@vulcanize/tracing-client';
import { Database } from './database';
import { Trace } from './entity/Trace';
import { Account } from './entity/Account';
const log = debug('vulcanize:indexer');
const addressesIn = (obj: any): any => {
if (!obj) {
return [];
}
if (_.isArray(obj)) {
return _.map(obj, addressesIn);
}
return [obj.from, obj.to, ...addressesIn(obj.calls)];
};
export class Indexer {
_db: Database
_ethClient: EthClient
@ -43,7 +58,7 @@ export class Indexer {
async watchAddress (address: string, startingBlock: number): Promise<boolean> {
// Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
await this._db.saveAddress(ethers.utils.getAddress(address), startingBlock);
await this._db.saveAccount(ethers.utils.getAddress(address), startingBlock);
return true;
}
@ -64,6 +79,8 @@ export class Indexer {
blockHash: tx.blockHash,
trace: JSON.stringify(trace)
});
await this.indexAppearances(entity);
}
return {
@ -73,4 +90,23 @@ export class Indexer {
trace: entity.trace
};
}
async getAppearances (address: string, fromBlockNumber: number, toBlockNumber: number): Promise<Trace[]> {
return this._db.getAppearances(address, fromBlockNumber, toBlockNumber);
}
async indexAppearances (trace: Trace): Promise<Trace> {
const traceObj = JSON.parse(trace.trace);
const addresses = _.uniq(_.compact(_.flattenDeep(addressesIn(traceObj)))).sort();
trace.accounts = _.map(addresses, address => {
const account = new Account();
account.address = address || '';
account.startingBlock = trace.blockNumber;
return account;
});
return await this._db.saveTraceEntity(trace);
}
}

View File

@ -5,6 +5,17 @@ import { Indexer } from './indexer';
const log = debug('vulcanize:resolver');
interface WatchAddressParams {
address: string,
startingBlock: number
}
interface AppearanceParams {
address: string,
fromBlockNumber: number,
toBlockNumber: number
}
export const createResolvers = async (indexer: Indexer): Promise<any> => {
assert(indexer);
@ -16,13 +27,18 @@ export const createResolvers = async (indexer: Indexer): Promise<any> => {
},
Mutation: {
watchAddress: (_: any, { address, startingBlock = 1 }: { address: string, startingBlock: number }): Promise<boolean> => {
watchAddress: (_: any, { address, startingBlock = 1 }: WatchAddressParams): Promise<boolean> => {
log('watchAddress', address, startingBlock);
return indexer.watchAddress(address, startingBlock);
}
},
Query: {
appearances: async (_: any, { address, fromBlockNumber, toBlockNumber }: AppearanceParams): Promise<any> => {
log('appearances', address, fromBlockNumber, toBlockNumber);
return indexer.getAppearances(address, fromBlockNumber, toBlockNumber);
},
traceTx: async (_: any, { txHash }: { txHash: string }): Promise<any> => {
log('traceTx', txHash);
return indexer.traceTx(txHash);

View File

@ -1,2 +1,3 @@
// https://medium.com/@steveruiz/using-a-javascript-library-without-type-declarations-in-a-typescript-project-3643490015f3
declare module 'canonical-json'
declare module 'lodash-contrib';

View File

@ -30,7 +30,7 @@
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
"strictPropertyInitialization": false, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */

View File

@ -151,7 +151,7 @@ export class Database {
.getCount();
if (numRows === 0) {
const entity = repo.create({ address, startingBlock: BigInt(startingBlock) });
const entity = repo.create({ address, startingBlock });
await repo.save(entity);
}
});

View File

@ -10,5 +10,5 @@ export class Contract {
address!: string;
@Column('numeric')
startingBlock!: bigint;
startingBlock!: number;
}