From 6a33c704b8ac628032e4ab1f85a77f79c66683a8 Mon Sep 17 00:00:00 2001 From: Ashwin Phatak Date: Fri, 18 Jun 2021 17:04:02 +0530 Subject: [PATCH] Basic GQL API to get traces, save to db. (#76) --- .../address-watcher/environments/local.toml | 1 + packages/address-watcher/package.json | 2 +- packages/address-watcher/src/config.ts | 1 + packages/address-watcher/src/database.ts | 16 +++++++- packages/address-watcher/src/entity/Trace.ts | 20 ++++++++++ packages/address-watcher/src/indexer.ts | 36 +++++++++++++++-- packages/address-watcher/src/resolvers.ts | 7 +++- packages/address-watcher/src/schema.ts | 31 +++++++++++--- packages/address-watcher/src/server.ts | 8 +++- packages/tracing-client/index.ts | 1 + .../tracing-client/src/cli/get-call-trace.ts | 5 ++- .../tracing-client/src/cli/get-tx-trace.ts | 5 ++- packages/tracing-client/src/tracing.ts | 40 +++++++++++++++---- 13 files changed, 147 insertions(+), 26 deletions(-) create mode 100644 packages/address-watcher/src/entity/Trace.ts diff --git a/packages/address-watcher/environments/local.toml b/packages/address-watcher/environments/local.toml index 568d739a..14d9a926 100644 --- a/packages/address-watcher/environments/local.toml +++ b/packages/address-watcher/environments/local.toml @@ -24,6 +24,7 @@ [upstream] gqlEndpoint = "http://127.0.0.1:8083/graphql" gqlSubscriptionEndpoint = "http://127.0.0.1:5000/graphql" + traceProviderEndpoint = "http://127.0.0.1:8545" [upstream.cache] name = "requests" diff --git a/packages/address-watcher/package.json b/packages/address-watcher/package.json index c087aac7..6af8d75f 100644 --- a/packages/address-watcher/package.json +++ b/packages/address-watcher/package.json @@ -5,7 +5,6 @@ "private": true, "scripts": { "server": "DEBUG=vulcanize:* nodemon src/server.ts -f environments/local.toml", - "server:mock": "MOCK=1 nodemon src/server.ts -f environments/local.toml", "test": "mocha -r ts-node/register src/**/*.spec.ts", "lint": "eslint .", "build": "tsc" @@ -26,6 +25,7 @@ "@vulcanize/cache": "^0.1.0", "@vulcanize/ipld-eth-client": "^0.1.0", "@vulcanize/solidity-mapper": "^0.1.0", + "@vulcanize/tracing-client": "^0.1.0", "apollo-server-express": "^2.25.0", "apollo-type-bigint": "^0.1.3", "debug": "^4.3.1", diff --git a/packages/address-watcher/src/config.ts b/packages/address-watcher/src/config.ts index b93cd8f0..2996634e 100644 --- a/packages/address-watcher/src/config.ts +++ b/packages/address-watcher/src/config.ts @@ -17,6 +17,7 @@ export interface Config { upstream: { gqlEndpoint: string; gqlSubscriptionEndpoint: string; + traceProviderEndpoint: string; cache: CacheConfig } } diff --git a/packages/address-watcher/src/database.ts b/packages/address-watcher/src/database.ts index fb4253bb..11cc9492 100644 --- a/packages/address-watcher/src/database.ts +++ b/packages/address-watcher/src/database.ts @@ -1,8 +1,9 @@ import assert from 'assert'; -import { Connection, ConnectionOptions, createConnection } from 'typeorm'; +import { Connection, ConnectionOptions, createConnection, DeepPartial } from 'typeorm'; import { SnakeNamingStrategy } from 'typeorm-naming-strategies'; import { Address } from './entity/Address'; +import { Trace } from './entity/Trace'; export class Database { _config: ConnectionOptions @@ -50,4 +51,17 @@ export class Database { } }); } + + async getTrace (txHash: string): Promise { + return this._conn.getRepository(Trace) + .createQueryBuilder('trace') + .where('tx_hash = :txHash', { txHash }) + .getOne(); + } + + async saveTrace ({ txHash, blockNumber, blockHash, trace }: DeepPartial): Promise { + const repo = this._conn.getRepository(Trace); + const entity = repo.create({ txHash, blockNumber, blockHash, trace }); + return repo.save(entity); + } } diff --git a/packages/address-watcher/src/entity/Trace.ts b/packages/address-watcher/src/entity/Trace.ts new file mode 100644 index 00000000..b6d05876 --- /dev/null +++ b/packages/address-watcher/src/entity/Trace.ts @@ -0,0 +1,20 @@ +import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm'; + +@Entity() +@Index(['txHash'], { unique: true }) +export class Trace { + @PrimaryGeneratedColumn() + id!: number; + + @Column('varchar', { length: 66 }) + txHash!: string; + + @Column('numeric') + blockNumber!: number; + + @Column('varchar', { length: 66 }) + blockHash!: string; + + @Column('text') + trace!: string; +} diff --git a/packages/address-watcher/src/indexer.ts b/packages/address-watcher/src/indexer.ts index 6ea6c583..da0d96eb 100644 --- a/packages/address-watcher/src/indexer.ts +++ b/packages/address-watcher/src/indexer.ts @@ -1,30 +1,34 @@ import assert from 'assert'; -// import debug from 'debug'; +import debug from 'debug'; import { ethers } from 'ethers'; import { PubSub } from 'apollo-server-express'; import { EthClient } from '@vulcanize/ipld-eth-client'; import { GetStorageAt } from '@vulcanize/solidity-mapper'; +import { TracingClient } from '@vulcanize/tracing-client'; import { Database } from './database'; -// const log = debug('vulcanize:indexer'); +const log = debug('vulcanize:indexer'); export class Indexer { _db: Database _ethClient: EthClient _pubsub: PubSub _getStorageAt: GetStorageAt + _tracingClient: TracingClient - constructor (db: Database, ethClient: EthClient, pubsub: PubSub) { + constructor (db: Database, ethClient: EthClient, pubsub: PubSub, tracingClient: TracingClient) { assert(db); assert(ethClient); assert(pubsub); + assert(tracingClient); this._db = db; this._ethClient = ethClient; this._pubsub = pubsub; this._getStorageAt = this._ethClient.getStorageAt.bind(this._ethClient); + this._tracingClient = tracingClient; } getEventIterator (): AsyncIterator { @@ -43,4 +47,30 @@ export class Indexer { return true; } + + async traceTx (txHash: string): Promise { + let entity = await this._db.getTrace(txHash); + if (entity) { + log('traceTx: db hit'); + } else { + log('traceTx: db miss, fetching from tracing API server'); + + const tx = await this._tracingClient.getTx(txHash); + const trace = await this._tracingClient.getTxTrace(txHash, 'callTraceWithAddresses', '15s'); + + entity = await this._db.saveTrace({ + txHash, + blockNumber: tx.blockNumber, + blockHash: tx.blockHash, + trace: JSON.stringify(trace) + }); + } + + return { + txHash, + blockNumber: entity.blockNumber, + blockHash: entity.blockHash, + trace: entity.trace + }; + } } diff --git a/packages/address-watcher/src/resolvers.ts b/packages/address-watcher/src/resolvers.ts index 1f0260f2..557e1d63 100644 --- a/packages/address-watcher/src/resolvers.ts +++ b/packages/address-watcher/src/resolvers.ts @@ -22,6 +22,11 @@ export const createResolvers = async (indexer: Indexer): Promise => { } }, - Query: {} + Query: { + traceTx: async (_: any, { txHash }: { txHash: string }): Promise => { + log('traceTx', txHash); + return indexer.traceTx(txHash); + } + } }; }; diff --git a/packages/address-watcher/src/schema.ts b/packages/address-watcher/src/schema.ts index f9c81ea0..a78591de 100644 --- a/packages/address-watcher/src/schema.ts +++ b/packages/address-watcher/src/schema.ts @@ -3,11 +3,17 @@ import { gql } from '@apollo/client/core'; export default gql` # Types -# Watched event, include additional context over and above the event data. -type WatchedEvent { - blockHash: String! +type TxTrace { txHash: String! + blockNumber: Int! + blockHash: String! + trace: String! +} + +# Watched address event, include additional context over and above the event data. +type WatchedAddressEvent { address: String! + txTrace: TxTrace! } # @@ -16,10 +22,23 @@ type WatchedEvent { type Query { - queryAppearances( + # + # Developer API methods + # + + appearances( address: String! + fromBlockNumber: Int! + toBlockNumber: Int! + ): [TxTrace!] + + # + # Low level utility methods + # + + traceTx( txHash: String! - ): [String!] + ): TxTrace } # @@ -28,7 +47,7 @@ type Query { type Subscription { # Watch for token events (at head of chain). - onAddressEvent: WatchedEvent! + onAddressEvent: WatchedAddressEvent! } # diff --git a/packages/address-watcher/src/server.ts b/packages/address-watcher/src/server.ts index 676261f0..dd44f6e8 100644 --- a/packages/address-watcher/src/server.ts +++ b/packages/address-watcher/src/server.ts @@ -10,6 +10,7 @@ import { createServer } from 'http'; import { getCache } from '@vulcanize/cache'; import { EthClient } from '@vulcanize/ipld-eth-client'; +import { TracingClient } from '@vulcanize/tracing-client'; import typeDefs from './schema'; @@ -44,18 +45,21 @@ export const main = async (): Promise => { await db.init(); assert(upstream, 'Missing upstream config'); - const { gqlEndpoint, gqlSubscriptionEndpoint, cache: cacheConfig } = upstream; + const { gqlEndpoint, gqlSubscriptionEndpoint, traceProviderEndpoint, cache: cacheConfig } = upstream; assert(gqlEndpoint, 'Missing upstream gqlEndpoint'); assert(gqlSubscriptionEndpoint, 'Missing upstream gqlSubscriptionEndpoint'); + assert(traceProviderEndpoint, 'Missing upstream traceProviderEndpoint'); const cache = await getCache(cacheConfig); const ethClient = new EthClient({ gqlEndpoint, gqlSubscriptionEndpoint, cache }); + const tracingClient = new TracingClient(traceProviderEndpoint); + // Note: In-memory pubsub works fine for now, as each watcher is a single process anyway. // Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries const pubsub = new PubSub(); - const indexer = new Indexer(db, ethClient, pubsub); + const indexer = new Indexer(db, ethClient, pubsub, tracingClient); const resolvers = await createResolvers(indexer); diff --git a/packages/tracing-client/index.ts b/packages/tracing-client/index.ts index e69de29b..c0cbe92b 100644 --- a/packages/tracing-client/index.ts +++ b/packages/tracing-client/index.ts @@ -0,0 +1 @@ +export * from './src/tracing'; diff --git a/packages/tracing-client/src/cli/get-call-trace.ts b/packages/tracing-client/src/cli/get-call-trace.ts index 7ef05e29..7a72d5ab 100644 --- a/packages/tracing-client/src/cli/get-call-trace.ts +++ b/packages/tracing-client/src/cli/get-call-trace.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import yargs from 'yargs'; -import { getCallTrace } from '../tracing'; +import { TracingClient } from '../tracing'; (async () => { const argv = await yargs.parserConfiguration({ @@ -45,7 +45,8 @@ import { getCallTrace } from '../tracing'; const txData = JSON.parse(fs.readFileSync(argv.txFile).toString("utf-8")); - const result = await getCallTrace(argv.providerUrl, argv.block, txData, tracer); + const tracingClient = new TracingClient(argv.providerUrl); + const result = await tracingClient.getCallTrace(argv.block, txData, tracer); console.log(JSON.stringify(result, null, 2)); })(); diff --git a/packages/tracing-client/src/cli/get-tx-trace.ts b/packages/tracing-client/src/cli/get-tx-trace.ts index 66ad9c35..386d2048 100644 --- a/packages/tracing-client/src/cli/get-tx-trace.ts +++ b/packages/tracing-client/src/cli/get-tx-trace.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import yargs from 'yargs'; -import { getTxTrace } from '../tracing'; +import { TracingClient } from '../tracing'; (async () => { const argv = await yargs.parserConfiguration({ @@ -42,7 +42,8 @@ import { getTxTrace } from '../tracing'; tracer = fs.readFileSync(tracerFile).toString("utf-8"); } - const result = await getTxTrace(argv.providerUrl, argv.txHash, tracer, argv.timeout); + const tracingClient = new TracingClient(argv.providerUrl); + const result = await tracingClient.getTxTrace(argv.txHash, tracer, argv.timeout); console.log(JSON.stringify(result, null, 2)); })(); diff --git a/packages/tracing-client/src/tracing.ts b/packages/tracing-client/src/tracing.ts index e3338696..4f0e13b5 100644 --- a/packages/tracing-client/src/tracing.ts +++ b/packages/tracing-client/src/tracing.ts @@ -1,11 +1,35 @@ +import assert from 'assert'; +import path from 'path'; +import fs from 'fs'; import { ethers } from 'ethers'; -export const getTxTrace = async (providerUrl: string, txHash: string, tracer: string | undefined, timeout: string | undefined): Promise => { - const provider = new ethers.providers.JsonRpcProvider(providerUrl); - return provider.send('debug_traceTransaction', [txHash, { tracer, timeout }]); -}; +const callTracerWithAddresses = fs.readFileSync(path.join(__dirname, 'tracers', 'call_address_tracer.js')).toString("utf-8"); -export const getCallTrace = async (providerUrl: string, block: string, txData: any, tracer: string | undefined): Promise => { - const provider = new ethers.providers.JsonRpcProvider(providerUrl); - return provider.send('debug_traceCall', [ txData, block, { tracer }]); -}; +export class TracingClient { + + _providerUrl: string; + _provider: ethers.providers.JsonRpcProvider; + + constructor(providerUrl: string) { + assert(providerUrl); + + this._providerUrl = providerUrl; + this._provider = new ethers.providers.JsonRpcProvider(providerUrl); + } + + async getTx (txHash: string): Promise { + return this._provider.getTransaction(txHash); + } + + async getTxTrace (txHash: string, tracer: string | undefined, timeout: string | undefined): Promise { + if (tracer === 'callTraceWithAddresses') { + tracer = callTracerWithAddresses; + } + + return this._provider.send('debug_traceTransaction', [txHash, { tracer, timeout }]); + }; + + async getCallTrace (block: string, txData: any, tracer: string | undefined): Promise { + return this._provider.send('debug_traceCall', [ txData, block, { tracer }]); + }; +}