diff --git a/packages/cli/src/base.ts b/packages/cli/src/base.ts index 04bc2d4d..262a21c9 100644 --- a/packages/cli/src/base.ts +++ b/packages/cli/src/base.ts @@ -11,7 +11,6 @@ import { JsonRpcProvider } from '@ethersproject/providers'; import { Config, getConfig, - initClients, JobQueue, DatabaseInterface, IndexerInterface, @@ -21,6 +20,8 @@ import { GraphWatcherInterface } from '@cerc-io/util'; +import { initClients } from './utils/index'; + export class BaseCmd { _config?: Config; _clients?: Clients; diff --git a/packages/cli/src/utils/index.ts b/packages/cli/src/utils/index.ts index a35cdc9f..df9133ee 100644 --- a/packages/cli/src/utils/index.ts +++ b/packages/cli/src/utils/index.ts @@ -4,9 +4,15 @@ import fs from 'fs'; import path from 'path'; +import assert from 'assert'; +import { providers } from 'ethers'; // @ts-expect-error https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1319854183 import { PeerIdObj } from '@cerc-io/peer'; +import { Config, EthClient, getCustomProvider } from '@cerc-io/util'; +import { getCache } from '@cerc-io/cache'; +import { EthClient as GqlEthClient } from '@cerc-io/ipld-eth-client'; +import { EthClient as RpcEthClient } from '@cerc-io/rpc-eth-client'; export function readPeerId (filePath: string): PeerIdObj { const peerIdFilePath = path.resolve(filePath); @@ -15,3 +21,43 @@ export function readPeerId (filePath: string): PeerIdObj { const peerIdJson = fs.readFileSync(peerIdFilePath, 'utf-8'); return JSON.parse(peerIdJson); } + +export const initClients = async (config: Config): Promise<{ + ethClient: EthClient, + ethProvider: providers.JsonRpcProvider +}> => { + const { database: dbConfig, upstream: upstreamConfig, server: serverConfig } = config; + + assert(serverConfig, 'Missing server config'); + assert(dbConfig, 'Missing database config'); + assert(upstreamConfig, 'Missing upstream config'); + + const { ethServer: { gqlApiEndpoint, rpcProviderEndpoint, rpcClient = false }, cache: cacheConfig } = upstreamConfig; + + assert(rpcProviderEndpoint, 'Missing upstream ethServer.rpcProviderEndpoint'); + + const cache = await getCache(cacheConfig); + + let ethClient: EthClient; + + if (rpcClient) { + ethClient = new RpcEthClient({ + rpcEndpoint: rpcProviderEndpoint, + cache + }); + } else { + assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); + + ethClient = new GqlEthClient({ + gqlEndpoint: gqlApiEndpoint, + cache + }); + } + + const ethProvider = getCustomProvider(rpcProviderEndpoint); + + return { + ethClient, + ethProvider + }; +}; diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts index e5006af3..9f000a5d 100644 --- a/packages/ipld-eth-client/src/eth-client.ts +++ b/packages/ipld-eth-client/src/eth-client.ts @@ -23,13 +23,10 @@ interface Vars { } export class EthClient { - _config: Config; _graphqlClient: GraphQLClient; _cache: Cache | undefined; constructor (config: Config) { - this._config = config; - const { gqlEndpoint, gqlSubscriptionEndpoint, cache } = config; assert(gqlEndpoint, 'Missing gql endpoint'); diff --git a/packages/ipld-eth-client/src/utils.ts b/packages/ipld-eth-client/src/utils.ts index 7135c054..5ebb4784 100644 --- a/packages/ipld-eth-client/src/utils.ts +++ b/packages/ipld-eth-client/src/utils.ts @@ -19,7 +19,7 @@ export const getMappingSlot = (mappingSlot: string, key: string): string => { export const getStorageLeafKey = (slot: string): string => ethers.utils.keccak256(slot); -export const topictoAddress = (topic: string): string => { +export const topicToAddress = (topic: string): string => { return ethers.utils.getAddress( ethers.utils.hexZeroPad( ethers.utils.hexStripZeros(topic), 20 diff --git a/packages/rpc-eth-client/.eslintignore b/packages/rpc-eth-client/.eslintignore new file mode 100644 index 00000000..653874b5 --- /dev/null +++ b/packages/rpc-eth-client/.eslintignore @@ -0,0 +1,5 @@ +# Don't lint node_modules. +node_modules + +# Don't lint build output. +dist diff --git a/packages/rpc-eth-client/.eslintrc.json b/packages/rpc-eth-client/.eslintrc.json new file mode 100644 index 00000000..014cdab2 --- /dev/null +++ b/packages/rpc-eth-client/.eslintrc.json @@ -0,0 +1,32 @@ +{ + "env": { + "browser": true, + "es2021": true + }, + "extends": [ + "semistandard", + "plugin:@typescript-eslint/recommended" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "plugins": [ + "@typescript-eslint" + ], + "rules": { + "indent": ["error", 2, { "SwitchCase": 1 }], + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/explicit-module-boundary-types": [ + "warn", + { + "allowArgumentsExplicitlyTypedAsAny": true + } + ], + "@typescript-eslint/no-unused-vars": [ + "error", + { "ignoreRestSiblings": true } + ] + } +} diff --git a/packages/rpc-eth-client/.npmignore b/packages/rpc-eth-client/.npmignore new file mode 100644 index 00000000..2739a4b8 --- /dev/null +++ b/packages/rpc-eth-client/.npmignore @@ -0,0 +1,5 @@ +/src/ +index.ts +tsconfig.json +.eslintrc.json +.eslintignore diff --git a/packages/rpc-eth-client/index.ts b/packages/rpc-eth-client/index.ts new file mode 100644 index 00000000..43dd8abc --- /dev/null +++ b/packages/rpc-eth-client/index.ts @@ -0,0 +1,5 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +export * from './src/eth-client'; diff --git a/packages/rpc-eth-client/package.json b/packages/rpc-eth-client/package.json new file mode 100644 index 00000000..a1433026 --- /dev/null +++ b/packages/rpc-eth-client/package.json @@ -0,0 +1,44 @@ +{ + "name": "@cerc-io/rpc-eth-client", + "version": "0.2.50", + "description": "RPC ETH Client", + "main": "dist/index.js", + "scripts": { + "lint": "eslint .", + "test:rpc": "mocha -r ts-node/register src/**/*.test.ts", + "build": "tsc" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/cerc-io/watcher-ts.git" + }, + "author": "", + "license": "AGPL-3.0", + "bugs": { + "url": "https://github.com/cerc-io/watcher-ts/issues" + }, + "homepage": "https://github.com/cerc-io/watcher-ts#readme", + "dependencies": { + "@cerc-io/cache": "^0.2.50", + "@cerc-io/util": "^0.2.50", + "@cerc-io/ipld-eth-client": "^0.2.50", + "cross-fetch": "^3.1.4", + "debug": "^4.3.1", + "ethers": "^5.4.4", + "left-pad": "^1.3.0", + "ws": "^8.11.0", + "zen-observable-ts": "^1.1.0" + }, + "devDependencies": { + "@types/ws": "^8.5.3", + "@typescript-eslint/eslint-plugin": "^5.47.1", + "@typescript-eslint/parser": "^5.47.1", + "eslint": "^8.35.0", + "eslint-config-semistandard": "^15.0.1", + "eslint-config-standard": "^16.0.3", + "eslint-plugin-import": "^2.27.5", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-promise": "^5.1.0", + "eslint-plugin-standard": "^5.0.0" + } +} diff --git a/packages/rpc-eth-client/src/eth-client.test.ts b/packages/rpc-eth-client/src/eth-client.test.ts new file mode 100644 index 00000000..635f1941 --- /dev/null +++ b/packages/rpc-eth-client/src/eth-client.test.ts @@ -0,0 +1,232 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import { expect } from 'chai'; +import 'mocha'; + +import { EthClient as GqlEthClient } from '@cerc-io/ipld-eth-client'; + +import { EthClient } from '../index'; + +const RPC_ENDPOINT = 'http://localhost:8545'; +const GQL_ENDPOINT = 'http://localhost:8083/graphql'; + +const BLOCK_HASH = '0xef53edd41f1aca301d6dd285656366da7e29f0da96366fde04f6d90ad750c973'; +const BLOCK_NUMBER = 28; + +describe('compare methods', () => { + let gqlEthClient: GqlEthClient; + let rpcEthClient: EthClient; + + before('initialize eth clients', async () => { + gqlEthClient = new GqlEthClient({ + gqlEndpoint: GQL_ENDPOINT, + cache: undefined + }); + + rpcEthClient = new EthClient({ + rpcEndpoint: RPC_ENDPOINT, + cache: undefined + }); + }); + + // Compare eth-call results + it('Compare getStorageAt method', async () => { + // TODO: Deploy contract in test and generate input params using solidity-mapper + const params = { + blockHash: BLOCK_HASH, + contract: '0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1', + slot: '0xf4db8e9deefce79f91199eb78ba5f619827e53284bc9b3b7f7a525da2596a022' + }; + + const gqlResult = await gqlEthClient.getStorageAt(params); + const rpcResult = await rpcEthClient.getStorageAt(params); + + expect(rpcResult.value).to.equal(gqlResult.value); + }); + + describe('Compare getBlockWithTransactions method', () => { + const compareBlock = (result: any, expected: any) => { + const { __typename, cid, ethTransactionCidsByHeaderId, ...expectedNode } = expected.allEthHeaderCids.nodes[0]; + const expectedTransactions = ethTransactionCidsByHeaderId.nodes.map(({ __typename, cid, ...tx }: any) => tx); + + const { ethTransactionCidsByHeaderId: { nodes: rpcTxs }, ...rpcNode } = result.allEthHeaderCids.nodes[0]; + expect(rpcNode).to.deep.equal(expectedNode); + expect(rpcTxs).to.deep.equal(expectedTransactions); + }; + + it('With blockHash', async () => { + // TODO: Get a block with transactions + const blockHash = BLOCK_HASH; + + const gqlResult = await gqlEthClient.getBlockWithTransactions({ blockHash }); + const rpcResult = await rpcEthClient.getBlockWithTransactions({ blockHash }); + + compareBlock(rpcResult, gqlResult); + }); + + it('With blockNumber', async () => { + const blockNumber = BLOCK_NUMBER; + + const gqlResult = await gqlEthClient.getBlockWithTransactions({ blockNumber }); + const rpcResult = await rpcEthClient.getBlockWithTransactions({ blockNumber }); + + compareBlock(rpcResult, gqlResult); + }); + }); + + describe('Compare getBlocks method', () => { + const compareBlock = (result: any, expected: any) => { + const { __typename, cid, ...expectedNode } = expected.allEthHeaderCids.nodes[0]; + expect(result.allEthHeaderCids.nodes[0]).to.deep.equal(expectedNode); + }; + + it('With blockHash', async () => { + const blockHash = BLOCK_HASH; + + const gqlResult = await gqlEthClient.getBlocks({ blockHash }); + const rpcResult = await rpcEthClient.getBlocks({ blockHash }); + + compareBlock(rpcResult, gqlResult); + }); + + it('With blockNumber', async () => { + const blockNumber = BLOCK_NUMBER; + + const gqlResult = await gqlEthClient.getBlocks({ blockNumber }); + const rpcResult = await rpcEthClient.getBlocks({ blockNumber }); + + compareBlock(rpcResult, gqlResult); + }); + }); + + describe('Compare getFullBlocks method', async () => { + const compareBlock = (result: any, expected: any) => { + const { + __typename, + cid, + blockByMhKey: expectedBlockByMhKey, + // blockByMhKey: { + // data: expectedData + // }, + ...expectedNode + } = expected.allEthHeaderCids.nodes[0]; + const { + blockByMhKey, + // blockByMhKey: { + // data + // }, + ...node + } = result.allEthHeaderCids.nodes[0]; + expect(node).to.deep.equal(expectedNode); + + // TODO: Match RLP encoded data + // TODO: Compare decoded data + // expect(data).to.equal(expectedData); + }; + + it('With blockHash', async () => { + const blockHash = BLOCK_HASH; + + const gqlResult = await gqlEthClient.getFullBlocks({ blockHash }); + const rpcResult = await rpcEthClient.getFullBlocks({ blockHash }); + + compareBlock(rpcResult, gqlResult); + }); + + it('With blockNumber', async () => { + const blockNumber = BLOCK_NUMBER; + + const gqlResult = await gqlEthClient.getFullBlocks({ blockNumber }); + const rpcResult = await rpcEthClient.getFullBlocks({ blockNumber }); + + compareBlock(rpcResult, gqlResult); + }); + }); + + it('Compare getFullTransaction method', async () => { + const txHash = '0xd459a61a7058dbc1a1ce3bd06aad551f75bbb088006d953c2f373e108c5e52fb'; + const gqlResult = await gqlEthClient.getFullTransaction(txHash); + const rpcResult = await rpcEthClient.getFullTransaction(txHash); + + const { ethTransactionCidByTxHash: { __typename, cid, blockByMhKey: { data: expectedRawTx }, ...expectedTx } } = gqlResult; + const { ethTransactionCidByTxHash: { blockByMhKey: { data: rawTx }, ...tx } } = rpcResult; + expect(tx).to.deep.equal(expectedTx); + expect(rawTx).to.deep.equal(expectedRawTx); + }); + + describe('Compare getBlockByHash method', async () => { + const compareBlock = (result: any, expected: any) => { + const { + __typename, + parent: expectedParent, + blockByMhKey: expectedBlockByMhKey, + ...expectedBlock + } = expected.block; + + const { parent, ...block } = result.block; + expect(block).to.deep.equal(expectedBlock); + expect(parent.hash).to.equal(expectedParent.hash); + }; + + it('With blockHash', async () => { + const gqlResult = await gqlEthClient.getBlockByHash(BLOCK_HASH); + const rpcResult = await rpcEthClient.getBlockByHash(BLOCK_HASH); + + compareBlock(rpcResult, gqlResult); + }); + + it('Without blockHash', async () => { + const gqlResult = await gqlEthClient.getBlockByHash(); + const rpcResult = await rpcEthClient.getBlockByHash(); + + compareBlock(rpcResult, gqlResult); + }); + }); + + describe('Compare getBlockByHash method', () => { + const compareLogs = (result: any, expected: any) => { + result.logs.forEach((log: any, index: number) => { + const { + __typename, + account: expectedAccount, + cid, + ipldBlock, + receiptCID, + transaction: expectedTransaction, + ...expectedLog + } = expected.logs[index]; + + const { account, transaction, ...rpcLog } = log; + expect(rpcLog).to.deep.equal(expectedLog); + expect(account.address).to.equal(expectedAccount.address); + expect(transaction.hash).to.equal(expectedTransaction.hash); + }); + }; + + it('Without addresses', async () => { + const blockHash = BLOCK_HASH; + const blockNumber = BLOCK_NUMBER.toString(); + + const gqlResult = await gqlEthClient.getLogs({ blockHash, blockNumber }); + const rpcResult = await rpcEthClient.getLogs({ blockHash, blockNumber }); + + compareLogs(rpcResult, gqlResult); + }); + + it('With addresses', async () => { + const addresses = [ + '0x36cefe5321b015ea74b1a08efd6d785360071d5d', + '0x24cfbe2986e09ab7b7a5e4f8a6bf629b81840ef1' + ]; + const blockHash = BLOCK_HASH; + const blockNumber = BLOCK_NUMBER.toString(); + + const gqlResult = await gqlEthClient.getLogs({ blockHash, blockNumber, addresses }); + const rpcResult = await rpcEthClient.getLogs({ blockHash, blockNumber, addresses }); + + compareLogs(rpcResult, gqlResult); + }); + }); +}); diff --git a/packages/rpc-eth-client/src/eth-client.ts b/packages/rpc-eth-client/src/eth-client.ts new file mode 100644 index 00000000..dfe0a171 --- /dev/null +++ b/packages/rpc-eth-client/src/eth-client.ts @@ -0,0 +1,315 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import assert from 'assert'; +import { errors, providers, utils } from 'ethers'; +import { TransactionReceipt } from '@ethersproject/abstract-provider'; + +import { Cache } from '@cerc-io/cache'; +import { encodeHeader, escapeHexString, getRawTransaction } from '@cerc-io/util'; +import { padKey } from '@cerc-io/ipld-eth-client'; + +export interface Config { + cache: Cache | undefined; + rpcEndpoint: string; +} + +interface Vars { + blockHash?: string; + blockNumber?: string; + contract?: string; + slot?: string; + addresses?: string[]; +} + +export class EthClient { + _provider: providers.JsonRpcProvider; + _cache: Cache | undefined; + + constructor (config: Config) { + const { rpcEndpoint, cache } = config; + assert(rpcEndpoint, 'Missing RPC endpoint'); + this._provider = new providers.JsonRpcProvider(rpcEndpoint); + + this._cache = cache; + } + + async getStorageAt ({ blockHash, contract, slot }: { blockHash: string, contract: string, slot: string }): Promise<{ value: string, proof: { data: string } }> { + slot = `0x${padKey(slot)}`; + + console.time(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`); + const value = await this._getCachedOrFetch( + 'getStorageAt', + { blockHash, contract, slot }, + async () => { + // TODO: Check if blockHash works with Lotus RPC + return this._provider.getStorageAt(contract, slot, blockHash); + } + ); + console.timeEnd(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`); + + return { + value, + proof: { + // TODO: Return proof with cid and ipldBlock + // To match getStorageAt method of ipld-eth-client which returns proof along with value. + data: JSON.stringify(null) + } + }; + } + + async getBlockWithTransactions ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { + const blockHashOrBlockNumber = blockHash ?? blockNumber; + assert(blockHashOrBlockNumber); + console.time(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`); + const result = await this._provider.getBlockWithTransactions(blockHashOrBlockNumber); + console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`); + + const allEthHeaderCids = { + nodes: [ + { + blockNumber: result.number.toString(), + blockHash: result.hash, + parentHash: result.parentHash, + timestamp: result.timestamp.toString(), + ethTransactionCidsByHeaderId: { + nodes: result.transactions.map((transaction) => ({ + txHash: transaction.hash, + // Transactions with block should be of type TransactionReceipt + index: (transaction as unknown as TransactionReceipt).transactionIndex, + src: transaction.from, + dst: transaction.to + })) + } + } + ] + }; + + return { allEthHeaderCids }; + } + + async getBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { + const blockHashOrBlockNumber = blockHash ?? blockNumber; + assert(blockHashOrBlockNumber); + let nodes: any[] = []; + console.time(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + + try { + const rawBlock = await this._provider.send( + blockHash ? 'eth_getBlockByHash' : 'eth_getBlockByNumber', + [utils.hexValue(blockHashOrBlockNumber), false] + ); + + if (rawBlock) { + const block = this._provider.formatter.block(rawBlock); + + nodes = [ + { + blockNumber: block.number.toString(), + blockHash: block.hash, + parentHash: block.parentHash, + timestamp: block.timestamp.toString(), + stateRoot: this._provider.formatter.hash(rawBlock.stateRoot), + td: this._provider.formatter.bigNumber(rawBlock.totalDifficulty).toString(), + txRoot: this._provider.formatter.hash(rawBlock.transactionsRoot), + receiptRoot: this._provider.formatter.hash(rawBlock.receiptsRoot) + } + ]; + } + } catch (err: any) { + // Check and ignore future block error + if (!(err.code === errors.SERVER_ERROR && err.error.message === "requested a future epoch (beyond 'latest')")) { + throw err; + } + } finally { + console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + } + + return { + allEthHeaderCids: { + nodes + } + }; + } + + async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { + const blockHashOrBlockNumber = blockHash ?? blockNumber; + assert(blockHashOrBlockNumber); + + console.time(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + const rawBlock = await this._provider.send( + blockHash ? 'eth_getBlockByHash' : 'eth_getBlockByNumber', + [utils.hexValue(blockHashOrBlockNumber), false] + ); + console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + + // Create block header + // https://github.com/cerc-io/go-ethereum/blob/v1.11.6-statediff-5.0.8/core/types/block.go#L64 + const header = { + Parent: rawBlock.parentHash, + UnclesDigest: rawBlock.sha3Uncles, + Beneficiary: rawBlock.miner, + StateRoot: rawBlock.stateRoot, + TxRoot: rawBlock.transactionsRoot, + RctRoot: rawBlock.receiptsRoot, + Bloom: rawBlock.logsBloom, + Difficulty: BigInt(rawBlock.difficulty), + Number: BigInt(rawBlock.number), + GasLimit: BigInt(rawBlock.gasLimit), + GasUsed: BigInt(rawBlock.gasUsed), + Time: Number(rawBlock.timestamp), + Extra: rawBlock.extraData, + MixDigest: rawBlock.mixHash, + Nonce: BigInt(rawBlock.nonce), + BaseFee: rawBlock.baseFeePerGas + }; + + const rlpData = encodeHeader(header); + + const allEthHeaderCids = { + nodes: [ + { + blockNumber: this._provider.formatter.number(rawBlock.number).toString(), + blockHash: this._provider.formatter.hash(rawBlock.hash), + parentHash: this._provider.formatter.hash(rawBlock.parentHash), + timestamp: this._provider.formatter.number(rawBlock.timestamp).toString(), + stateRoot: this._provider.formatter.hash(rawBlock.stateRoot), + td: this._provider.formatter.bigNumber(rawBlock.totalDifficulty).toString(), + txRoot: this._provider.formatter.hash(rawBlock.transactionsRoot), + receiptRoot: this._provider.formatter.hash(rawBlock.receiptsRoot), + uncleRoot: this._provider.formatter.hash(rawBlock.sha3Uncles), + bloom: escapeHexString(this._provider.formatter.hex(rawBlock.logsBloom)), + blockByMhKey: { + data: escapeHexString(rlpData) + } + } + ] + }; + + return { allEthHeaderCids }; + } + + async getFullTransaction (txHash: string): Promise { + console.time(`time:eth-client#getFullTransaction-${JSON.stringify({ txHash })}`); + const tx = await this._provider.getTransaction(txHash); + console.timeEnd(`time:eth-client#getFullTransaction-${JSON.stringify({ txHash })}`); + const txReceipt = await tx.wait(); + + return { + ethTransactionCidByTxHash: { + txHash: tx.hash, + index: txReceipt.transactionIndex, + src: tx.from, + dst: tx.to, + blockByMhKey: { + data: escapeHexString(getRawTransaction(tx)) + } + } + }; + } + + async getBlockByHash (blockHash?: string): Promise { + const blockTag: providers.BlockTag = blockHash ?? 'latest'; + + console.time(`time:eth-client#getBlockByHash-${blockHash}`); + const block = await this._provider.getBlock(blockTag); + console.timeEnd(`time:eth-client#getBlockByHash-${blockHash}`); + + return { + block: { + number: block.number, + hash: block.hash, + parent: { + hash: block.parentHash + }, + timestamp: block.timestamp + } + }; + } + + async getLogs (vars: { blockHash: string, blockNumber: string, addresses?: string[] }): Promise { + const { blockNumber, addresses = [] } = vars; + + console.time(`time:eth-client#getLogs-${JSON.stringify(vars)}`); + const result = await this._getCachedOrFetch( + 'getLogs', + vars, + async () => { + const logsByAddressPromises = addresses?.map(address => this._provider.getLogs({ + fromBlock: Number(blockNumber), + toBlock: Number(blockNumber), + address + })); + const logsByAddress = await Promise.all(logsByAddressPromises); + let logs = logsByAddress.flat(); + + // If no addresses provided to filter + if (!logs.length) { + logs = await this._provider.getLogs({ + fromBlock: Number(blockNumber), + toBlock: Number(blockNumber) + }); + } + + return logs.map(log => { + log.address = log.address.toLowerCase(); + return log; + }); + } + ); + + const txHashesSet = result.reduce((acc, log) => { + acc.add(log.transactionHash); + return acc; + }, new Set()); + + const txReceipts = await Promise.all(Array.from(txHashesSet).map(txHash => this._provider.getTransactionReceipt(txHash))); + + const txReceiptMap = txReceipts.reduce((acc, txReceipt) => { + acc.set(txReceipt.transactionHash, txReceipt); + return acc; + }, new Map()); + console.timeEnd(`time:eth-client#getLogs-${JSON.stringify(vars)}`); + + return { + logs: result.map((log) => ({ + account: { + address: log.address + }, + transaction: { + hash: log.transactionHash + }, + topics: log.topics, + data: log.data, + index: log.logIndex, + status: txReceiptMap.get(log.transactionHash)?.status + })) + }; + } + + async _getCachedOrFetch (queryName: string, vars: Vars, fetch: () => Promise): Promise { + const keyObj = { + queryName, + vars + }; + + // Check if request cached in db, if cache is enabled. + if (this._cache) { + const [value, found] = await this._cache.get(keyObj) || [undefined, false]; + if (found) { + return value; + } + } + + // Result not cached or cache disabled, need to perform fetch. + const result = await fetch(); + + // Cache the result and return it, if cache is enabled. + if (this._cache) { + await this._cache.put(keyObj, result); + } + + return result; + } +} diff --git a/packages/rpc-eth-client/tsconfig.json b/packages/rpc-eth-client/tsconfig.json new file mode 100644 index 00000000..22d7de31 --- /dev/null +++ b/packages/rpc-eth-client/tsconfig.json @@ -0,0 +1,75 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Basic Options */ + // "incremental": true, /* Enable incremental compilation */ + "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ + // "lib": [], /* Specify library files to be included in the compilation. */ + // "allowJs": true, /* Allow javascript files to be compiled. */ + // "checkJs": true, /* Report errors in .js files. */ + // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ + "declaration": true, /* Generates corresponding '.d.ts' file. */ + // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ + // "outFile": "./", /* Concatenate and emit output to single file. */ + "outDir": "dist", /* Redirect output structure to the directory. */ + // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + // "composite": true, /* Enable project compilation */ + // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ + // "removeComments": true, /* Do not emit comments to output. */ + // "noEmit": true, /* Do not emit outputs. */ + // "importHelpers": true, /* Import emit helpers from 'tslib'. */ + // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ + // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ + + /* Strict Type-Checking Options */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* Enable strict null checks. */ + // "strictFunctionTypes": true, /* Enable strict checking of function types. */ + // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ + // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ + // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ + // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ + + /* Additional Checks */ + // "noUnusedLocals": true, /* Report errors on unused locals. */ + // "noUnusedParameters": true, /* Report errors on unused parameters. */ + // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */ + + /* Module Resolution Options */ + "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ + // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ + "typeRoots": [ + "./types" + ], /* List of folders to include type definitions from. */ + // "types": [], /* Type declaration files to be included in compilation. */ + // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ + "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ + // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + + /* Source Map Options */ + // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ + // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ + + /* Experimental Options */ + // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + + /* Advanced Options */ + "skipLibCheck": true, /* Skip type checking of declaration files. */ + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + }, + "exclude": ["src/**/*.test.ts", "dist", "**/node_modules"] +} diff --git a/packages/rpc-eth-client/types/common/main.d.ts b/packages/rpc-eth-client/types/common/main.d.ts new file mode 100644 index 00000000..aa0c891f --- /dev/null +++ b/packages/rpc-eth-client/types/common/main.d.ts @@ -0,0 +1,6 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +// https://medium.com/@steveruiz/using-a-javascript-library-without-type-declarations-in-a-typescript-project-3643490015f3 +declare module 'canonical-json' diff --git a/packages/rpc-eth-client/types/common/package.json b/packages/rpc-eth-client/types/common/package.json new file mode 100644 index 00000000..5861d0f0 --- /dev/null +++ b/packages/rpc-eth-client/types/common/package.json @@ -0,0 +1,6 @@ +{ + "name": "common", + "version": "0.1.0", + "license": "AGPL-3.0", + "typings": "main.d.ts" +} diff --git a/packages/util/package.json b/packages/util/package.json index 7bb021b4..19d65206 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -41,7 +41,6 @@ }, "devDependencies": { "@cerc-io/cache": "^0.2.50", - "@cerc-io/ipld-eth-client": "^0.2.50", "@nomiclabs/hardhat-waffle": "^2.0.1", "@types/express": "^4.17.14", "@types/fs-extra": "^9.0.11", diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index d49435c9..832aca89 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -2,18 +2,13 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; import fs from 'fs-extra'; import path from 'path'; import toml from 'toml'; import debug from 'debug'; import { ConnectionOptions } from 'typeorm'; -import { Config as CacheConfig, getCache } from '@cerc-io/cache'; -import { EthClient } from '@cerc-io/ipld-eth-client'; -import { JsonRpcProvider } from '@ethersproject/providers'; - -import { getCustomProvider } from './misc'; +import { Config as CacheConfig } from '@cerc-io/cache'; const log = debug('vulcanize:config'); @@ -213,6 +208,7 @@ export interface UpstreamConfig { ethServer: { gqlApiEndpoint: string; rpcProviderEndpoint: string; + rpcClient: boolean; } traceProviderEndpoint: string; } @@ -247,33 +243,3 @@ export const getConfig = async (configFile: string): Promise => { - const { database: dbConfig, upstream: upstreamConfig, server: serverConfig } = config; - - assert(serverConfig, 'Missing server config'); - assert(dbConfig, 'Missing database config'); - assert(upstreamConfig, 'Missing upstream config'); - - const { ethServer: { gqlApiEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstreamConfig; - - assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint'); - assert(rpcProviderEndpoint, 'Missing upstream ethServer.rpcProviderEndpoint'); - - const cache = await getCache(cacheConfig); - - const ethClient = new EthClient({ - gqlEndpoint: gqlApiEndpoint, - cache - }); - - const ethProvider = getCustomProvider(rpcProviderEndpoint); - - return { - ethClient, - ethProvider - }; -}; diff --git a/packages/util/src/eth.ts b/packages/util/src/eth.ts index 2a8d9f42..9791aa5e 100644 --- a/packages/util/src/eth.ts +++ b/packages/util/src/eth.ts @@ -1,8 +1,30 @@ import debug from 'debug'; -import { utils } from 'ethers'; +import { UnsignedTransaction, utils } from 'ethers'; + +import { TransactionResponse } from '@ethersproject/providers'; +import { SignatureLike } from '@ethersproject/bytes'; const log = debug('vulcanize:eth'); +interface Header { + Parent: string; + UnclesDigest: string; + Beneficiary: string; + StateRoot: string; + TxRoot: string; + RctRoot: string; + Bloom: string; + Difficulty: bigint; + Number: bigint; + GasLimit: bigint; + GasUsed: bigint; + Time: number, + Extra: string; + MixDigest: string; + Nonce: bigint; + BaseFee?: bigint; +} + function decodeInteger(value : string, defaultValue: bigint): bigint function decodeInteger(value : string) : bigint | undefined function decodeInteger (value : string, defaultValue?: bigint): bigint | undefined { @@ -19,7 +41,7 @@ function decodeNumber (value : string, defaultValue?: number): number | undefine return Number(value); } -export function decodeHeader (rlp : Uint8Array): any { +export function decodeHeader (rlp : Uint8Array): Header | undefined { try { const data = utils.RLP.decode(rlp); @@ -52,6 +74,58 @@ export function decodeHeader (rlp : Uint8Array): any { } } +export function encodeHeader (header: Header): string { + return utils.RLP.encode([ + header.Parent, + header.UnclesDigest, + header.Beneficiary, + header.StateRoot, + header.TxRoot, + header.RctRoot, + header.Bloom, + utils.hexlify(header.Difficulty), + utils.hexlify(header.Number), + utils.hexlify(header.GasLimit), + utils.hexlify(header.GasUsed), + utils.hexlify(header.Time), + header.Extra, + header.MixDigest, + utils.hexlify(header.Nonce), + ...(header.BaseFee ? [utils.hexlify(header.BaseFee)] : []) + ]); +} + export function decodeData (hexLiteral: string): Uint8Array { return Uint8Array.from(Buffer.from(hexLiteral.slice(2), 'hex')); } + +// Method to escape hex string as stored in ipld-eth-db +// https://github.com/cerc-io/go-ethereum/blob/v1.11.6-statediff-5.0.8/statediff/indexer/database/file/sql_writer.go#L140 +export function escapeHexString (hex: string): string { + const value = hex.slice(2); + return `\\x${value}`; +} + +// https://docs.ethers.org/v5/cookbook/transactions/#cookbook--compute-raw-transaction +export function getRawTransaction (tx: TransactionResponse): string { + function addKey ( + accum: {[key: string]: any}, + key: string + ) { + const txKey = key as keyof TransactionResponse; + if (txKey in tx) { accum[key] = tx[txKey]; } + return accum; + } + + // Extract the relevant parts of the transaction and signature + const txFields = 'accessList chainId data gasPrice gasLimit maxFeePerGas maxPriorityFeePerGas nonce to type value'.split(' '); + const sigFields = 'v r s'.split(' '); + + // Seriailze the signed transaction + const raw = utils.serializeTransaction(txFields.reduce(addKey, {}) as UnsignedTransaction, sigFields.reduce(addKey, {}) as SignatureLike); + + // Double check things went well + if (utils.keccak256(raw) !== tx.hash) { throw new Error('serializing failed!'); } + + return raw; +} diff --git a/packages/util/src/events.ts b/packages/util/src/events.ts index 19c6be6e..6fbe092f 100644 --- a/packages/util/src/events.ts +++ b/packages/util/src/events.ts @@ -6,10 +6,8 @@ import assert from 'assert'; import debug from 'debug'; import { PubSub } from 'graphql-subscriptions'; -import { EthClient } from '@cerc-io/ipld-eth-client'; - import { JobQueue } from './job-queue'; -import { BlockProgressInterface, EventInterface, IndexerInterface } from './types'; +import { BlockProgressInterface, EventInterface, IndexerInterface, EthClient } from './types'; import { MAX_REORG_DEPTH, JOB_KIND_PRUNE, JOB_KIND_INDEX, UNKNOWN_EVENT_NAME, JOB_KIND_EVENTS, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING } from './constants'; import { createPruningJob, processBlockByNumber } from './common'; import { OrderDirection } from './database'; diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index 6e37de19..a416ab63 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -24,3 +24,4 @@ export * from './graph/utils'; export * from './graph/state-utils'; export * from './graph/types'; export * from './payments'; +export * from './eth'; diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index 69f0a5a7..2dce5648 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -10,7 +10,6 @@ import { ethers } from 'ethers'; import _ from 'lodash'; import * as codec from '@ipld/dag-cbor'; -import { EthClient } from '@cerc-io/ipld-eth-client'; import { GetStorageAt, getStorageValue, StorageLayout } from '@cerc-io/solidity-mapper'; import { @@ -21,7 +20,8 @@ import { ContractInterface, SyncStatusInterface, StateInterface, - StateKind + StateKind, + EthClient } from './types'; import { UNKNOWN_EVENT_NAME, JOB_KIND_CONTRACT, QUEUE_EVENT_PROCESSING, DIFF_MERGE_BATCH_SIZE } from './constants'; import { JobQueue } from './job-queue'; diff --git a/packages/util/src/misc.ts b/packages/util/src/misc.ts index cc5dff92..8892527b 100644 --- a/packages/util/src/misc.ts +++ b/packages/util/src/misc.ts @@ -12,8 +12,6 @@ import Decimal from 'decimal.js'; import { GraphQLResolveInfo } from 'graphql'; import _ from 'lodash'; -import { EthClient } from '@cerc-io/ipld-eth-client'; - import { DEFAULT_CONFIG_PATH } from './constants'; import { GQLCacheConfig, Config } from './config'; import { JobQueue } from './job-queue'; @@ -21,7 +19,7 @@ import { GraphDecimal } from './graph/graph-decimal'; import * as EthDecoder from './eth'; import { getCachedBlockSize } from './block-size-cache'; import { ResultEvent } from './indexer'; -import { EventInterface } from './types'; +import { EventInterface, EthClient } from './types'; import { BlockHeight } from './database'; const JSONbigNative = JSONbig({ useNativeBigInt: true }); diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 21bec412..f556ae7e 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -5,7 +5,6 @@ import { Connection, DeepPartial, EntityTarget, FindConditions, FindManyOptions, ObjectLiteral, QueryRunner } from 'typeorm'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; -import { EthClient } from '@cerc-io/ipld-eth-client'; import { ServerConfig } from './config'; import { Where, QueryOptions, Database } from './database'; @@ -190,6 +189,38 @@ export interface GraphWatcherInterface { setIndexer (indexer: IndexerInterface): void; } +export interface EthClient { + getStorageAt({ blockHash, contract, slot }: { + blockHash: string; + contract: string; + slot: string; + }): Promise<{ + value: string; + proof: { + data: string; + }; + }>; + getBlockWithTransactions({ blockNumber, blockHash }: { + blockNumber?: number; + blockHash?: string; + }): Promise; + getBlocks({ blockNumber, blockHash }: { + blockNumber?: number; + blockHash?: string; + }): Promise; + getFullBlocks({ blockNumber, blockHash }: { + blockNumber?: number; + blockHash?: string; + }): Promise; + getFullTransaction(txHash: string, blockNumber?: number): Promise; + getBlockByHash(blockHash?: string): Promise; + getLogs(vars: { + blockHash: string, + blockNumber: string, + addresses?: string[] + }): Promise; +} + export type Clients = { ethClient: EthClient; [key: string]: any;