Implement rpc-eth-client for Ethereum compatible JSON-RPC endpoint (#398)

* Implement rpc-eth-client with getStorageAt method

* Add test for comparing RPC and GQL eth-client getStorageAt method

* Add getBlockWithTransactions and getBlocks method

* Implement getFullBlocks with RLP encoded data

* Implement getFullTransaction method with raw tx

* Implement getBlockByHash and getLogs methods

* Add flag and interface to switch between RPC and GQL eth clients

* Fix getBlocks to return empty array when block not present

* Return empty array in getBlocks for missing block and use blockNumber in getLogs

* Fix getRawTransaction method for zero signature.v value

* Remove duplicate util from rpc-eth-client

packages/cli/src/base.ts

@@ -11,7 +11,6 @@ import { JsonRpcProvider } from '@ethersproject/providers';
 import {
   Config,
   getConfig,
-  initClients,
   JobQueue,
   DatabaseInterface,
   IndexerInterface,
@@ -21,6 +20,8 @@ import {
   GraphWatcherInterface
 } from '@cerc-io/util';
 
+import { initClients } from './utils/index';
+
 export class BaseCmd {
   _config?: Config;
   _clients?: Clients;

packages/cli/src/utils/index.ts

@@ -4,9 +4,15 @@
 
 import fs from 'fs';
 import path from 'path';
+import assert from 'assert';
+import { providers } from 'ethers';
 
 // @ts-expect-error https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1319854183
 import { PeerIdObj } from '@cerc-io/peer';
+import { Config, EthClient, getCustomProvider } from '@cerc-io/util';
+import { getCache } from '@cerc-io/cache';
+import { EthClient as GqlEthClient } from '@cerc-io/ipld-eth-client';
+import { EthClient as RpcEthClient } from '@cerc-io/rpc-eth-client';
 
 export function readPeerId (filePath: string): PeerIdObj {
   const peerIdFilePath = path.resolve(filePath);
@@ -15,3 +21,43 @@ export function readPeerId (filePath: string): PeerIdObj {
   const peerIdJson = fs.readFileSync(peerIdFilePath, 'utf-8');
   return JSON.parse(peerIdJson);
 }
+
+export const initClients = async (config: Config): Promise<{
+  ethClient: EthClient,
+  ethProvider: providers.JsonRpcProvider
+}> => {
+  const { database: dbConfig, upstream: upstreamConfig, server: serverConfig } = config;
+
+  assert(serverConfig, 'Missing server config');
+  assert(dbConfig, 'Missing database config');
+  assert(upstreamConfig, 'Missing upstream config');
+
+  const { ethServer: { gqlApiEndpoint, rpcProviderEndpoint, rpcClient = false }, cache: cacheConfig } = upstreamConfig;
+
+  assert(rpcProviderEndpoint, 'Missing upstream ethServer.rpcProviderEndpoint');
+
+  const cache = await getCache(cacheConfig);
+
+  let ethClient: EthClient;
+
+  if (rpcClient) {
+    ethClient = new RpcEthClient({
+      rpcEndpoint: rpcProviderEndpoint,
+      cache
+    });
+  } else {
+    assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
+
+    ethClient = new GqlEthClient({
+      gqlEndpoint: gqlApiEndpoint,
+      cache
+    });
+  }
+
+  const ethProvider = getCustomProvider(rpcProviderEndpoint);
+
+  return {
+    ethClient,
+    ethProvider
+  };
+};

packages/ipld-eth-client/src/eth-client.ts

@@ -23,13 +23,10 @@ interface Vars {
 }
 
 export class EthClient {
-  _config: Config;
   _graphqlClient: GraphQLClient;
   _cache: Cache | undefined;
 
   constructor (config: Config) {
-    this._config = config;
-
     const { gqlEndpoint, gqlSubscriptionEndpoint, cache } = config;
 
     assert(gqlEndpoint, 'Missing gql endpoint');

packages/ipld-eth-client/src/utils.ts

@@ -19,7 +19,7 @@ export const getMappingSlot = (mappingSlot: string, key: string): string => {
 
 export const getStorageLeafKey = (slot: string): string => ethers.utils.keccak256(slot);
 
-export const topictoAddress = (topic: string): string => {
+export const topicToAddress = (topic: string): string => {
   return ethers.utils.getAddress(
     ethers.utils.hexZeroPad(
       ethers.utils.hexStripZeros(topic), 20

packages/rpc-eth-client/.eslintignore

@@ -0,0 +1,5 @@
+# Don't lint node_modules.
+node_modules
+
+# Don't lint build output.
+dist

packages/rpc-eth-client/.eslintrc.json

@@ -0,0 +1,32 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": [
+        "semistandard",
+        "plugin:@typescript-eslint/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+        "ecmaVersion": 12,
+        "sourceType": "module"
+    },
+    "plugins": [
+        "@typescript-eslint"
+    ],
+    "rules": {
+        "indent": ["error", 2, { "SwitchCase": 1 }],
+        "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/explicit-module-boundary-types": [
+            "warn",
+            {
+                "allowArgumentsExplicitlyTypedAsAny": true
+            }
+        ],
+        "@typescript-eslint/no-unused-vars": [
+            "error",
+            { "ignoreRestSiblings": true }
+        ]
+    }
+}

packages/rpc-eth-client/.npmignore

@@ -0,0 +1,5 @@
+/src/
+index.ts
+tsconfig.json
+.eslintrc.json
+.eslintignore

packages/rpc-eth-client/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+export * from './src/eth-client';

packages/rpc-eth-client/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@cerc-io/rpc-eth-client",
+  "version": "0.2.50",
+  "description": "RPC ETH Client",
+  "main": "dist/index.js",
+  "scripts": {
+    "lint": "eslint .",
+    "test:rpc": "mocha -r ts-node/register src/**/*.test.ts",
+    "build": "tsc"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cerc-io/watcher-ts.git"
+  },
+  "author": "",
+  "license": "AGPL-3.0",
+  "bugs": {
+    "url": "https://github.com/cerc-io/watcher-ts/issues"
+  },
+  "homepage": "https://github.com/cerc-io/watcher-ts#readme",
+  "dependencies": {
+    "@cerc-io/cache": "^0.2.50",
+    "@cerc-io/util": "^0.2.50",
+    "@cerc-io/ipld-eth-client": "^0.2.50",
+    "cross-fetch": "^3.1.4",
+    "debug": "^4.3.1",
+    "ethers": "^5.4.4",
+    "left-pad": "^1.3.0",
+    "ws": "^8.11.0",
+    "zen-observable-ts": "^1.1.0"
+  },
+  "devDependencies": {
+    "@types/ws": "^8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.47.1",
+    "@typescript-eslint/parser": "^5.47.1",
+    "eslint": "^8.35.0",
+    "eslint-config-semistandard": "^15.0.1",
+    "eslint-config-standard": "^16.0.3",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^5.1.0",
+    "eslint-plugin-standard": "^5.0.0"
+  }
+}

packages/rpc-eth-client/src/eth-client.test.ts

@@ -0,0 +1,232 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+import { expect } from 'chai';
+import 'mocha';
+
+import { EthClient as GqlEthClient } from '@cerc-io/ipld-eth-client';
+
+import { EthClient } from '../index';
+
+const RPC_ENDPOINT = 'http://localhost:8545';
+const GQL_ENDPOINT = 'http://localhost:8083/graphql';
+
+const BLOCK_HASH = '0xef53edd41f1aca301d6dd285656366da7e29f0da96366fde04f6d90ad750c973';
+const BLOCK_NUMBER = 28;
+
+describe('compare methods', () => {
+  let gqlEthClient: GqlEthClient;
+  let rpcEthClient: EthClient;
+
+  before('initialize eth clients', async () => {
+    gqlEthClient = new GqlEthClient({
+      gqlEndpoint: GQL_ENDPOINT,
+      cache: undefined
+    });
+
+    rpcEthClient = new EthClient({
+      rpcEndpoint: RPC_ENDPOINT,
+      cache: undefined
+    });
+  });
+
+  // Compare eth-call results
+  it('Compare getStorageAt method', async () => {
+    // TODO: Deploy contract in test and generate input params using solidity-mapper
+    const params = {
+      blockHash: BLOCK_HASH,
+      contract: '0x1ca7c995f8eF0A2989BbcE08D5B7Efe50A584aa1',
+      slot: '0xf4db8e9deefce79f91199eb78ba5f619827e53284bc9b3b7f7a525da2596a022'
+    };
+
+    const gqlResult = await gqlEthClient.getStorageAt(params);
+    const rpcResult = await rpcEthClient.getStorageAt(params);
+
+    expect(rpcResult.value).to.equal(gqlResult.value);
+  });
+
+  describe('Compare getBlockWithTransactions method', () => {
+    const compareBlock = (result: any, expected: any) => {
+      const { __typename, cid, ethTransactionCidsByHeaderId, ...expectedNode } = expected.allEthHeaderCids.nodes[0];
+      const expectedTransactions = ethTransactionCidsByHeaderId.nodes.map(({ __typename, cid, ...tx }: any) => tx);
+
+      const { ethTransactionCidsByHeaderId: { nodes: rpcTxs }, ...rpcNode } = result.allEthHeaderCids.nodes[0];
+      expect(rpcNode).to.deep.equal(expectedNode);
+      expect(rpcTxs).to.deep.equal(expectedTransactions);
+    };
+
+    it('With blockHash', async () => {
+      // TODO: Get a block with transactions
+      const blockHash = BLOCK_HASH;
+
+      const gqlResult = await gqlEthClient.getBlockWithTransactions({ blockHash });
+      const rpcResult = await rpcEthClient.getBlockWithTransactions({ blockHash });
+
+      compareBlock(rpcResult, gqlResult);
+    });
+
+    it('With blockNumber', async () => {
+      const blockNumber = BLOCK_NUMBER;
+
+      const gqlResult = await gqlEthClient.getBlockWithTransactions({ blockNumber });
+      const rpcResult = await rpcEthClient.getBlockWithTransactions({ blockNumber });
+
+      compareBlock(rpcResult, gqlResult);
+    });
+  });
+
+  describe('Compare getBlocks method', () => {
+    const compareBlock = (result: any, expected: any) => {
+      const { __typename, cid, ...expectedNode } = expected.allEthHeaderCids.nodes[0];
+      expect(result.allEthHeaderCids.nodes[0]).to.deep.equal(expectedNode);
+    };
+
+    it('With blockHash', async () => {
+      const blockHash = BLOCK_HASH;
+
+      const gqlResult = await gqlEthClient.getBlocks({ blockHash });
+      const rpcResult = await rpcEthClient.getBlocks({ blockHash });
+
+      compareBlock(rpcResult, gqlResult);
+    });
+
+    it('With blockNumber', async () => {
+      const blockNumber = BLOCK_NUMBER;
+
+      const gqlResult = await gqlEthClient.getBlocks({ blockNumber });
+      const rpcResult = await rpcEthClient.getBlocks({ blockNumber });
+
+      compareBlock(rpcResult, gqlResult);
+    });
+  });
+
+  describe('Compare getFullBlocks method', async () => {
+    const compareBlock = (result: any, expected: any) => {
+      const {
+        __typename,
+        cid,
+        blockByMhKey: expectedBlockByMhKey,
+        // blockByMhKey: {
+        //   data: expectedData
+        // },
+        ...expectedNode
+      } = expected.allEthHeaderCids.nodes[0];
+      const {
+        blockByMhKey,
+        // blockByMhKey: {
+        //   data
+        // },
+        ...node
+      } = result.allEthHeaderCids.nodes[0];
+      expect(node).to.deep.equal(expectedNode);
+
+      // TODO: Match RLP encoded data
+      // TODO: Compare decoded data
+      // expect(data).to.equal(expectedData);
+    };
+
+    it('With blockHash', async () => {
+      const blockHash = BLOCK_HASH;
+
+      const gqlResult = await gqlEthClient.getFullBlocks({ blockHash });
+      const rpcResult = await rpcEthClient.getFullBlocks({ blockHash });
+
+      compareBlock(rpcResult, gqlResult);
+    });
+
+    it('With blockNumber', async () => {
+      const blockNumber = BLOCK_NUMBER;
+
+      const gqlResult = await gqlEthClient.getFullBlocks({ blockNumber });
+      const rpcResult = await rpcEthClient.getFullBlocks({ blockNumber });
+
+      compareBlock(rpcResult, gqlResult);
+    });
+  });
+
+  it('Compare getFullTransaction method', async () => {
+    const txHash = '0xd459a61a7058dbc1a1ce3bd06aad551f75bbb088006d953c2f373e108c5e52fb';
+    const gqlResult = await gqlEthClient.getFullTransaction(txHash);
+    const rpcResult = await rpcEthClient.getFullTransaction(txHash);
+
+    const { ethTransactionCidByTxHash: { __typename, cid, blockByMhKey: { data: expectedRawTx }, ...expectedTx } } = gqlResult;
+    const { ethTransactionCidByTxHash: { blockByMhKey: { data: rawTx }, ...tx } } = rpcResult;
+    expect(tx).to.deep.equal(expectedTx);
+    expect(rawTx).to.deep.equal(expectedRawTx);
+  });
+
+  describe('Compare getBlockByHash method', async () => {
+    const compareBlock = (result: any, expected: any) => {
+      const {
+        __typename,
+        parent: expectedParent,
+        blockByMhKey: expectedBlockByMhKey,
+        ...expectedBlock
+      } = expected.block;
+
+      const { parent, ...block } = result.block;
+      expect(block).to.deep.equal(expectedBlock);
+      expect(parent.hash).to.equal(expectedParent.hash);
+    };
+
+    it('With blockHash', async () => {
+      const gqlResult = await gqlEthClient.getBlockByHash(BLOCK_HASH);
+      const rpcResult = await rpcEthClient.getBlockByHash(BLOCK_HASH);
+
+      compareBlock(rpcResult, gqlResult);
+    });
+
+    it('Without blockHash', async () => {
+      const gqlResult = await gqlEthClient.getBlockByHash();
+      const rpcResult = await rpcEthClient.getBlockByHash();
+
+      compareBlock(rpcResult, gqlResult);
+    });
+  });
+
+  describe('Compare getBlockByHash method', () => {
+    const compareLogs = (result: any, expected: any) => {
+      result.logs.forEach((log: any, index: number) => {
+        const {
+          __typename,
+          account: expectedAccount,
+          cid,
+          ipldBlock,
+          receiptCID,
+          transaction: expectedTransaction,
+          ...expectedLog
+        } = expected.logs[index];
+
+        const { account, transaction, ...rpcLog } = log;
+        expect(rpcLog).to.deep.equal(expectedLog);
+        expect(account.address).to.equal(expectedAccount.address);
+        expect(transaction.hash).to.equal(expectedTransaction.hash);
+      });
+    };
+
+    it('Without addresses', async () => {
+      const blockHash = BLOCK_HASH;
+      const blockNumber = BLOCK_NUMBER.toString();
+
+      const gqlResult = await gqlEthClient.getLogs({ blockHash, blockNumber });
+      const rpcResult = await rpcEthClient.getLogs({ blockHash, blockNumber });
+
+      compareLogs(rpcResult, gqlResult);
+    });
+
+    it('With addresses', async () => {
+      const addresses = [
+        '0x36cefe5321b015ea74b1a08efd6d785360071d5d',
+        '0x24cfbe2986e09ab7b7a5e4f8a6bf629b81840ef1'
+      ];
+      const blockHash = BLOCK_HASH;
+      const blockNumber = BLOCK_NUMBER.toString();
+
+      const gqlResult = await gqlEthClient.getLogs({ blockHash, blockNumber, addresses });
+      const rpcResult = await rpcEthClient.getLogs({ blockHash, blockNumber, addresses });
+
+      compareLogs(rpcResult, gqlResult);
+    });
+  });
+});

packages/rpc-eth-client/src/eth-client.ts

@@ -0,0 +1,315 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+import assert from 'assert';
+import { errors, providers, utils } from 'ethers';
+import { TransactionReceipt } from '@ethersproject/abstract-provider';
+
+import { Cache } from '@cerc-io/cache';
+import { encodeHeader, escapeHexString, getRawTransaction } from '@cerc-io/util';
+import { padKey } from '@cerc-io/ipld-eth-client';
+
+export interface Config {
+  cache: Cache | undefined;
+  rpcEndpoint: string;
+}
+
+interface Vars {
+  blockHash?: string;
+  blockNumber?: string;
+  contract?: string;
+  slot?: string;
+  addresses?: string[];
+}
+
+export class EthClient {
+  _provider: providers.JsonRpcProvider;
+  _cache: Cache | undefined;
+
+  constructor (config: Config) {
+    const { rpcEndpoint, cache } = config;
+    assert(rpcEndpoint, 'Missing RPC endpoint');
+    this._provider = new providers.JsonRpcProvider(rpcEndpoint);
+
+    this._cache = cache;
+  }
+
+  async getStorageAt ({ blockHash, contract, slot }: { blockHash: string, contract: string, slot: string }): Promise<{ value: string, proof: { data: string } }> {
+    slot = `0x${padKey(slot)}`;
+
+    console.time(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`);
+    const value = await this._getCachedOrFetch(
+      'getStorageAt',
+      { blockHash, contract, slot },
+      async () => {
+        // TODO: Check if blockHash works with Lotus RPC
+        return this._provider.getStorageAt(contract, slot, blockHash);
+      }
+    );
+    console.timeEnd(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`);
+
+    return {
+      value,
+      proof: {
+        // TODO: Return proof with cid and ipldBlock
+        // To match getStorageAt method of ipld-eth-client which returns proof along with value.
+        data: JSON.stringify(null)
+      }
+    };
+  }
+
+  async getBlockWithTransactions ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<any> {
+    const blockHashOrBlockNumber = blockHash ?? blockNumber;
+    assert(blockHashOrBlockNumber);
+    console.time(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
+    const result = await this._provider.getBlockWithTransactions(blockHashOrBlockNumber);
+    console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
+
+    const allEthHeaderCids = {
+      nodes: [
+        {
+          blockNumber: result.number.toString(),
+          blockHash: result.hash,
+          parentHash: result.parentHash,
+          timestamp: result.timestamp.toString(),
+          ethTransactionCidsByHeaderId: {
+            nodes: result.transactions.map((transaction) => ({
+              txHash: transaction.hash,
+              // Transactions with block should be of type TransactionReceipt
+              index: (transaction as unknown as TransactionReceipt).transactionIndex,
+              src: transaction.from,
+              dst: transaction.to
+            }))
+          }
+        }
+      ]
+    };
+
+    return { allEthHeaderCids };
+  }
+
+  async getBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<any> {
+    const blockHashOrBlockNumber = blockHash ?? blockNumber;
+    assert(blockHashOrBlockNumber);
+    let nodes: any[] = [];
+    console.time(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
+
+    try {
+      const rawBlock = await this._provider.send(
+        blockHash ? 'eth_getBlockByHash' : 'eth_getBlockByNumber',
+        [utils.hexValue(blockHashOrBlockNumber), false]
+      );
+
+      if (rawBlock) {
+        const block = this._provider.formatter.block(rawBlock);
+
+        nodes = [
+          {
+            blockNumber: block.number.toString(),
+            blockHash: block.hash,
+            parentHash: block.parentHash,
+            timestamp: block.timestamp.toString(),
+            stateRoot: this._provider.formatter.hash(rawBlock.stateRoot),
+            td: this._provider.formatter.bigNumber(rawBlock.totalDifficulty).toString(),
+            txRoot: this._provider.formatter.hash(rawBlock.transactionsRoot),
+            receiptRoot: this._provider.formatter.hash(rawBlock.receiptsRoot)
+          }
+        ];
+      }
+    } catch (err: any) {
+      // Check and ignore future block error
+      if (!(err.code === errors.SERVER_ERROR && err.error.message === "requested a future epoch (beyond 'latest')")) {
+        throw err;
+      }
+    } finally {
+      console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
+    }
+
+    return {
+      allEthHeaderCids: {
+        nodes
+      }
+    };
+  }
+
+  async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<any> {
+    const blockHashOrBlockNumber = blockHash ?? blockNumber;
+    assert(blockHashOrBlockNumber);
+
+    console.time(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
+    const rawBlock = await this._provider.send(
+      blockHash ? 'eth_getBlockByHash' : 'eth_getBlockByNumber',
+      [utils.hexValue(blockHashOrBlockNumber), false]
+    );
+    console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
+
+    // Create block header
+    // https://github.com/cerc-io/go-ethereum/blob/v1.11.6-statediff-5.0.8/core/types/block.go#L64
+    const header = {
+      Parent: rawBlock.parentHash,
+      UnclesDigest: rawBlock.sha3Uncles,
+      Beneficiary: rawBlock.miner,
+      StateRoot: rawBlock.stateRoot,
+      TxRoot: rawBlock.transactionsRoot,
+      RctRoot: rawBlock.receiptsRoot,
+      Bloom: rawBlock.logsBloom,
+      Difficulty: BigInt(rawBlock.difficulty),
+      Number: BigInt(rawBlock.number),
+      GasLimit: BigInt(rawBlock.gasLimit),
+      GasUsed: BigInt(rawBlock.gasUsed),
+      Time: Number(rawBlock.timestamp),
+      Extra: rawBlock.extraData,
+      MixDigest: rawBlock.mixHash,
+      Nonce: BigInt(rawBlock.nonce),
+      BaseFee: rawBlock.baseFeePerGas
+    };
+
+    const rlpData = encodeHeader(header);
+
+    const allEthHeaderCids = {
+      nodes: [
+        {
+          blockNumber: this._provider.formatter.number(rawBlock.number).toString(),
+          blockHash: this._provider.formatter.hash(rawBlock.hash),
+          parentHash: this._provider.formatter.hash(rawBlock.parentHash),
+          timestamp: this._provider.formatter.number(rawBlock.timestamp).toString(),
+          stateRoot: this._provider.formatter.hash(rawBlock.stateRoot),
+          td: this._provider.formatter.bigNumber(rawBlock.totalDifficulty).toString(),
+          txRoot: this._provider.formatter.hash(rawBlock.transactionsRoot),
+          receiptRoot: this._provider.formatter.hash(rawBlock.receiptsRoot),
+          uncleRoot: this._provider.formatter.hash(rawBlock.sha3Uncles),
+          bloom: escapeHexString(this._provider.formatter.hex(rawBlock.logsBloom)),
+          blockByMhKey: {
+            data: escapeHexString(rlpData)
+          }
+        }
+      ]
+    };
+
+    return { allEthHeaderCids };
+  }
+
+  async getFullTransaction (txHash: string): Promise<any> {
+    console.time(`time:eth-client#getFullTransaction-${JSON.stringify({ txHash })}`);
+    const tx = await this._provider.getTransaction(txHash);
+    console.timeEnd(`time:eth-client#getFullTransaction-${JSON.stringify({ txHash })}`);
+    const txReceipt = await tx.wait();
+
+    return {
+      ethTransactionCidByTxHash: {
+        txHash: tx.hash,
+        index: txReceipt.transactionIndex,
+        src: tx.from,
+        dst: tx.to,
+        blockByMhKey: {
+          data: escapeHexString(getRawTransaction(tx))
+        }
+      }
+    };
+  }
+
+  async getBlockByHash (blockHash?: string): Promise<any> {
+    const blockTag: providers.BlockTag = blockHash ?? 'latest';
+
+    console.time(`time:eth-client#getBlockByHash-${blockHash}`);
+    const block = await this._provider.getBlock(blockTag);
+    console.timeEnd(`time:eth-client#getBlockByHash-${blockHash}`);
+
+    return {
+      block: {
+        number: block.number,
+        hash: block.hash,
+        parent: {
+          hash: block.parentHash
+        },
+        timestamp: block.timestamp
+      }
+    };
+  }
+
+  async getLogs (vars: { blockHash: string, blockNumber: string, addresses?: string[] }): Promise<any> {
+    const { blockNumber, addresses = [] } = vars;
+
+    console.time(`time:eth-client#getLogs-${JSON.stringify(vars)}`);
+    const result = await this._getCachedOrFetch(
+      'getLogs',
+      vars,
+      async () => {
+        const logsByAddressPromises = addresses?.map(address => this._provider.getLogs({
+          fromBlock: Number(blockNumber),
+          toBlock: Number(blockNumber),
+          address
+        }));
+        const logsByAddress = await Promise.all(logsByAddressPromises);
+        let logs = logsByAddress.flat();
+
+        // If no addresses provided to filter
+        if (!logs.length) {
+          logs = await this._provider.getLogs({
+            fromBlock: Number(blockNumber),
+            toBlock: Number(blockNumber)
+          });
+        }
+
+        return logs.map(log => {
+          log.address = log.address.toLowerCase();
+          return log;
+        });
+      }
+    );
+
+    const txHashesSet = result.reduce((acc, log) => {
+      acc.add(log.transactionHash);
+      return acc;
+    }, new Set<string>());
+
+    const txReceipts = await Promise.all(Array.from(txHashesSet).map(txHash => this._provider.getTransactionReceipt(txHash)));
+
+    const txReceiptMap = txReceipts.reduce((acc, txReceipt) => {
+      acc.set(txReceipt.transactionHash, txReceipt);
+      return acc;
+    }, new Map<string, providers.TransactionReceipt>());
+    console.timeEnd(`time:eth-client#getLogs-${JSON.stringify(vars)}`);
+
+    return {
+      logs: result.map((log) => ({
+        account: {
+          address: log.address
+        },
+        transaction: {
+          hash: log.transactionHash
+        },
+        topics: log.topics,
+        data: log.data,
+        index: log.logIndex,
+        status: txReceiptMap.get(log.transactionHash)?.status
+      }))
+    };
+  }
+
+  async _getCachedOrFetch<Result> (queryName: string, vars: Vars, fetch: () => Promise<Result>): Promise<Result> {
+    const keyObj = {
+      queryName,
+      vars
+    };
+
+    // Check if request cached in db, if cache is enabled.
+    if (this._cache) {
+      const [value, found] = await this._cache.get(keyObj) || [undefined, false];
+      if (found) {
+        return value;
+      }
+    }
+
+    // Result not cached or cache disabled, need to perform fetch.
+    const result = await fetch();
+
+    // Cache the result and return it, if cache is enabled.
+    if (this._cache) {
+      await this._cache.put(keyObj, result);
+    }
+
+    return result;
+  }
+}

packages/rpc-eth-client/tsconfig.json

@@ -0,0 +1,75 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+    /* Basic Options */
+    // "incremental": true,                         /* Enable incremental compilation */
+    "target": "es5",                                /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
+    "module": "commonjs",                           /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
+    // "lib": [],                                   /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                             /* Allow javascript files to be compiled. */
+    // "checkJs": true,                             /* Report errors in .js files. */
+    // "jsx": "preserve",                           /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
+    "declaration": true,                         /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                      /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    "sourceMap": true,                              /* Generates corresponding '.map' file. */
+    // "outFile": "./",                             /* Concatenate and emit output to single file. */
+    "outDir": "dist",                              /* Redirect output structure to the directory. */
+    // "rootDir": "./",                             /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                           /* Enable project compilation */
+    // "tsBuildInfoFile": "./",                     /* Specify file to store incremental compilation information */
+    // "removeComments": true,                      /* Do not emit comments to output. */
+    // "noEmit": true,                              /* Do not emit outputs. */
+    // "importHelpers": true,                       /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,                  /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,                     /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                                 /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                       /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                    /* Enable strict null checks. */
+    // "strictFunctionTypes": true,                 /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,                 /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,        /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                      /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                        /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                      /* Report errors on unused locals. */
+    // "noUnusedParameters": true,                  /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,                   /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,          /* Report errors for fallthrough cases in switch statement. */
+    // "noUncheckedIndexedAccess": true,            /* Include 'undefined' in index signature results */
+    // "noImplicitOverride": true,                  /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
+    // "noPropertyAccessFromIndexSignature": true,  /* Require undeclared properties from index signatures to use element accesses. */
+
+    /* Module Resolution Options */
+    "moduleResolution": "node",                  /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                             /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                                 /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                              /* List of root folders whose combined content represents the structure of the project at runtime. */
+    "typeRoots": [
+      "./types"
+    ],                                              /* List of folders to include type definitions from. */
+    // "types": [],                                 /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,        /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true,                        /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,                    /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,                /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                            /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                               /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,                     /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                       /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,              /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,               /* Enables experimental support for emitting type metadata for decorators. */
+
+    /* Advanced Options */
+    "skipLibCheck": true,                           /* Skip type checking of declaration files. */
+    "forceConsistentCasingInFileNames": true        /* Disallow inconsistently-cased references to the same file. */
+  },
+  "exclude": ["src/**/*.test.ts", "dist", "**/node_modules"]
+}

packages/rpc-eth-client/types/common/main.d.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+// https://medium.com/@steveruiz/using-a-javascript-library-without-type-declarations-in-a-typescript-project-3643490015f3
+declare module 'canonical-json'

packages/rpc-eth-client/types/common/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "common",
+  "version": "0.1.0",
+  "license": "AGPL-3.0",
+  "typings": "main.d.ts"
+}

packages/util/package.json

@@ -41,7 +41,6 @@
   },
   "devDependencies": {
     "@cerc-io/cache": "^0.2.50",
-    "@cerc-io/ipld-eth-client": "^0.2.50",
     "@nomiclabs/hardhat-waffle": "^2.0.1",
     "@types/express": "^4.17.14",
     "@types/fs-extra": "^9.0.11",

packages/util/src/config.ts

@@ -2,18 +2,13 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import assert from 'assert';
 import fs from 'fs-extra';
 import path from 'path';
 import toml from 'toml';
 import debug from 'debug';
 import { ConnectionOptions } from 'typeorm';
 
-import { Config as CacheConfig, getCache } from '@cerc-io/cache';
-import { EthClient } from '@cerc-io/ipld-eth-client';
-import { JsonRpcProvider } from '@ethersproject/providers';
-
-import { getCustomProvider } from './misc';
+import { Config as CacheConfig } from '@cerc-io/cache';
 
 const log = debug('vulcanize:config');
 
@@ -213,6 +208,7 @@ export interface UpstreamConfig {
   ethServer: {
     gqlApiEndpoint: string;
     rpcProviderEndpoint: string;
+    rpcClient: boolean;
   }
   traceProviderEndpoint: string;
 }
@@ -247,33 +243,3 @@ export const getConfig = async<ConfigType> (configFile: string): Promise<ConfigT
 
   return config;
 };
-
-export const initClients = async (config: Config): Promise<{
-  ethClient: EthClient,
-  ethProvider: JsonRpcProvider
-}> => {
-  const { database: dbConfig, upstream: upstreamConfig, server: serverConfig } = config;
-
-  assert(serverConfig, 'Missing server config');
-  assert(dbConfig, 'Missing database config');
-  assert(upstreamConfig, 'Missing upstream config');
-
-  const { ethServer: { gqlApiEndpoint, rpcProviderEndpoint }, cache: cacheConfig } = upstreamConfig;
-
-  assert(gqlApiEndpoint, 'Missing upstream ethServer.gqlApiEndpoint');
-  assert(rpcProviderEndpoint, 'Missing upstream ethServer.rpcProviderEndpoint');
-
-  const cache = await getCache(cacheConfig);
-
-  const ethClient = new EthClient({
-    gqlEndpoint: gqlApiEndpoint,
-    cache
-  });
-
-  const ethProvider = getCustomProvider(rpcProviderEndpoint);
-
-  return {
-    ethClient,
-    ethProvider
-  };
-};

packages/util/src/eth.ts

@@ -1,8 +1,30 @@
 import debug from 'debug';
-import { utils } from 'ethers';
+import { UnsignedTransaction, utils } from 'ethers';
+
+import { TransactionResponse } from '@ethersproject/providers';
+import { SignatureLike } from '@ethersproject/bytes';
 
 const log = debug('vulcanize:eth');
 
+interface Header {
+  Parent: string;
+  UnclesDigest: string;
+  Beneficiary: string;
+  StateRoot: string;
+  TxRoot: string;
+  RctRoot: string;
+  Bloom: string;
+  Difficulty: bigint;
+  Number: bigint;
+  GasLimit: bigint;
+  GasUsed: bigint;
+  Time: number,
+  Extra: string;
+  MixDigest: string;
+  Nonce: bigint;
+  BaseFee?: bigint;
+}
+
 function decodeInteger(value : string, defaultValue: bigint): bigint
 function decodeInteger(value : string) : bigint | undefined
 function decodeInteger (value : string, defaultValue?: bigint): bigint | undefined {
@@ -19,7 +41,7 @@ function decodeNumber (value : string, defaultValue?: number): number | undefine
   return Number(value);
 }
 
-export function decodeHeader (rlp : Uint8Array): any {
+export function decodeHeader (rlp : Uint8Array): Header | undefined {
   try {
     const data = utils.RLP.decode(rlp);
 
@@ -52,6 +74,58 @@ export function decodeHeader (rlp : Uint8Array): any {
   }
 }
 
+export function encodeHeader (header: Header): string {
+  return utils.RLP.encode([
+    header.Parent,
+    header.UnclesDigest,
+    header.Beneficiary,
+    header.StateRoot,
+    header.TxRoot,
+    header.RctRoot,
+    header.Bloom,
+    utils.hexlify(header.Difficulty),
+    utils.hexlify(header.Number),
+    utils.hexlify(header.GasLimit),
+    utils.hexlify(header.GasUsed),
+    utils.hexlify(header.Time),
+    header.Extra,
+    header.MixDigest,
+    utils.hexlify(header.Nonce),
+    ...(header.BaseFee ? [utils.hexlify(header.BaseFee)] : [])
+  ]);
+}
+
 export function decodeData (hexLiteral: string): Uint8Array {
   return Uint8Array.from(Buffer.from(hexLiteral.slice(2), 'hex'));
 }
+
+// Method to escape hex string as stored in ipld-eth-db
+// https://github.com/cerc-io/go-ethereum/blob/v1.11.6-statediff-5.0.8/statediff/indexer/database/file/sql_writer.go#L140
+export function escapeHexString (hex: string): string {
+  const value = hex.slice(2);
+  return `\\x${value}`;
+}
+
+// https://docs.ethers.org/v5/cookbook/transactions/#cookbook--compute-raw-transaction
+export function getRawTransaction (tx: TransactionResponse): string {
+  function addKey (
+    accum: {[key: string]: any},
+    key: string
+  ) {
+    const txKey = key as keyof TransactionResponse;
+    if (txKey in tx) { accum[key] = tx[txKey]; }
+    return accum;
+  }
+
+  // Extract the relevant parts of the transaction and signature
+  const txFields = 'accessList chainId data gasPrice gasLimit maxFeePerGas maxPriorityFeePerGas nonce to type value'.split(' ');
+  const sigFields = 'v r s'.split(' ');
+
+  // Seriailze the signed transaction
+  const raw = utils.serializeTransaction(txFields.reduce(addKey, {}) as UnsignedTransaction, sigFields.reduce(addKey, {}) as SignatureLike);
+
+  // Double check things went well
+  if (utils.keccak256(raw) !== tx.hash) { throw new Error('serializing failed!'); }
+
+  return raw;
+}

packages/util/src/events.ts

@@ -6,10 +6,8 @@ import assert from 'assert';
 import debug from 'debug';
 import { PubSub } from 'graphql-subscriptions';
 
-import { EthClient } from '@cerc-io/ipld-eth-client';
-
 import { JobQueue } from './job-queue';
-import { BlockProgressInterface, EventInterface, IndexerInterface } from './types';
+import { BlockProgressInterface, EventInterface, IndexerInterface, EthClient } from './types';
 import { MAX_REORG_DEPTH, JOB_KIND_PRUNE, JOB_KIND_INDEX, UNKNOWN_EVENT_NAME, JOB_KIND_EVENTS, QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING } from './constants';
 import { createPruningJob, processBlockByNumber } from './common';
 import { OrderDirection } from './database';

packages/util/src/index.ts

@@ -24,3 +24,4 @@ export * from './graph/utils';
 export * from './graph/state-utils';
 export * from './graph/types';
 export * from './payments';
+export * from './eth';

packages/util/src/indexer.ts

@@ -10,7 +10,6 @@ import { ethers } from 'ethers';
 import _ from 'lodash';
 
 import * as codec from '@ipld/dag-cbor';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { GetStorageAt, getStorageValue, StorageLayout } from '@cerc-io/solidity-mapper';
 
 import {
@@ -21,7 +20,8 @@ import {
   ContractInterface,
   SyncStatusInterface,
   StateInterface,
-  StateKind
+  StateKind,
+  EthClient
 } from './types';
 import { UNKNOWN_EVENT_NAME, JOB_KIND_CONTRACT, QUEUE_EVENT_PROCESSING, DIFF_MERGE_BATCH_SIZE } from './constants';
 import { JobQueue } from './job-queue';

packages/util/src/misc.ts

@@ -12,8 +12,6 @@ import Decimal from 'decimal.js';
 import { GraphQLResolveInfo } from 'graphql';
 import _ from 'lodash';
 
-import { EthClient } from '@cerc-io/ipld-eth-client';
-
 import { DEFAULT_CONFIG_PATH } from './constants';
 import { GQLCacheConfig, Config } from './config';
 import { JobQueue } from './job-queue';
@@ -21,7 +19,7 @@ import { GraphDecimal } from './graph/graph-decimal';
 import * as EthDecoder from './eth';
 import { getCachedBlockSize } from './block-size-cache';
 import { ResultEvent } from './indexer';
-import { EventInterface } from './types';
+import { EventInterface, EthClient } from './types';
 import { BlockHeight } from './database';
 
 const JSONbigNative = JSONbig({ useNativeBigInt: true });

packages/util/src/types.ts

@@ -5,7 +5,6 @@
 import { Connection, DeepPartial, EntityTarget, FindConditions, FindManyOptions, ObjectLiteral, QueryRunner } from 'typeorm';
 
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 
 import { ServerConfig } from './config';
 import { Where, QueryOptions, Database } from './database';
@@ -190,6 +189,38 @@ export interface GraphWatcherInterface {
   setIndexer (indexer: IndexerInterface): void;
 }
 
+export interface EthClient {
+  getStorageAt({ blockHash, contract, slot }: {
+    blockHash: string;
+    contract: string;
+    slot: string;
+  }): Promise<{
+    value: string;
+    proof: {
+        data: string;
+    };
+  }>;
+  getBlockWithTransactions({ blockNumber, blockHash }: {
+    blockNumber?: number;
+    blockHash?: string;
+  }): Promise<any>;
+  getBlocks({ blockNumber, blockHash }: {
+    blockNumber?: number;
+    blockHash?: string;
+  }): Promise<any>;
+  getFullBlocks({ blockNumber, blockHash }: {
+    blockNumber?: number;
+    blockHash?: string;
+  }): Promise<any>;
+  getFullTransaction(txHash: string, blockNumber?: number): Promise<any>;
+  getBlockByHash(blockHash?: string): Promise<any>;
+  getLogs(vars: {
+    blockHash: string,
+    blockNumber: string,
+    addresses?: string[]
+  }): Promise<any>;
+}
+
 export type Clients = {
   ethClient: EthClient;
   [key: string]: any;