cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2024-11-19 12:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ETH RPC API to get logs (#536)

Browse Source 
* Add eth_getLogs API handler

* Transform events into logs

* Update codegen templates

* Allow GET requests

* Increment package verisons

* Remove unnecessary todo

* Add limit on getLogs results size

* Fix config template
This commit is contained in:
prathamesh0 2024-09-16 19:05:45 +05:30 committed by GitHub
parent b46d8816b5
commit d413d724c7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 207 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "0.2.106",
"version": "0.2.107",
"npmClient": "yarn",
"useWorkspaces": true,
"command": {

2
packages/cache/package.json vendored
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cache",
"version": "0.2.106",
"version": "0.2.107",
"description": "Generic object cache",
"main": "dist/index.js",
"scripts": {

12
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/cli",
"version": "0.2.106",
"version": "0.2.107",
"main": "dist/index.js",
"license": "AGPL-3.0",
"scripts": {
@ -15,13 +15,13 @@
},
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.106",
"@cerc-io/ipld-eth-client": "^0.2.106",
"@cerc-io/cache": "^0.2.107",
"@cerc-io/ipld-eth-client": "^0.2.107",
"@cerc-io/libp2p": "^0.42.2-laconic-0.1.4",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.106",
"@cerc-io/rpc-eth-client": "^0.2.106",
"@cerc-io/util": "^0.2.106",
"@cerc-io/peer": "^0.2.107",
"@cerc-io/rpc-eth-client": "^0.2.107",
"@cerc-io/util": "^0.2.107",
"@ethersproject/providers": "^5.4.4",
"@graphql-tools/utils": "^9.1.1",
"@ipld/dag-cbor": "^8.0.0",

9
packages/cli/src/server.ts
View File

@ -323,7 +323,14 @@ export class ServerCmd {
// Create an Express app
const app: Application = express();
const server = await createAndStartServer(app, typeDefs, resolvers, ethRPCHandlers, config.server, paymentsManager);
const server = await createAndStartServer(
app,
typeDefs,
resolvers,
ethRPCHandlers,
config.server,
paymentsManager
);
await startGQLMetricsServer(config);

4
packages/codegen/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/codegen",
"version": "0.2.106",
"version": "0.2.107",
"description": "Code generator",
"private": true,
"main": "index.js",
@ -20,7 +20,7 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/util": "^0.2.106",
"@cerc-io/util": "^0.2.107",
"@graphql-tools/load-files": "^6.5.2",
"@npmcli/package-json": "^5.0.0",
"@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",

6
packages/codegen/src/templates/config-template.handlebars
View File

@ -25,6 +25,12 @@
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = true
# Enable ETH JSON RPC server at /rpc
enableEthRPCServer = true
# Max number of logs that can be returned in a single getLogs request (default: 10000)
ethGetLogsResultLimit = 10000
# Server GQL config
[server.gql]
path = "/graphql"

6
packages/codegen/src/templates/database-template.handlebars
View File

@ -199,6 +199,12 @@ export class Database implements DatabaseInterface {
return this._baseDatabase.getEventsInRange(repo, fromBlockNumber, toBlockNumber);
}
async getEvents (options: FindManyOptions<Event>): Promise<Array<Event>> {
const repo = this._conn.getRepository(Event);
return this._baseDatabase.getEvents(repo, options);
}
async saveEventEntity (queryRunner: QueryRunner, entity: Event): Promise<Event> {
const repo = queryRunner.manager.getRepository(Event);
return this._baseDatabase.saveEventEntity(repo, entity);

8
packages/codegen/src/templates/indexer-template.handlebars
View File

@ -188,6 +188,10 @@ export class Indexer implements IndexerInterface {
return this._storageLayoutMap;
}
get contractMap (): Map<string, ethers.utils.Interface> {
return this._contractMap;
}
{{#if (subgraphPath)}}
get graphWatcher (): GraphWatcher {
return this._graphWatcher;
@ -671,6 +675,10 @@ export class Indexer implements IndexerInterface {
return this._baseIndexer.getEventsInRange(fromBlockNumber, toBlockNumber, this._serverConfig.gql.maxEventsBlockRange);
}
async getEvents (options: FindManyOptions<Event>): Promise<Array<Event>> {
return this._db.getEvents(options);
}
async getSyncStatus (): Promise<SyncStatus | undefined> {
return this._baseIndexer.getSyncStatus();
}

10
packages/codegen/src/templates/package-template.handlebars
View File

@ -41,12 +41,12 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.106",
"@cerc-io/ipld-eth-client": "^0.2.106",
"@cerc-io/solidity-mapper": "^0.2.106",
"@cerc-io/util": "^0.2.106",
"@cerc-io/cli": "^0.2.107",
"@cerc-io/ipld-eth-client": "^0.2.107",
"@cerc-io/solidity-mapper": "^0.2.107",
"@cerc-io/util": "^0.2.107",
{{#if (subgraphPath)}}
"@cerc-io/graph-node": "^0.2.106",
"@cerc-io/graph-node": "^0.2.107",
{{/if}}
"@ethersproject/providers": "^5.4.4",
"debug": "^4.3.1",

10
packages/graph-node/package.json
View File

@ -1,10 +1,10 @@
{
"name": "@cerc-io/graph-node",
"version": "0.2.106",
"version": "0.2.107",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {
"@cerc-io/solidity-mapper": "^0.2.106",
"@cerc-io/solidity-mapper": "^0.2.107",
"@ethersproject/providers": "^5.4.4",
"@graphprotocol/graph-ts": "^0.22.0",
"@nomiclabs/hardhat-ethers": "^2.0.2",
@ -51,9 +51,9 @@
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2",
"@cerc-io/cache": "^0.2.106",
"@cerc-io/ipld-eth-client": "^0.2.106",
"@cerc-io/util": "^0.2.106",
"@cerc-io/cache": "^0.2.107",
"@cerc-io/ipld-eth-client": "^0.2.107",
"@cerc-io/util": "^0.2.107",
"@types/json-diff": "^0.5.2",
"@types/yargs": "^17.0.0",
"bn.js": "^4.11.9",

6
packages/graph-node/test/utils/indexer.ts
View File

@ -91,6 +91,12 @@ export class Indexer implements IndexerInterface {
return undefined;
}
async getEvents (options: FindManyOptions<EventInterface>): Promise<Array<EventInterface>> {
assert(options);
return [];
}
async getSyncStatus (): Promise<SyncStatusInterface | undefined> {
return undefined;
}

6
packages/ipld-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/ipld-eth-client",
"version": "0.2.106",
"version": "0.2.107",
"description": "IPLD ETH Client",
"main": "dist/index.js",
"scripts": {
@ -20,8 +20,8 @@
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.7.1",
"@cerc-io/cache": "^0.2.106",
"@cerc-io/util": "^0.2.106",
"@cerc-io/cache": "^0.2.107",
"@cerc-io/util": "^0.2.107",
"cross-fetch": "^3.1.4",
"debug": "^4.3.1",
"ethers": "^5.4.4",

2
packages/peer/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/peer",
"version": "0.2.106",
"version": "0.2.107",
"description": "libp2p module",
"main": "dist/index.js",
"exports": "./dist/index.js",

8
packages/rpc-eth-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/rpc-eth-client",
"version": "0.2.106",
"version": "0.2.107",
"description": "RPC ETH Client",
"main": "dist/index.js",
"scripts": {
@ -19,9 +19,9 @@
},
"homepage": "https://github.com/cerc-io/watcher-ts#readme",
"dependencies": {
"@cerc-io/cache": "^0.2.106",
"@cerc-io/ipld-eth-client": "^0.2.106",
"@cerc-io/util": "^0.2.106",
"@cerc-io/cache": "^0.2.107",
"@cerc-io/ipld-eth-client": "^0.2.107",
"@cerc-io/util": "^0.2.107",
"chai": "^4.3.4",
"ethers": "^5.4.4",
"left-pad": "^1.3.0",

2
packages/solidity-mapper/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/solidity-mapper",
"version": "0.2.106",
"version": "0.2.107",
"main": "dist/index.js",
"license": "AGPL-3.0",
"devDependencies": {

2
packages/test/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/test",
"version": "0.2.106",
"version": "0.2.107",
"main": "dist/index.js",
"license": "AGPL-3.0",
"private": true,

2
packages/tracing-client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@cerc-io/tracing-client",
"version": "0.2.106",
"version": "0.2.107",
"description": "ETH VM tracing client",
"main": "dist/index.js",
"scripts": {

8
packages/util/package.json
View File

@ -1,13 +1,13 @@
{
"name": "@cerc-io/util",
"version": "0.2.106",
"version": "0.2.107",
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@apollo/utils.keyvaluecache": "^1.0.1",
"@cerc-io/nitro-node": "^0.1.15",
"@cerc-io/peer": "^0.2.106",
"@cerc-io/solidity-mapper": "^0.2.106",
"@cerc-io/peer": "^0.2.107",
"@cerc-io/solidity-mapper": "^0.2.107",
"@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1",
"@ethersproject/properties": "^5.7.0",
"@ethersproject/providers": "^5.4.4",
@ -55,7 +55,7 @@
"yargs": "^17.0.1"
},
"devDependencies": {
"@cerc-io/cache": "^0.2.106",
"@cerc-io/cache": "^0.2.107",
"@nomiclabs/hardhat-waffle": "^2.0.1",
"@types/bunyan": "^1.8.8",
"@types/express": "^4.17.14",

3
packages/util/src/config.ts
View File

@ -255,6 +255,9 @@ export interface ServerConfig {
// Enable ETH JSON RPC server at /rpc
enableEthRPCServer: boolean;
// Max number of logs that can be returned in a single getLogs request
ethGetLogsResultLimit?: number;
}
export interface FundingAmountsConfig {

2
packages/util/src/constants.ts
View File

@ -30,3 +30,5 @@ export const DEFAULT_PREFETCH_BATCH_SIZE = 10;
export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB
export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber'];
export const DEFAULT_ETH_GET_LOGS_RESULT_LIMIT = 10000;

6
packages/util/src/database.ts
View File

@ -523,6 +523,12 @@ export class Database {
return events;
}
async getEvents (repo: Repository<EventInterface>, options: FindManyOptions<EventInterface>): Promise<Array<EventInterface>> {
const events = repo.find(options);
return events;
}
async saveEventEntity (repo: Repository<EventInterface>, entity: EventInterface): Promise<EventInterface> {
const event = await repo.save(entity);
eventCount.inc(1);

124
packages/util/src/eth-rpc-handlers.ts
View File

@ -1,9 +1,10 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
import { utils } from 'ethers';
import { Between, Equal, FindConditions, In, LessThanOrEqual, MoreThanOrEqual } from 'typeorm';
import { JsonRpcProvider } from '@ethersproject/providers';
import { IndexerInterface } from './types';
import { EventInterface, IndexerInterface } from './types';
import { DEFAULT_ETH_GET_LOGS_RESULT_LIMIT } from './constants';
const CODE_INVALID_PARAMS = -32602;
const CODE_INTERNAL_ERROR = -32603;
@ -16,7 +17,10 @@ const ERROR_CONTRACT_NOT_RECOGNIZED = 'Contract not recognized';
const ERROR_CONTRACT_METHOD_NOT_FOUND = 'Contract method not found';
const ERROR_METHOD_NOT_IMPLEMENTED = 'Method not implemented';
const ERROR_INVALID_BLOCK_TAG = 'Invalid block tag';
const ERROR_INVALID_BLOCK_HASH = 'Invalid block hash';
const ERROR_BLOCK_NOT_FOUND = 'Block not found';
const ERROR_TOPICS_FILTER_NOT_SUPPORTED = 'Topics filter not supported';
const ERROR_LIMIT_EXCEEDED = 'Query results exceeds limit';
const DEFAULT_BLOCK_TAG = 'latest';
@ -53,7 +57,7 @@ export const createEthRPCHandlers = async (
const { to, data } = args[0];
const blockTag = args.length > 1 ? args[1] : DEFAULT_BLOCK_TAG;
const blockHash = await parseBlockTag(indexer, ethProvider, blockTag);
const blockHash = await parseEthCallBlockTag(indexer, ethProvider, blockTag);
const watchedContract = indexer.getWatchedContracts().find(contract => contract.address === to);
if (!watchedContract) {
@ -100,12 +104,87 @@ export const createEthRPCHandlers = async (
},
eth_getLogs: async (args: any, callback: any) => {
// TODO: Implement
try {
if (args.length === 0) {
throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_CONTRACT_INSUFFICIENT_PARAMS);
}
const params = args[0];
// Parse arg params into where options
const where: FindConditions<EventInterface> = {};
// TODO: Support topics filter
if (params.topics) {
throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_TOPICS_FILTER_NOT_SUPPORTED);
}
// Address filter, address or a list of addresses
if (params.address) {
if (Array.isArray(params.address)) {
if (params.address.length > 0) {
where.contract = In(params.address);
}
} else {
where.contract = Equal(params.address);
}
}
// Block hash takes precedence over fromBlock / toBlock if provided
if (params.blockHash) {
// Validate input block hash
if (!utils.isHexString(params.blockHash, 32)) {
throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_INVALID_BLOCK_HASH);
}
where.block = {
blockHash: params.blockHash
};
} else if (params.fromBlock || params.toBlock) {
const fromBlockNumber = params.fromBlock ? await parseEthGetLogsBlockTag(indexer, params.fromBlock) : null;
const toBlockNumber = params.toBlock ? await parseEthGetLogsBlockTag(indexer, params.toBlock) : null;
if (fromBlockNumber && toBlockNumber) {
// Both fromBlock and toBlock set
where.block = { blockNumber: Between(fromBlockNumber, toBlockNumber) };
} else if (fromBlockNumber) {
// Only fromBlock set
where.block = { blockNumber: MoreThanOrEqual(fromBlockNumber) };
} else if (toBlockNumber) {
// Only toBlock set
where.block = { blockNumber: LessThanOrEqual(toBlockNumber) };
}
}
// Fetch events from the db
// Load block relation
const resultLimit = indexer.serverConfig.ethGetLogsResultLimit || DEFAULT_ETH_GET_LOGS_RESULT_LIMIT;
const events = await indexer.getEvents({ where, relations: ['block'], take: resultLimit + 1 });
// Limit number of results can be returned by a single query
if (events.length > resultLimit) {
throw new ErrorWithCode(CODE_SERVER_ERROR, `${ERROR_LIMIT_EXCEEDED}: ${resultLimit}`);
}
// Transform events into result logs
const result = await transformEventsToLogs(events);
callback(null, result);
} catch (error: any) {
let callBackError;
if (error instanceof ErrorWithCode) {
callBackError = { code: error.code, message: error.message };
} else {
callBackError = { code: CODE_SERVER_ERROR, message: error.message };
}
callback(callBackError);
}
}
};
};
const parseBlockTag = async (indexer: IndexerInterface, ethProvider: JsonRpcProvider, blockTag: string): Promise<string> => {
const parseEthCallBlockTag = async (indexer: IndexerInterface, ethProvider: JsonRpcProvider, blockTag: string): Promise<string> => {
if (utils.isHexString(blockTag)) {
// Return value if hex string is of block hash length
if (utils.hexDataLength(blockTag) === 32) {
@ -132,3 +211,38 @@ const parseBlockTag = async (indexer: IndexerInterface, ethProvider: JsonRpcProv
throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_INVALID_BLOCK_TAG);
};
const parseEthGetLogsBlockTag = async (indexer: IndexerInterface, blockTag: string): Promise<number> => {
if (utils.isHexString(blockTag)) {
return Number(blockTag);
}
if (blockTag === DEFAULT_BLOCK_TAG) {
const syncStatus = await indexer.getSyncStatus();
if (!syncStatus) {
throw new ErrorWithCode(CODE_INTERNAL_ERROR, 'SyncStatus not found');
}
return syncStatus.latestProcessedBlockNumber;
}
throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_INVALID_BLOCK_TAG);
};
const transformEventsToLogs = async (events: Array<EventInterface>): Promise<any[]> => {
return events.map(event => {
const parsedExtraInfo = JSON.parse(event.extraInfo);
return {
address: event.contract.toLowerCase(),
blockHash: event.block.blockHash,
blockNumber: `0x${event.block.blockNumber.toString(16)}`,
transactionHash: event.txHash,
transactionIndex: `0x${parsedExtraInfo.tx.index.toString(16)}`,
logIndex: `0x${parsedExtraInfo.logIndex.toString(16)}`,
data: parsedExtraInfo.data,
topics: parsedExtraInfo.topics,
removed: event.block.isPruned
};
});
};

8
packages/util/src/server.ts
View File

@ -110,7 +110,13 @@ export const createAndStartServer = async (
app.use(
ETH_RPC_PATH,
jsonParser(),
// TODO: Handle GET requests as well to match Geth's behaviour
(req: any, res: any, next: () => void) => {
// Convert all GET requests to POST to avoid getting rejected from jayson server middleware
if (jayson.Utils.isMethod(req, 'GET')) {
req.method = 'POST';
}
next();
},
rpcServer.middleware()
);
}

1
packages/util/src/types.ts
View File

@ -169,6 +169,7 @@ export interface IndexerInterface {
getBlockProgressEntities (where: FindConditions<BlockProgressInterface>, options: FindManyOptions<BlockProgressInterface>): Promise<BlockProgressInterface[]>
getEntitiesForBlock (blockHash: string, tableName: string): Promise<any[]>
getEvent (id: string): Promise<EventInterface | undefined>
getEvents (options: FindManyOptions<EventInterface>): Promise<Array<EventInterface>>
getSyncStatus (): Promise<SyncStatusInterface | undefined>
getStateSyncStatus (): Promise<StateSyncStatusInterface | undefined>
getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise<Array<EthFullBlock | null>>