From 8af7417df6a608e64771cc2cde7b11c94a318f4a Mon Sep 17 00:00:00 2001 From: nikugogoi Date: Thu, 1 Sep 2022 14:17:43 +0530 Subject: [PATCH] Implement query for multiple entities and nested relation fields in eden-watcher (#166) * Implement query for multiple entities in eden-watcher * Implement nested relation queries * Implement GQL query params first, skip, orderBy, orderDirection * Add blockNumber index to subgraph entities * Add logs for timing eth-calls and storage calls * Add prometheus metrics to monitor GQL queries * Fix default limit and order of 1-N related field in GQL entitiy query * Add timer logs for block processing * Run transpiled js in all watchers Co-authored-by: prathamesh0 --- packages/codegen/src/entity.ts | 6 +- .../src/templates/config-template.handlebars | 2 + .../src/templates/indexer-template.handlebars | 6 +- .../src/templates/package-template.handlebars | 13 +- .../templates/resolvers-template.handlebars | 17 +- .../src/templates/server-template.handlebars | 8 +- .../templates/tsconfig-template.handlebars | 4 +- packages/eden-watcher/environments/local.toml | 2 + packages/eden-watcher/package.json | 17 +- packages/eden-watcher/src/entity/Account.ts | 3 +- packages/eden-watcher/src/entity/Block.ts | 3 +- packages/eden-watcher/src/entity/Claim.ts | 3 +- .../eden-watcher/src/entity/Distribution.ts | 3 +- .../eden-watcher/src/entity/Distributor.ts | 3 +- packages/eden-watcher/src/entity/Epoch.ts | 3 +- packages/eden-watcher/src/entity/Network.ts | 3 +- packages/eden-watcher/src/entity/Producer.ts | 3 +- .../eden-watcher/src/entity/ProducerEpoch.ts | 3 +- .../entity/ProducerRewardCollectorChange.ts | 3 +- .../eden-watcher/src/entity/ProducerSet.ts | 3 +- .../src/entity/ProducerSetChange.ts | 3 +- .../eden-watcher/src/entity/RewardSchedule.ts | 3 +- .../src/entity/RewardScheduleEntry.ts | 3 +- packages/eden-watcher/src/entity/Slash.ts | 3 +- packages/eden-watcher/src/entity/Slot.ts | 3 +- packages/eden-watcher/src/entity/SlotClaim.ts | 3 +- packages/eden-watcher/src/entity/Staker.ts | 3 +- packages/eden-watcher/src/indexer.ts | 12 +- packages/eden-watcher/src/resolvers.ts | 100 ++++++- packages/eden-watcher/src/schema.gql | 34 +++ packages/eden-watcher/src/server.ts | 4 +- packages/eden-watcher/tsconfig.json | 2 +- packages/erc20-watcher/src/indexer.ts | 2 + packages/erc721-watcher/package.json | 13 +- packages/erc721-watcher/src/indexer.ts | 4 + packages/erc721-watcher/tsconfig.json | 2 +- packages/graph-node/src/database.ts | 278 +++++++++++++----- packages/graph-node/src/loader.ts | 8 + packages/graph-node/src/watcher.ts | 49 ++- packages/graph-test-watcher/package.json | 13 +- .../graph-test-watcher/src/entity/Author.ts | 3 +- .../graph-test-watcher/src/entity/Blog.ts | 3 +- .../graph-test-watcher/src/entity/Category.ts | 3 +- packages/graph-test-watcher/src/indexer.ts | 8 +- packages/graph-test-watcher/tsconfig.json | 2 +- packages/ipld-eth-client/src/eth-client.ts | 30 +- .../mobymask-watcher/environments/local.toml | 2 + packages/mobymask-watcher/package.json | 13 +- packages/mobymask-watcher/src/indexer.ts | 4 + packages/mobymask-watcher/src/resolvers.ts | 29 +- packages/mobymask-watcher/src/server.ts | 4 +- packages/mobymask-watcher/tsconfig.json | 2 +- packages/util/index.ts | 1 + packages/util/src/common.ts | 8 +- packages/util/src/config.ts | 5 + packages/util/src/database.ts | 124 +++----- packages/util/src/gql-metrics.ts | 51 ++++ packages/util/src/job-runner.ts | 9 + packages/util/src/misc.ts | 2 + yarn.lock | 49 ++- 60 files changed, 777 insertions(+), 225 deletions(-) create mode 100644 packages/util/src/gql-metrics.ts diff --git a/packages/codegen/src/entity.ts b/packages/codegen/src/entity.ts index 858fb2de..a2d5e556 100644 --- a/packages/codegen/src/entity.ts +++ b/packages/codegen/src/entity.ts @@ -209,11 +209,15 @@ export class Entity { entityObject.imports.push( { - toImport: new Set(['Entity', 'PrimaryColumn', 'Column']), + toImport: new Set(['Entity', 'PrimaryColumn', 'Column', 'Index']), from: 'typeorm' } ); + entityObject.indexOn.push({ + columns: ['blockNumber'] + }); + // Add common columns. entityObject.columns.push({ name: 'id', diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index 35135e92..e9558cc2 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -27,6 +27,8 @@ [metrics] host = "127.0.0.1" port = 9000 + [metrics.gql] + port = 9001 [database] type = "postgres" diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 4f5f4236..9474afe4 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -430,7 +430,7 @@ export class Indexer implements IPLDIndexerInterface { async getSubgraphEntity (entity: new () => Entity, id: string, block?: BlockHeight): Promise { const relations = this._relationsMap.get(entity) || {}; - const data = await this._graphWatcher.getEntity(entity, id, relations, block); + const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block); return data; } @@ -704,6 +704,7 @@ export class Indexer implements IPLDIndexerInterface { const blockPromise = this._ethClient.getBlockByHash(blockHash); let logs: any[]; + console.time('time:indexer#_fetchAndSaveEvents-fetch-logs'); if (this._serverConfig.filterLogs) { const watchedContracts = this._baseIndexer.getWatchedContracts(); @@ -725,6 +726,7 @@ export class Indexer implements IPLDIndexerInterface { } else { ({ logs } = await this._ethClient.getLogs({ blockHash })); } + console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs'); let [ { block }, @@ -816,8 +818,10 @@ export class Indexer implements IPLDIndexerInterface { parentHash: block.parent.hash }; + console.time('time:indexer#_fetchAndSaveEvents-save-block-events'); const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents); await dbTx.commitTransaction(); + console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events'); return blockProgress; } catch (error) { diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index 213d038e..c449e9f3 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -6,9 +6,13 @@ "main": "dist/index.js", "scripts": { "lint": "eslint .", - "build": "tsc", - "server": "DEBUG=vulcanize:* ts-node src/server.ts", - "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts", + "build": "yarn clean && tsc && yarn copy-assets", + "clean": "rm -rf ./dist", + "copy-assets": "copyfiles -u 1 src/**/*.gql dist/", + "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", + "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts", + "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", + "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts", "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts", "fill": "DEBUG=vulcanize:* ts-node src/fill.ts", "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts", @@ -65,6 +69,7 @@ "eslint-plugin-promise": "^5.1.0", "eslint-plugin-standard": "^5.0.0", "ts-node": "^10.0.0", - "typescript": "^4.3.2" + "typescript": "^4.3.2", + "copyfiles": "^2.4.1" } } diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index 871b55cd..fe3e43e9 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, StateKind } from '@vulcanize/util'; +import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -68,6 +68,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch {{~#each this.params}}, {{this.name}}: {{this.type~}} {{/each}} }): Promise => { log('{{this.name}}', blockHash, contractAddress {{~#each this.params}}, {{this.name~}} {{/each}}); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('{{this.name}}').inc(1); + return indexer.{{this.name}}(blockHash, contractAddress {{~#each this.params}}, {{this.name~}} {{/each}}); }, @@ -77,6 +80,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch {{~#each subgraphQueries}} {{this.queryName}}: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('{{this.queryName}}', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('{{this.queryName}}').inc(1); return indexer.getSubgraphEntity({{this.entityName}}, id, block); }, @@ -84,6 +89,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch {{/each}} events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => { log('events', blockHash, contractAddress, name); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('events').inc(1); const block = await indexer.getBlockProgress(blockHash); if (!block || !block.isComplete) { @@ -96,6 +103,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => { log('eventsInRange', fromBlockNumber, toBlockNumber); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('eventsInRange').inc(1); const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber); if (expected !== actual) { @@ -108,6 +117,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getStateByCID: async (_: any, { cid }: { cid: string }) => { log('getStateByCID', cid); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getStateByCID').inc(1); const ipldBlock = await indexer.getIPLDBlockByCid(cid); @@ -116,6 +127,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getState').inc(1); const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); @@ -124,6 +137,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getSyncStatus: async () => { log('getSyncStatus'); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getSyncStatus').inc(1); return indexer.getSyncStatus(); } diff --git a/packages/codegen/src/templates/server-template.handlebars b/packages/codegen/src/templates/server-template.handlebars index 8f496d8b..ced80d12 100644 --- a/packages/codegen/src/templates/server-template.handlebars +++ b/packages/codegen/src/templates/server-template.handlebars @@ -14,7 +14,7 @@ import debug from 'debug'; import 'graphql-import-node'; import { createServer } from 'http'; -import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util'; +import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util'; {{#if (subgraphPath)}} import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node'; {{/if}} @@ -45,10 +45,10 @@ export const main = async (): Promise => { const db = new Database(config.database); await db.init(); {{#if (subgraphPath)}} - + const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*')); await graphDb.init(); - + const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server); {{/if}} @@ -98,6 +98,8 @@ export const main = async (): Promise => { log(`Server is listening on host ${host} port ${port}`); }); + startGQLMetricsServer(config); + return { app, server }; }; diff --git a/packages/codegen/src/templates/tsconfig-template.handlebars b/packages/codegen/src/templates/tsconfig-template.handlebars index b30e1629..9d406d30 100644 --- a/packages/codegen/src/templates/tsconfig-template.handlebars +++ b/packages/codegen/src/templates/tsconfig-template.handlebars @@ -12,9 +12,9 @@ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - // "sourceMap": true, /* Generates corresponding '.map' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ // "outFile": "./", /* Concatenate and emit output to single file. */ - "outDir": "dist", /* Redirect output structure to the directory. */ + "outDir": "dist", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ // "composite": true, /* Enable project compilation */ // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ diff --git a/packages/eden-watcher/environments/local.toml b/packages/eden-watcher/environments/local.toml index 64f6af10..498c1285 100644 --- a/packages/eden-watcher/environments/local.toml +++ b/packages/eden-watcher/environments/local.toml @@ -25,6 +25,8 @@ [metrics] host = "127.0.0.1" port = 9000 + [metrics.gql] + port = 9001 [database] type = "postgres" diff --git a/packages/eden-watcher/package.json b/packages/eden-watcher/package.json index 1c94060a..0c5507fe 100644 --- a/packages/eden-watcher/package.json +++ b/packages/eden-watcher/package.json @@ -6,9 +6,13 @@ "main": "dist/index.js", "scripts": { "lint": "eslint .", - "build": "tsc", - "server": "DEBUG=vulcanize:* ts-node src/server.ts", - "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts", + "build": "yarn clean && tsc && yarn copy-assets", + "clean": "rm -rf ./dist", + "copy-assets": "copyfiles -u 1 src/**/*.gql dist/", + "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", + "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts", + "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", + "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts", "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts", "fill": "DEBUG=vulcanize:* ts-node src/fill.ts", "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts", @@ -32,13 +36,14 @@ "@apollo/client": "^3.3.19", "@ethersproject/providers": "^5.4.4", "@ipld/dag-cbor": "^6.0.12", + "@vulcanize/graph-node": "^0.1.0", "@vulcanize/ipld-eth-client": "^0.1.0", "@vulcanize/solidity-mapper": "^0.1.0", "@vulcanize/util": "^0.1.0", - "@vulcanize/graph-node": "^0.1.0", "apollo-server-express": "^2.25.0", "apollo-type-bigint": "^0.1.3", "debug": "^4.3.1", + "decimal.js": "^10.3.1", "ethers": "^5.4.4", "express": "^4.17.1", "graphql": "^15.5.0", @@ -46,8 +51,7 @@ "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", "typeorm": "^0.2.32", - "yargs": "^17.0.1", - "decimal.js": "^10.3.1" + "yargs": "^17.0.1" }, "devDependencies": { "@ethersproject/abi": "^5.3.0", @@ -55,6 +59,7 @@ "@types/yargs": "^17.0.0", "@typescript-eslint/eslint-plugin": "^4.25.0", "@typescript-eslint/parser": "^4.25.0", + "copyfiles": "^2.4.1", "eslint": "^7.27.0", "eslint-config-semistandard": "^15.0.1", "eslint-config-standard": "^16.0.3", diff --git a/packages/eden-watcher/src/entity/Account.ts b/packages/eden-watcher/src/entity/Account.ts index 741b2172..d3a3ac80 100644 --- a/packages/eden-watcher/src/entity/Account.ts +++ b/packages/eden-watcher/src/entity/Account.ts @@ -2,11 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Account { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Block.ts b/packages/eden-watcher/src/entity/Block.ts index 83b99833..4c873396 100644 --- a/packages/eden-watcher/src/entity/Block.ts +++ b/packages/eden-watcher/src/entity/Block.ts @@ -2,10 +2,11 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Block { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Claim.ts b/packages/eden-watcher/src/entity/Claim.ts index 3b1a4fd8..b7e6a85e 100644 --- a/packages/eden-watcher/src/entity/Claim.ts +++ b/packages/eden-watcher/src/entity/Claim.ts @@ -2,11 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Claim { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Distribution.ts b/packages/eden-watcher/src/entity/Distribution.ts index a376a8a3..98388406 100644 --- a/packages/eden-watcher/src/entity/Distribution.ts +++ b/packages/eden-watcher/src/entity/Distribution.ts @@ -2,11 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Distribution { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Distributor.ts b/packages/eden-watcher/src/entity/Distributor.ts index b02669d5..960c9910 100644 --- a/packages/eden-watcher/src/entity/Distributor.ts +++ b/packages/eden-watcher/src/entity/Distributor.ts @@ -2,9 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; @Entity() +@Index(['blockNumber']) export class Distributor { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Epoch.ts b/packages/eden-watcher/src/entity/Epoch.ts index b8dea145..615cc50c 100644 --- a/packages/eden-watcher/src/entity/Epoch.ts +++ b/packages/eden-watcher/src/entity/Epoch.ts @@ -2,12 +2,13 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import Decimal from 'decimal.js'; import { bigintTransformer, decimalTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Epoch { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Network.ts b/packages/eden-watcher/src/entity/Network.ts index 4de8f07e..2c118e93 100644 --- a/packages/eden-watcher/src/entity/Network.ts +++ b/packages/eden-watcher/src/entity/Network.ts @@ -2,11 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintArrayTransformer, bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Network { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Producer.ts b/packages/eden-watcher/src/entity/Producer.ts index 57a62cdd..c2975b08 100644 --- a/packages/eden-watcher/src/entity/Producer.ts +++ b/packages/eden-watcher/src/entity/Producer.ts @@ -2,10 +2,11 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Producer { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/ProducerEpoch.ts b/packages/eden-watcher/src/entity/ProducerEpoch.ts index f32a842f..84534ba5 100644 --- a/packages/eden-watcher/src/entity/ProducerEpoch.ts +++ b/packages/eden-watcher/src/entity/ProducerEpoch.ts @@ -2,12 +2,13 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import Decimal from 'decimal.js'; import { bigintTransformer, decimalTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class ProducerEpoch { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/ProducerRewardCollectorChange.ts b/packages/eden-watcher/src/entity/ProducerRewardCollectorChange.ts index 9437fd59..838366b9 100644 --- a/packages/eden-watcher/src/entity/ProducerRewardCollectorChange.ts +++ b/packages/eden-watcher/src/entity/ProducerRewardCollectorChange.ts @@ -2,10 +2,11 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class ProducerRewardCollectorChange { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/ProducerSet.ts b/packages/eden-watcher/src/entity/ProducerSet.ts index dba58e37..148ca883 100644 --- a/packages/eden-watcher/src/entity/ProducerSet.ts +++ b/packages/eden-watcher/src/entity/ProducerSet.ts @@ -2,9 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; @Entity() +@Index(['blockNumber']) export class ProducerSet { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/ProducerSetChange.ts b/packages/eden-watcher/src/entity/ProducerSetChange.ts index 8f0010c7..dbc60d8d 100644 --- a/packages/eden-watcher/src/entity/ProducerSetChange.ts +++ b/packages/eden-watcher/src/entity/ProducerSetChange.ts @@ -2,7 +2,7 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; enum ProducerSetChangeType { @@ -11,6 +11,7 @@ enum ProducerSetChangeType { } @Entity() +@Index(['blockNumber']) export class ProducerSetChange { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/RewardSchedule.ts b/packages/eden-watcher/src/entity/RewardSchedule.ts index 6e6b7c28..13855ee1 100644 --- a/packages/eden-watcher/src/entity/RewardSchedule.ts +++ b/packages/eden-watcher/src/entity/RewardSchedule.ts @@ -2,9 +2,10 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; @Entity() +@Index(['blockNumber']) export class RewardSchedule { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/RewardScheduleEntry.ts b/packages/eden-watcher/src/entity/RewardScheduleEntry.ts index b811ba85..09d2d46e 100644 --- a/packages/eden-watcher/src/entity/RewardScheduleEntry.ts +++ b/packages/eden-watcher/src/entity/RewardScheduleEntry.ts @@ -2,10 +2,11 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class RewardScheduleEntry { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Slash.ts b/packages/eden-watcher/src/entity/Slash.ts index 067791d4..6900a6c9 100644 --- a/packages/eden-watcher/src/entity/Slash.ts +++ b/packages/eden-watcher/src/entity/Slash.ts @@ -2,11 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Slash { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Slot.ts b/packages/eden-watcher/src/entity/Slot.ts index 9846a7d8..f3c7066b 100644 --- a/packages/eden-watcher/src/entity/Slot.ts +++ b/packages/eden-watcher/src/entity/Slot.ts @@ -2,12 +2,13 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import Decimal from 'decimal.js'; import { bigintTransformer, decimalTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Slot { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/SlotClaim.ts b/packages/eden-watcher/src/entity/SlotClaim.ts index 1d4acf7b..ac2eab43 100644 --- a/packages/eden-watcher/src/entity/SlotClaim.ts +++ b/packages/eden-watcher/src/entity/SlotClaim.ts @@ -2,12 +2,13 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import Decimal from 'decimal.js'; import { bigintTransformer, decimalTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class SlotClaim { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/entity/Staker.ts b/packages/eden-watcher/src/entity/Staker.ts index b888dbaa..5bfd16db 100644 --- a/packages/eden-watcher/src/entity/Staker.ts +++ b/packages/eden-watcher/src/entity/Staker.ts @@ -2,10 +2,11 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Staker { @PrimaryColumn('varchar') id!: string; diff --git a/packages/eden-watcher/src/indexer.ts b/packages/eden-watcher/src/indexer.ts index 0e34681b..1aecc0a1 100644 --- a/packages/eden-watcher/src/indexer.ts +++ b/packages/eden-watcher/src/indexer.ts @@ -357,13 +357,15 @@ export class Indexer implements IPLDIndexerInterface { } async getSubgraphEntity (entity: new () => Entity, id: string, block?: BlockHeight): Promise { - const relations = this._relationsMap.get(entity) || {}; - - const data = await this._graphWatcher.getEntity(entity, id, relations, block); + const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block); return data; } + async getSubgraphEntities (entity: new () => Entity, block: BlockHeight, where: { [key: string]: any } = {}, queryOptions: QueryOptions = {}): Promise { + return this._graphWatcher.getEntities(entity, this._relationsMap, block, where, queryOptions); + } + async triggerIndexingOnEvent (event: Event): Promise { const resultEvent = this.getResultEvent(event); @@ -956,6 +958,7 @@ export class Indexer implements IPLDIndexerInterface { const blockPromise = this._ethClient.getBlockByHash(blockHash); let logs: any[]; + console.time('time:indexer#_fetchAndSaveEvents-fetch-logs'); if (this._serverConfig.filterLogs) { const watchedContracts = this._baseIndexer.getWatchedContracts(); @@ -977,6 +980,7 @@ export class Indexer implements IPLDIndexerInterface { } else { ({ logs } = await this._ethClient.getLogs({ blockHash })); } + console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs'); let [ { block }, @@ -1068,8 +1072,10 @@ export class Indexer implements IPLDIndexerInterface { parentHash: block.parent.hash }; + console.time('time:indexer#_fetchAndSaveEvents-save-block-events'); const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents); await dbTx.commitTransaction(); + console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events'); return blockProgress; } catch (error) { diff --git a/packages/eden-watcher/src/resolvers.ts b/packages/eden-watcher/src/resolvers.ts index 707fb86b..7b68ffa0 100644 --- a/packages/eden-watcher/src/resolvers.ts +++ b/packages/eden-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { BlockHeight, StateKind } from '@vulcanize/util'; +import { BlockHeight, OrderDirection, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -79,114 +79,204 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch Query: { producer: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('producer', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('producer').inc(1); return indexer.getSubgraphEntity(Producer, id, block); }, + producers: async (_: any, { block = {}, first, skip }: { block: BlockHeight, first: number, skip: number }) => { + log('producers', block, first, skip); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('producers').inc(1); + + return indexer.getSubgraphEntities( + Producer, + block, + {}, + { limit: first, skip } + ); + }, + producerSet: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('producerSet', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('producerSet').inc(1); return indexer.getSubgraphEntity(ProducerSet, id, block); }, producerSetChange: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('producerSetChange', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('producerSetChange').inc(1); return indexer.getSubgraphEntity(ProducerSetChange, id, block); }, producerRewardCollectorChange: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('producerRewardCollectorChange', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('producerRewardCollectorChange').inc(1); return indexer.getSubgraphEntity(ProducerRewardCollectorChange, id, block); }, rewardScheduleEntry: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('rewardScheduleEntry', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('rewardScheduleEntry').inc(1); return indexer.getSubgraphEntity(RewardScheduleEntry, id, block); }, rewardSchedule: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('rewardSchedule', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('rewardSchedule').inc(1); return indexer.getSubgraphEntity(RewardSchedule, id, block); }, producerEpoch: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('producerEpoch', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('producerEpoch').inc(1); return indexer.getSubgraphEntity(ProducerEpoch, id, block); }, block: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('block', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('block').inc(1); return indexer.getSubgraphEntity(Block, id, block); }, + blocks: async (_: any, { block = {}, where, first, skip, orderBy, orderDirection }: { block: BlockHeight, where: { [key: string]: any }, first: number, skip: number, orderBy: string, orderDirection: OrderDirection }) => { + log('blocks', block, where, first, skip, orderBy, orderDirection); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('blocks').inc(1); + + return indexer.getSubgraphEntities( + Block, + block, + where, + { limit: first, skip, orderBy, orderDirection } + ); + }, + epoch: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('epoch', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('epoch').inc(1); return indexer.getSubgraphEntity(Epoch, id, block); }, + epoches: async (_: any, { block = {}, where, first, skip }: { block: BlockHeight, where: { [key: string]: any }, first: number, skip: number }) => { + log('epoches', block, where, first, skip); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('epoches').inc(1); + + return indexer.getSubgraphEntities( + Epoch, + block, + where, + { limit: first, skip } + ); + }, + slotClaim: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('slotClaim', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('slotClaim').inc(1); return indexer.getSubgraphEntity(SlotClaim, id, block); }, slot: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('slot', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('slot').inc(1); return indexer.getSubgraphEntity(Slot, id, block); }, staker: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('staker', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('staker').inc(1); return indexer.getSubgraphEntity(Staker, id, block); }, + stakers: async (_: any, { block = {}, where, first, skip, orderBy, orderDirection }: { block: BlockHeight, where: { [key: string]: any }, first: number, skip: number, orderBy: string, orderDirection: OrderDirection }) => { + log('stakers', block, where, first, skip, orderBy, orderDirection); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('stakers').inc(1); + + return indexer.getSubgraphEntities( + Staker, + block, + where, + { limit: first, skip, orderBy, orderDirection } + ); + }, + network: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('network', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('network').inc(1); return indexer.getSubgraphEntity(Network, id, block); }, distributor: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('distributor', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('distributor').inc(1); return indexer.getSubgraphEntity(Distributor, id, block); }, distribution: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('distribution', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('distribution').inc(1); return indexer.getSubgraphEntity(Distribution, id, block); }, claim: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('claim', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('claim').inc(1); return indexer.getSubgraphEntity(Claim, id, block); }, slash: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('slash', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('slash').inc(1); return indexer.getSubgraphEntity(Slash, id, block); }, account: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => { log('account', id, block); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('account').inc(1); return indexer.getSubgraphEntity(Account, id, block); }, events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => { log('events', blockHash, contractAddress, name); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('events').inc(1); const block = await indexer.getBlockProgress(blockHash); if (!block || !block.isComplete) { @@ -199,6 +289,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => { log('eventsInRange', fromBlockNumber, toBlockNumber); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('eventsInRange').inc(1); const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber); if (expected !== actual) { @@ -211,6 +303,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getStateByCID: async (_: any, { cid }: { cid: string }) => { log('getStateByCID', cid); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getStateByCID').inc(1); const ipldBlock = await indexer.getIPLDBlockByCid(cid); @@ -219,6 +313,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getState').inc(1); const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); @@ -227,6 +323,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getSyncStatus: async () => { log('getSyncStatus'); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getSyncStatus').inc(1); return indexer.getSyncStatus(); } diff --git a/packages/eden-watcher/src/schema.gql b/packages/eden-watcher/src/schema.gql index 90f03f5a..257270e7 100644 --- a/packages/eden-watcher/src/schema.gql +++ b/packages/eden-watcher/src/schema.gql @@ -218,10 +218,16 @@ type SyncStatus { latestCanonicalBlockNumber: Int! } +enum OrderDirection { + asc + desc +} + type Query { events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!] eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!] producer(id: String!, block: Block_height): Producer! + producers(block: Block_height, first: Int = 100, skip: Int = 0): [Producer!]! producerSet(id: String!, block: Block_height): ProducerSet! producerSetChange(id: String!, block: Block_height): ProducerSetChange! producerRewardCollectorChange(id: String!, block: Block_height): ProducerRewardCollectorChange! @@ -229,10 +235,13 @@ type Query { rewardSchedule(id: String!, block: Block_height): RewardSchedule! producerEpoch(id: String!, block: Block_height): ProducerEpoch! block(id: String!, block: Block_height): Block! + blocks(where: Block_filter, block: Block_height, orderBy: Block_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Block!]! epoch(id: String!, block: Block_height): Epoch! + epoches(where: Epoch_filter, block: Block_height, first: Int = 100, skip: Int = 0): [Epoch!]! slotClaim(id: String!, block: Block_height): SlotClaim! slot(id: String!, block: Block_height): Slot! staker(id: String!, block: Block_height): Staker! + stakers(where: Staker_filter, block: Block_height, orderBy: Staker_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Staker!]! network(id: String!, block: Block_height): Network! distributor(id: String!, block: Block_height): Distributor! distribution(id: String!, block: Block_height): Distribution! @@ -311,6 +320,18 @@ type Block { size: BigInt } +input Block_filter { + fromActiveProducer: Boolean + number_gte: BigInt + number_lte: BigInt + timestamp_lte: BigInt +} + +enum Block_orderBy { + number + timestamp +} + type Epoch { id: ID! finalized: Boolean! @@ -323,6 +344,11 @@ type Epoch { producerRewards: [ProducerEpoch!]! } +input Epoch_filter { + epochNumber_gte: BigInt + epochNumber_lte: BigInt +} + type ProducerEpoch { id: ID! address: Bytes! @@ -361,6 +387,14 @@ type Staker { rank: BigInt } +input Staker_filter { + rank_gte: BigInt +} + +enum Staker_orderBy { + rank +} + type Network { id: ID! slot0: Slot diff --git a/packages/eden-watcher/src/server.ts b/packages/eden-watcher/src/server.ts index c2f6e485..d04f0b21 100644 --- a/packages/eden-watcher/src/server.ts +++ b/packages/eden-watcher/src/server.ts @@ -14,7 +14,7 @@ import debug from 'debug'; import 'graphql-import-node'; import { createServer } from 'http'; -import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util'; +import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util'; import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node'; import { createResolvers } from './resolvers'; @@ -92,6 +92,8 @@ export const main = async (): Promise => { log(`Server is listening on host ${host} port ${port}`); }); + startGQLMetricsServer(config); + return { app, server }; }; diff --git a/packages/eden-watcher/tsconfig.json b/packages/eden-watcher/tsconfig.json index b30e1629..f009958d 100644 --- a/packages/eden-watcher/tsconfig.json +++ b/packages/eden-watcher/tsconfig.json @@ -12,7 +12,7 @@ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - // "sourceMap": true, /* Generates corresponding '.map' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "dist", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 3d585416..12fec01b 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -450,8 +450,10 @@ export class Indexer implements IndexerInterface { parentHash: block.parent.hash }; + console.time('time:indexer#_fetchAndSaveEvents-save-block-events'); const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents); await dbTx.commitTransaction(); + console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events'); return blockProgress; } catch (error) { diff --git a/packages/erc721-watcher/package.json b/packages/erc721-watcher/package.json index 528ccda4..50e615ba 100644 --- a/packages/erc721-watcher/package.json +++ b/packages/erc721-watcher/package.json @@ -6,9 +6,13 @@ "main": "dist/index.js", "scripts": { "lint": "eslint .", - "build": "tsc", - "server": "DEBUG=vulcanize:* ts-node src/server.ts", - "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts", + "build": "yarn clean && tsc && yarn copy-assets", + "clean": "rm -rf ./dist", + "copy-assets": "copyfiles -u 1 src/**/*.gql dist/", + "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", + "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts", + "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", + "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts", "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts", "fill": "DEBUG=vulcanize:* ts-node src/fill.ts", "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts", @@ -70,6 +74,7 @@ "eslint-plugin-standard": "^5.0.0", "ts-node": "^10.0.0", "typescript": "^4.3.2", - "hardhat": "^2.3.0" + "hardhat": "^2.3.0", + "copyfiles": "^2.4.1" } } diff --git a/packages/erc721-watcher/src/indexer.ts b/packages/erc721-watcher/src/indexer.ts index 4d5407d2..7ea83b25 100644 --- a/packages/erc721-watcher/src/indexer.ts +++ b/packages/erc721-watcher/src/indexer.ts @@ -989,6 +989,7 @@ export class Indexer implements IPLDIndexerInterface { const blockPromise = this._ethClient.getBlockByHash(blockHash); let logs: any[]; + console.time('time:indexer#_fetchAndSaveEvents-fetch-logs'); if (this._serverConfig.filterLogs) { const watchedContracts = this._baseIndexer.getWatchedContracts(); @@ -1010,6 +1011,7 @@ export class Indexer implements IPLDIndexerInterface { } else { ({ logs } = await this._ethClient.getLogs({ blockHash })); } + console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs'); let [ { block }, @@ -1101,8 +1103,10 @@ export class Indexer implements IPLDIndexerInterface { parentHash: block.parent.hash }; + console.time('time:indexer#_fetchAndSaveEvents-save-block-events'); const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents); await dbTx.commitTransaction(); + console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events'); return blockProgress; } catch (error) { diff --git a/packages/erc721-watcher/tsconfig.json b/packages/erc721-watcher/tsconfig.json index b30e1629..f009958d 100644 --- a/packages/erc721-watcher/tsconfig.json +++ b/packages/erc721-watcher/tsconfig.json @@ -12,7 +12,7 @@ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - // "sourceMap": true, /* Generates corresponding '.map' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "dist", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index aded4879..8d6e417b 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -4,6 +4,7 @@ import assert from 'assert'; import { + Brackets, Connection, ConnectionOptions, FindOneOptions, @@ -14,7 +15,9 @@ import { import { BlockHeight, BlockProgressInterface, - Database as BaseDatabase + Database as BaseDatabase, + QueryOptions, + Where } from '@vulcanize/util'; import { Block, fromEntityValue, toEntityValue } from './utils'; @@ -92,7 +95,7 @@ export class Database { return entities.map((entity: any) => entity.id); } - async getEntityWithRelations (entity: (new () => Entity) | string, id: string, relations: { [key: string]: any }, block: BlockHeight = {}): Promise { + async getEntityWithRelations (entity: (new () => Entity), id: string, relationsMap: Map, block: BlockHeight = {}): Promise { const queryRunner = this._conn.createQueryRunner(); let { hash: blockHash, number: blockNumber } = block; @@ -124,75 +127,9 @@ export class Database { entityData = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); } + // Get relational fields if (entityData) { - // Populate relational fields. - // TODO: Implement query for nested relations. - const relationQueryPromises = Object.entries(relations).map(async ([field, data]) => { - assert(entityData); - const { entity: relatedEntity, isArray, isDerived, field: derivedField } = data; - - const repo = queryRunner.manager.getRepository(relatedEntity); - let selectQueryBuilder = repo.createQueryBuilder('entity'); - - if (isDerived) { - // For derived relational field. - selectQueryBuilder = selectQueryBuilder.where(`entity.${derivedField} = :id`, { id: entityData.id }); - - if (isArray) { - selectQueryBuilder = selectQueryBuilder.distinctOn(['entity.id']) - .orderBy('entity.id') - .limit(DEFAULT_LIMIT); - } else { - selectQueryBuilder = selectQueryBuilder.limit(1); - } - } else { - if (isArray) { - // For one to many relational field. - selectQueryBuilder = selectQueryBuilder.where('entity.id IN (:...ids)', { ids: entityData[field] }) - .distinctOn(['entity.id']) - .orderBy('entity.id') - .limit(DEFAULT_LIMIT); - - // Subquery example if distinctOn is not performant. - // - // SELECT c.* - // FROM - // categories c, - // ( - // SELECT id, MAX(block_number) as block_number - // FROM categories - // WHERE - // id IN ('nature', 'tech', 'issues') - // AND - // block_number <= 127 - // GROUP BY id - // ) a - // WHERE - // c.id = a.id AND c.block_number = a.block_number - } else { - // For one to one relational field. - selectQueryBuilder = selectQueryBuilder.where('entity.id = :id', { id: entityData[field] }) - .limit(1); - } - - selectQueryBuilder = selectQueryBuilder.addOrderBy('entity.block_number', 'DESC'); - } - - if (blockNumber) { - selectQueryBuilder = selectQueryBuilder.andWhere( - 'entity.block_number <= :blockNumber', - { blockNumber } - ); - } - - if (isArray) { - entityData[field] = await selectQueryBuilder.getMany(); - } else { - entityData[field] = await selectQueryBuilder.getOne(); - } - }); - - await Promise.all(relationQueryPromises); + [entityData] = await this.loadRelations(block, relationsMap, entity, [entityData], 1); } return entityData; @@ -201,6 +138,207 @@ export class Database { } } + async getEntities (entity: new () => Entity, relationsMap: Map, block: BlockHeight, where: Where = {}, queryOptions: QueryOptions = {}, depth = 1): Promise { + const queryRunner = this._conn.createQueryRunner(); + try { + const repo = queryRunner.manager.getRepository(entity); + const { tableName } = repo.metadata; + + let subQuery = repo.createQueryBuilder('subTable') + .select('subTable.id', 'id') + .addSelect('MAX(subTable.block_number)', 'block_number') + .addFrom('block_progress', 'blockProgress') + .where('subTable.block_hash = blockProgress.block_hash') + .andWhere('blockProgress.is_pruned = :isPruned', { isPruned: false }) + .groupBy('subTable.id'); + + if (block.hash) { + const { canonicalBlockNumber, blockHashes } = await this._baseDatabase.getFrothyRegion(queryRunner, block.hash); + + subQuery = subQuery + .andWhere(new Brackets(qb => { + qb.where('subTable.block_hash IN (:...blockHashes)', { blockHashes }) + .orWhere('subTable.block_number <= :canonicalBlockNumber', { canonicalBlockNumber }); + })); + } + + if (block.number) { + subQuery = subQuery.andWhere('subTable.block_number <= :blockNumber', { blockNumber: block.number }); + } + + let selectQueryBuilder = repo.createQueryBuilder(tableName) + .innerJoin( + `(${subQuery.getQuery()})`, + 'latestEntities', + `${tableName}.id = "latestEntities"."id" AND ${tableName}.block_number = "latestEntities"."block_number"` + ) + .setParameters(subQuery.getParameters()); + + selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where); + + if (queryOptions.orderBy) { + selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions); + } + + selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' }); + + if (queryOptions.skip) { + selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip); + } + + if (queryOptions.limit) { + selectQueryBuilder = selectQueryBuilder.limit(queryOptions.limit); + } + + const entities = await selectQueryBuilder.getMany(); + + if (!entities.length) { + return []; + } + + return this.loadRelations(block, relationsMap, entity, entities, depth); + } finally { + await queryRunner.release(); + } + } + + async loadRelations (block: BlockHeight, relationsMap: Map, entity: new () => Entity, entities: Entity[], depth: number): Promise { + // Only support two-level nesting of relations + if (depth > 2) { + return entities; + } + + const relations = relationsMap.get(entity); + if (relations === undefined) { + return entities; + } + + const relationPromises = Object.entries(relations).map(async ([field, data]) => { + const { entity: relationEntity, isArray, isDerived, field: foreignKey } = data; + + if (isDerived) { + const where: Where = { + [foreignKey]: [{ + value: entities.map((entity: any) => entity.id), + not: false, + operator: 'in' + }] + }; + + const relatedEntities = await this.getEntities( + relationEntity, + relationsMap, + block, + where, + {}, + depth + 1 + ); + + const relatedEntitiesMap = relatedEntities.reduce((acc: {[key:string]: any[]}, entity: any) => { + // Related entity might be loaded with data. + const parentEntityId = entity[foreignKey].id ?? entity[foreignKey]; + + if (!acc[parentEntityId]) { + acc[parentEntityId] = []; + } + + if (acc[parentEntityId].length < DEFAULT_LIMIT) { + acc[parentEntityId].push(entity); + } + + return acc; + }, {}); + + entities.forEach((entity: any) => { + if (relatedEntitiesMap[entity.id]) { + entity[field] = relatedEntitiesMap[entity.id]; + } else { + entity[field] = []; + } + }); + + return; + } + + if (isArray) { + const relatedIds = entities.reduce((acc: Set, entity: any) => { + entity[field].forEach((relatedEntityId: string) => acc.add(relatedEntityId)); + + return acc; + }, new Set()); + + const where: Where = { + id: [{ + value: Array.from(relatedIds), + not: false, + operator: 'in' + }] + }; + + const relatedEntities = await this.getEntities( + relationEntity, + relationsMap, + block, + where, + {}, + depth + 1 + ); + + entities.forEach((entity: any) => { + const relatedEntityIds: Set = entity[field].reduce((acc: Set, id: string) => { + acc.add(id); + + return acc; + }, new Set()); + + entity[field] = []; + + relatedEntities.forEach((relatedEntity: any) => { + if (relatedEntityIds.has(relatedEntity.id) && entity[field].length < DEFAULT_LIMIT) { + entity[field].push(relatedEntity); + } + }); + }); + + return; + } + + // field is neither an array nor derivedFrom + const where: Where = { + id: [{ + value: entities.map((entity: any) => entity[field]), + not: false, + operator: 'in' + }] + }; + + const relatedEntities = await this.getEntities( + relationEntity, + relationsMap, + block, + where, + {}, + depth + 1 + ); + + const relatedEntitiesMap = relatedEntities.reduce((acc: {[key:string]: any}, entity: any) => { + acc[entity.id] = entity; + + return acc; + }, {}); + + entities.forEach((entity: any) => { + if (relatedEntitiesMap[entity[field]]) { + entity[field] = relatedEntitiesMap[entity[field]]; + } + }); + }); + + await Promise.all(relationPromises); + + return entities; + } + async saveEntity (entity: string, data: any): Promise { const repo = this._conn.getRepository(entity); diff --git a/packages/graph-node/src/loader.ts b/packages/graph-node/src/loader.ts index 3477f19e..f1ba826e 100644 --- a/packages/graph-node/src/loader.ts +++ b/packages/graph-node/src/loader.ts @@ -74,7 +74,9 @@ export const instantiate = async ( const entityId = __getString(id); assert(context.block); + console.time(`time:loader#index.store.get-db-${entityName}`); const entityData = await database.getEntity(entityName, entityId, context.block.blockHash); + console.timeEnd(`time:loader#index.store.get-db-${entityName}`); if (!entityData) { return null; @@ -95,7 +97,9 @@ export const instantiate = async ( assert(context.block); let dbData = await database.fromGraphEntity(instanceExports, context.block, entityName, entityInstance); + console.time(`time:loader#index.store.set-db-${entityName}`); await database.saveEntity(entityName, dbData); + console.timeEnd(`time:loader#index.store.set-db-${entityName}`); // Resolve any field name conflicts in the dbData for auto-diff. dbData = resolveEntityFieldConflicts(dbData); @@ -206,7 +210,9 @@ export const instantiate = async ( assert(context.block); // TODO: Check for function overloading. + console.time(`time:loader#ethereum.call-${functionName}`); let result = await contract[functionName](...functionParams, { blockTag: context.block.blockHash }); + console.timeEnd(`time:loader#ethereum.call-${functionName}`); // Using function signature does not work. const { outputs } = contract.interface.getFunction(functionName); @@ -279,6 +285,7 @@ export const instantiate = async ( assert(storageLayout); assert(context.block); + console.time(`time:loader#ethereum.storageValue-${variableString}`); const result = await indexer.getStorageValue( storageLayout, context.block.blockHash, @@ -286,6 +293,7 @@ export const instantiate = async ( variableString, ...mappingKeyValues ); + console.timeEnd(`time:loader#ethereum.storageValue-${variableString}`); const storageValueType = getStorageValueType(storageLayout, variableString, mappingKeyValues); diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index 7de09028..2f1088c1 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -11,11 +11,11 @@ import { ContractInterface, utils, providers } from 'ethers'; import { ResultObject } from '@vulcanize/assemblyscript/lib/loader'; import { EthClient } from '@vulcanize/ipld-eth-client'; -import { IndexerInterface, getFullBlock, BlockHeight, ServerConfig, getFullTransaction } from '@vulcanize/util'; +import { IndexerInterface, getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions } from '@vulcanize/util'; import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils'; import { Context, GraphData, instantiate } from './loader'; -import { Database } from './database'; +import { Database, DEFAULT_LIMIT } from './database'; const log = debug('vulcanize:graph-watcher'); @@ -248,14 +248,55 @@ export class GraphWatcher { this._indexer = indexer; } - async getEntity (entity: new () => Entity, id: string, relations: { [key: string]: any }, block?: BlockHeight): Promise { + async getEntity (entity: new () => Entity, id: string, relationsMap: Map, block?: BlockHeight): Promise { // Get entity from the database. - const result = await this._database.getEntityWithRelations(entity, id, relations, block) as any; + const result = await this._database.getEntityWithRelations(entity, id, relationsMap, block); // Resolve any field name conflicts in the entity result. return resolveEntityFieldConflicts(result); } + async getEntities (entity: new () => Entity, relationsMap: Map, block: BlockHeight, where: { [key: string]: any } = {}, queryOptions: QueryOptions): Promise { + where = Object.entries(where).reduce((acc: { [key: string]: any }, [fieldWithSuffix, value]) => { + const [field, ...suffix] = fieldWithSuffix.split('_'); + + if (!acc[field]) { + acc[field] = []; + } + + const filter = { + value, + not: false, + operator: 'equals' + }; + + let operator = suffix.shift(); + + if (operator === 'not') { + filter.not = true; + operator = suffix.shift(); + } + + if (operator) { + filter.operator = operator; + } + + acc[field].push(filter); + + return acc; + }, {}); + + if (!queryOptions.limit) { + queryOptions.limit = DEFAULT_LIMIT; + } + + // Get entities from the database. + const entities = await this._database.getEntities(entity, relationsMap, block, where, queryOptions); + + // Resolve any field name conflicts in the entity result. + return entities.map(entity => resolveEntityFieldConflicts(entity)); + } + /** * Method to reinstantiate WASM instance for specified dataSource. * @param dataSourceName diff --git a/packages/graph-test-watcher/package.json b/packages/graph-test-watcher/package.json index b27d31dd..7fcdc75f 100644 --- a/packages/graph-test-watcher/package.json +++ b/packages/graph-test-watcher/package.json @@ -6,9 +6,13 @@ "main": "dist/index.js", "scripts": { "lint": "eslint .", - "build": "tsc", - "server": "DEBUG=vulcanize:* ts-node src/server.ts", - "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts", + "build": "yarn clean && tsc && yarn copy-assets", + "clean": "rm -rf ./dist", + "copy-assets": "copyfiles -u 1 src/**/*.gql dist/", + "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", + "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts", + "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", + "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts", "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts", "fill": "DEBUG=vulcanize:* ts-node src/fill.ts", "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts", @@ -66,6 +70,7 @@ "eslint-plugin-promise": "^5.1.0", "eslint-plugin-standard": "^5.0.0", "ts-node": "^10.0.0", - "typescript": "^4.3.2" + "typescript": "^4.3.2", + "copyfiles": "^2.4.1" } } diff --git a/packages/graph-test-watcher/src/entity/Author.ts b/packages/graph-test-watcher/src/entity/Author.ts index bd3c3ec2..e8e3ea24 100644 --- a/packages/graph-test-watcher/src/entity/Author.ts +++ b/packages/graph-test-watcher/src/entity/Author.ts @@ -2,12 +2,13 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import Decimal from 'decimal.js'; import { bigintTransformer, decimalTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Author { @PrimaryColumn('varchar') id!: string; diff --git a/packages/graph-test-watcher/src/entity/Blog.ts b/packages/graph-test-watcher/src/entity/Blog.ts index 80b2f592..6d8c54bd 100644 --- a/packages/graph-test-watcher/src/entity/Blog.ts +++ b/packages/graph-test-watcher/src/entity/Blog.ts @@ -2,7 +2,7 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintArrayTransformer } from '@vulcanize/util'; @@ -12,6 +12,7 @@ enum BlogType { } @Entity() +@Index(['blockNumber']) export class Blog { @PrimaryColumn('varchar') id!: string; diff --git a/packages/graph-test-watcher/src/entity/Category.ts b/packages/graph-test-watcher/src/entity/Category.ts index f866e9b3..1d8a5d7e 100644 --- a/packages/graph-test-watcher/src/entity/Category.ts +++ b/packages/graph-test-watcher/src/entity/Category.ts @@ -2,11 +2,12 @@ // Copyright 2021 Vulcanize, Inc. // -import { Entity, PrimaryColumn, Column } from 'typeorm'; +import { Entity, PrimaryColumn, Column, Index } from 'typeorm'; import { bigintTransformer } from '@vulcanize/util'; @Entity() +@Index(['blockNumber']) export class Category { @PrimaryColumn('varchar') id!: string; diff --git a/packages/graph-test-watcher/src/indexer.ts b/packages/graph-test-watcher/src/indexer.ts index 5c0126c6..bc37bce1 100644 --- a/packages/graph-test-watcher/src/indexer.ts +++ b/packages/graph-test-watcher/src/indexer.ts @@ -376,9 +376,7 @@ export class Indexer implements IPLDIndexerInterface { } async getSubgraphEntity (entity: new () => Entity, id: string, block: BlockHeight): Promise { - const relations = this._relationsMap.get(entity) || {}; - - const data = await this._graphWatcher.getEntity(entity, id, relations, block); + const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block); return data; } @@ -663,6 +661,7 @@ export class Indexer implements IPLDIndexerInterface { const blockPromise = this._ethClient.getBlockByHash(blockHash); let logs: any[]; + console.time('time:indexer#_fetchAndSaveEvents-fetch-logs'); if (this._serverConfig.filterLogs) { const watchedContracts = this._baseIndexer.getWatchedContracts(); @@ -684,6 +683,7 @@ export class Indexer implements IPLDIndexerInterface { } else { ({ logs } = await this._ethClient.getLogs({ blockHash })); } + console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs'); let [ { block }, @@ -775,8 +775,10 @@ export class Indexer implements IPLDIndexerInterface { parentHash: block.parent.hash }; + console.time('time:indexer#_fetchAndSaveEvents-save-block-events'); const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents); await dbTx.commitTransaction(); + console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events'); return blockProgress; } catch (error) { diff --git a/packages/graph-test-watcher/tsconfig.json b/packages/graph-test-watcher/tsconfig.json index b30e1629..f009958d 100644 --- a/packages/graph-test-watcher/tsconfig.json +++ b/packages/graph-test-watcher/tsconfig.json @@ -12,7 +12,7 @@ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - // "sourceMap": true, /* Generates corresponding '.map' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "dist", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts index 0f644a47..8883abcf 100644 --- a/packages/ipld-eth-client/src/eth-client.ts +++ b/packages/ipld-eth-client/src/eth-client.ts @@ -40,7 +40,9 @@ export class EthClient { async getStorageAt ({ blockHash, contract, slot }: { blockHash: string, contract: string, slot: string }): Promise<{ value: string, proof: { data: string } }> { slot = `0x${padKey(slot)}`; + console.time(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`); const result = await this._getCachedOrFetch('getStorageAt', { blockHash, contract, slot }); + console.timeEnd(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`); const { getStorageAt: { value, cid, ipldBlock } } = result; return { @@ -62,46 +64,64 @@ export class EthClient { } async getBlockWithTransactions ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { - return this._graphqlClient.query( + console.time(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`); + const result = await this._graphqlClient.query( ethQueries.getBlockWithTransactions, { blockNumber: blockNumber?.toString(), blockHash } ); + console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`); + + return result; } async getBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { - return this._graphqlClient.query( + console.time(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + const result = await this._graphqlClient.query( ethQueries.getBlocks, { blockNumber: blockNumber?.toString(), blockHash } ); + console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + + return result; } async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise { - return this._graphqlClient.query( + console.time(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + const result = await this._graphqlClient.query( ethQueries.getFullBlocks, { blockNumber: blockNumber?.toString(), blockHash } ); + console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`); + + return result; } async getFullTransaction (txHash: string): Promise { - return this._graphqlClient.query( + console.time(`time:eth-client#getFullTransaction-${txHash}`); + const result = this._graphqlClient.query( ethQueries.getFullTransaction, { txHash } ); + console.timeEnd(`time:eth-client#getFullTransaction-${txHash}`); + + return result; } async getBlockByHash (blockHash?: string): Promise { + console.time(`time:eth-client#getBlockByHash-${blockHash}`); const result = await this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash }); + console.timeEnd(`time:eth-client#getBlockByHash-${blockHash}`); return { block: { @@ -113,7 +133,9 @@ export class EthClient { } async getLogs (vars: Vars): Promise { + console.time(`time:eth-client#getLogs-${JSON.stringify(vars)}`); const result = await this._getCachedOrFetch('getLogs', vars); + console.timeEnd(`time:eth-client#getLogs-${JSON.stringify(vars)}`); const { getLogs } = result; diff --git a/packages/mobymask-watcher/environments/local.toml b/packages/mobymask-watcher/environments/local.toml index 5a4a34fa..603a0b59 100644 --- a/packages/mobymask-watcher/environments/local.toml +++ b/packages/mobymask-watcher/environments/local.toml @@ -22,6 +22,8 @@ [metrics] host = "127.0.0.1" port = 9000 + [metrics.gql] + port = 9001 [database] type = "postgres" diff --git a/packages/mobymask-watcher/package.json b/packages/mobymask-watcher/package.json index 09cd120c..c5a0fa94 100644 --- a/packages/mobymask-watcher/package.json +++ b/packages/mobymask-watcher/package.json @@ -6,9 +6,13 @@ "main": "dist/index.js", "scripts": { "lint": "eslint .", - "build": "tsc", - "server": "DEBUG=vulcanize:* ts-node src/server.ts", - "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts", + "build": "yarn clean && tsc && yarn copy-assets", + "clean": "rm -rf ./dist", + "copy-assets": "copyfiles -u 1 src/**/*.gql dist/", + "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js", + "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts", + "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js", + "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts", "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts", "fill": "DEBUG=vulcanize:* ts-node src/fill.ts", "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts", @@ -62,6 +66,7 @@ "eslint-plugin-promise": "^5.1.0", "eslint-plugin-standard": "^5.0.0", "ts-node": "^10.0.0", - "typescript": "^4.3.2" + "typescript": "^4.3.2", + "copyfiles": "^2.4.1" } } diff --git a/packages/mobymask-watcher/src/indexer.ts b/packages/mobymask-watcher/src/indexer.ts index a89d2810..3c704cdb 100644 --- a/packages/mobymask-watcher/src/indexer.ts +++ b/packages/mobymask-watcher/src/indexer.ts @@ -733,6 +733,7 @@ export class Indexer implements IPLDIndexerInterface { const blockPromise = this._ethClient.getBlockByHash(blockHash); let logs: any[]; + console.time('time:indexer#_fetchAndSaveEvents-fetch-logs'); if (this._serverConfig.filterLogs) { const watchedContracts = this._baseIndexer.getWatchedContracts(); @@ -754,6 +755,7 @@ export class Indexer implements IPLDIndexerInterface { } else { ({ logs } = await this._ethClient.getLogs({ blockHash })); } + console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs'); let [ { block }, @@ -845,8 +847,10 @@ export class Indexer implements IPLDIndexerInterface { parentHash: block.parent.hash }; + console.time('time:indexer#_fetchAndSaveEvents-save-block-events'); const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents); await dbTx.commitTransaction(); + console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events'); return blockProgress; } catch (error) { diff --git a/packages/mobymask-watcher/src/resolvers.ts b/packages/mobymask-watcher/src/resolvers.ts index 0ca9b64c..50bab1c0 100644 --- a/packages/mobymask-watcher/src/resolvers.ts +++ b/packages/mobymask-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, StateKind } from '@vulcanize/util'; +import { ValueResult, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -60,31 +60,48 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch Query: { multiNonce: (_: any, { blockHash, contractAddress, key0, key1 }: { blockHash: string, contractAddress: string, key0: string, key1: bigint }): Promise => { log('multiNonce', blockHash, contractAddress, key0, key1); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('multiNonce').inc(1); + return indexer.multiNonce(blockHash, contractAddress, key0, key1); }, _owner: (_: any, { blockHash, contractAddress }: { blockHash: string, contractAddress: string }): Promise => { log('_owner', blockHash, contractAddress); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('_owner').inc(1); + return indexer._owner(blockHash, contractAddress); }, isRevoked: (_: any, { blockHash, contractAddress, key0 }: { blockHash: string, contractAddress: string, key0: string }): Promise => { log('isRevoked', blockHash, contractAddress, key0); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('isRevoked').inc(1); + return indexer.isRevoked(blockHash, contractAddress, key0); }, isPhisher: (_: any, { blockHash, contractAddress, key0 }: { blockHash: string, contractAddress: string, key0: string }): Promise => { log('isPhisher', blockHash, contractAddress, key0); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('isPhisher').inc(1); + return indexer.isPhisher(blockHash, contractAddress, key0); }, isMember: (_: any, { blockHash, contractAddress, key0 }: { blockHash: string, contractAddress: string, key0: string }): Promise => { log('isMember', blockHash, contractAddress, key0); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('isMember').inc(1); + return indexer.isMember(blockHash, contractAddress, key0); }, events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => { log('events', blockHash, contractAddress, name); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('events').inc(1); const block = await indexer.getBlockProgress(blockHash); if (!block || !block.isComplete) { @@ -97,6 +114,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => { log('eventsInRange', fromBlockNumber, toBlockNumber); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('eventsInRange').inc(1); const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber); return events.map(event => indexer.getResultEvent(event)); @@ -104,6 +123,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getStateByCID: async (_: any, { cid }: { cid: string }) => { log('getStateByCID', cid); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getStateByCID').inc(1); const ipldBlock = await indexer.getIPLDBlockByCid(cid); @@ -112,6 +133,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getState').inc(1); const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); @@ -120,12 +143,16 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch getSyncStatus: async () => { log('getSyncStatus'); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('getSyncStatus').inc(1); return indexer.getSyncStatus(); }, latestBlock: async () => { log('latestBlock'); + gqlTotalQueryCount.inc(1); + gqlQueryCount.labels('latestBlock').inc(1); return indexer.getLatestBlock(); } diff --git a/packages/mobymask-watcher/src/server.ts b/packages/mobymask-watcher/src/server.ts index 6d60c429..1ddbd68c 100644 --- a/packages/mobymask-watcher/src/server.ts +++ b/packages/mobymask-watcher/src/server.ts @@ -14,7 +14,7 @@ import debug from 'debug'; import 'graphql-import-node'; import { createServer } from 'http'; -import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util'; +import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util'; import { createResolvers } from './resolvers'; import { Indexer } from './indexer'; @@ -83,6 +83,8 @@ export const main = async (): Promise => { log(`Server is listening on host ${host} port ${port}`); }); + startGQLMetricsServer(config); + return { app, server }; }; diff --git a/packages/mobymask-watcher/tsconfig.json b/packages/mobymask-watcher/tsconfig.json index b30e1629..f009958d 100644 --- a/packages/mobymask-watcher/tsconfig.json +++ b/packages/mobymask-watcher/tsconfig.json @@ -12,7 +12,7 @@ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - // "sourceMap": true, /* Generates corresponding '.map' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "dist", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ diff --git a/packages/util/index.ts b/packages/util/index.ts index 841fb26b..37d3b544 100644 --- a/packages/util/index.ts +++ b/packages/util/index.ts @@ -19,3 +19,4 @@ export * from './src/ipld-database'; export * from './src/ipfs'; export * from './src/index-block'; export * from './src/metrics'; +export * from './src/gql-metrics'; diff --git a/packages/util/src/common.ts b/packages/util/src/common.ts index 1289a5c5..3f5283ef 100644 --- a/packages/util/src/common.ts +++ b/packages/util/src/common.ts @@ -67,9 +67,7 @@ export const processBlockByNumber = async ( }); if (!blocks.length) { - console.time('time:common#processBlockByNumber-ipld-eth-server'); blocks = await indexer.getBlocks({ blockNumber }); - console.timeEnd('time:common#processBlockByNumber-ipld-eth-server'); } if (blocks.length) { @@ -135,7 +133,7 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block log(`Processing events batch from index ${events[0].index} to ${events[0].index + events.length - 1}`); } - console.time('time:common#processBacthEvents-processing_events_batch'); + console.time('time:common#processBatchEvents-processing_events_batch'); for (let event of events) { // Process events in loop @@ -189,7 +187,7 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block block = await indexer.updateBlockProgress(block, event.index); } - console.timeEnd('time:common#processBacthEvents-processing_events_batch'); + console.timeEnd('time:common#processBatchEvents-processing_events_batch'); } if (indexer.processBlockAfterEvents) { @@ -199,5 +197,7 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block } block.isComplete = true; + console.time('time:common#processBatchEvents-updateBlockProgress'); await indexer.updateBlockProgress(block, block.lastProcessedEventIndex); + console.timeEnd('time:common#processBatchEvents-updateBlockProgress'); }; diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index 9ce6cbfd..e0ff9f23 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -56,9 +56,14 @@ export interface UpstreamConfig { } } +export interface GQLMetricsConfig { + port: number; +} + export interface MetricsConfig { host: string; port: number; + gql: GQLMetricsConfig; } export interface Config { diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 4cc6d375..cd69c91f 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -4,7 +4,6 @@ import assert from 'assert'; import { - Brackets, Connection, ConnectionOptions, createConnection, @@ -24,9 +23,6 @@ import { BlockProgressInterface, ContractInterface, EventInterface, SyncStatusIn import { MAX_REORG_DEPTH, UNKNOWN_EVENT_NAME } from './constants'; import { blockProgressCount, eventCount } from './metrics'; -const DEFAULT_LIMIT = 100; -const DEFAULT_SKIP = 0; - const OPERATOR_MAP = { equals: '=', gt: '>', @@ -207,13 +203,21 @@ export class Database { .innerJoinAndSelect('event.block', 'block') .where('block.block_hash = :blockHash AND block.is_pruned = false', { blockHash }); - queryBuilder = this._buildQuery(repo, queryBuilder, where, queryOptions); + queryBuilder = this.buildQuery(repo, queryBuilder, where); + + if (queryOptions.orderBy) { + queryBuilder = this.orderQuery(repo, queryBuilder, queryOptions); + } + queryBuilder.addOrderBy('event.id', 'ASC'); - const { limit = DEFAULT_LIMIT, skip = DEFAULT_SKIP } = queryOptions; + if (queryOptions.skip) { + queryBuilder = queryBuilder.offset(queryOptions.skip); + } - queryBuilder = queryBuilder.offset(skip) - .limit(limit); + if (queryOptions.limit) { + queryBuilder = queryBuilder.limit(queryOptions.limit); + } return queryBuilder.getMany(); } @@ -388,56 +392,6 @@ export class Database { return event; } - async getModelEntities (queryRunner: QueryRunner, entity: new () => Entity, block: BlockHeight, where: Where = {}, queryOptions: QueryOptions = {}, relations: Relation[] = []): Promise { - const repo = queryRunner.manager.getRepository(entity); - const { tableName } = repo.metadata; - - let subQuery = repo.createQueryBuilder('subTable') - .select('MAX(subTable.block_number)') - .where(`subTable.id = ${tableName}.id`); - - if (block.hash) { - const { canonicalBlockNumber, blockHashes } = await this.getFrothyRegion(queryRunner, block.hash); - - subQuery = subQuery - .andWhere(new Brackets(qb => { - qb.where('subTable.block_hash IN (:...blockHashes)', { blockHashes }) - .orWhere('subTable.block_number <= :canonicalBlockNumber', { canonicalBlockNumber }); - })); - } - - if (block.number) { - subQuery = subQuery.andWhere('subTable.block_number <= :blockNumber', { blockNumber: block.number }); - } - - let selectQueryBuilder = repo.createQueryBuilder(tableName) - .where(`${tableName}.block_number IN (${subQuery.getQuery()})`) - .setParameters(subQuery.getParameters()); - - relations.forEach(relation => { - let alias, property; - - if (typeof relation === 'string') { - [, alias] = relation.split('.'); - property = relation; - } else { - alias = relation.alias; - property = relation.property; - } - - selectQueryBuilder = selectQueryBuilder.leftJoinAndSelect(property, alias); - }); - - selectQueryBuilder = this._buildQuery(repo, selectQueryBuilder, where, queryOptions); - - const { limit = DEFAULT_LIMIT, skip = DEFAULT_SKIP } = queryOptions; - - selectQueryBuilder = selectQueryBuilder.skip(skip) - .take(limit); - - return selectQueryBuilder.getMany(); - } - async getFrothyEntity (queryRunner: QueryRunner, repo: Repository, data: { blockHash: string, id: string }): Promise<{ blockHash: string, blockNumber: number, id: string }> { // Hierarchical query for getting the entity in the frothy region. const heirerchicalQuery = ` @@ -596,16 +550,21 @@ export class Database { eventCount.set(this._eventCount); } - _buildQuery (repo: Repository, selectQueryBuilder: SelectQueryBuilder, where: Where = {}, queryOptions: QueryOptions = {}): SelectQueryBuilder { - const { tableName } = repo.metadata; - + buildQuery (repo: Repository, selectQueryBuilder: SelectQueryBuilder, where: Where = {}): SelectQueryBuilder { Object.entries(where).forEach(([field, filters]) => { filters.forEach((filter, index) => { // Form the where clause. let { not, operator, value } = filter; const columnMetadata = repo.metadata.findColumnWithPropertyName(field); assert(columnMetadata); - let whereClause = `${tableName}.${columnMetadata.propertyAliasName} `; + let whereClause = `"${selectQueryBuilder.alias}"."${columnMetadata.databaseName}" `; + + if (columnMetadata.relationMetadata) { + // For relation fields, use the id column. + const idColumn = columnMetadata.relationMetadata.joinColumns.find(column => column.referencedColumn?.propertyName === 'id'); + assert(idColumn); + whereClause = `"${selectQueryBuilder.alias}"."${idColumn.databaseName}" `; + } if (not) { if (operator === 'equals') { @@ -617,9 +576,7 @@ export class Database { whereClause += `${OPERATOR_MAP[operator]} `; - if (['contains', 'starts'].some(el => el === operator)) { - whereClause += '%:'; - } else if (operator === 'in') { + if (operator === 'in') { whereClause += '(:...'; } else { // Convert to string type value as bigint type throws error in query. @@ -631,9 +588,7 @@ export class Database { const variableName = `${field}${index}`; whereClause += variableName; - if (['contains', 'ends'].some(el => el === operator)) { - whereClause += '%'; - } else if (operator === 'in') { + if (operator === 'in') { whereClause += ')'; if (!value.length) { @@ -641,18 +596,35 @@ export class Database { } } + if (['contains', 'starts'].some(el => el === operator)) { + value = `%${value}`; + } + + if (['contains', 'ends'].some(el => el === operator)) { + value += '%'; + } + selectQueryBuilder = selectQueryBuilder.andWhere(whereClause, { [variableName]: value }); }); }); - const { orderBy, orderDirection } = queryOptions; - - if (orderBy) { - const columnMetadata = repo.metadata.findColumnWithPropertyName(orderBy); - assert(columnMetadata); - selectQueryBuilder = selectQueryBuilder.orderBy(`${tableName}.${columnMetadata.propertyAliasName}`, orderDirection === 'desc' ? 'DESC' : 'ASC'); - } - return selectQueryBuilder; } + + orderQuery ( + repo: Repository, + selectQueryBuilder: SelectQueryBuilder, + orderOptions: { orderBy?: string, orderDirection?: string } + ): SelectQueryBuilder { + const { orderBy, orderDirection } = orderOptions; + assert(orderBy); + + const columnMetadata = repo.metadata.findColumnWithPropertyName(orderBy); + assert(columnMetadata); + + return selectQueryBuilder.addOrderBy( + `${selectQueryBuilder.alias}.${columnMetadata.propertyAliasName}`, + orderDirection === 'desc' ? 'DESC' : 'ASC' + ); + } } diff --git a/packages/util/src/gql-metrics.ts b/packages/util/src/gql-metrics.ts new file mode 100644 index 00000000..eec55254 --- /dev/null +++ b/packages/util/src/gql-metrics.ts @@ -0,0 +1,51 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import * as client from 'prom-client'; +import express, { Application } from 'express'; +import debug from 'debug'; +import assert from 'assert'; + +import { Config } from './config'; + +const log = debug('vulcanize:gql-metrics'); + +const gqlRegistry = new client.Registry(); + +// Create custom metrics +export const gqlTotalQueryCount = new client.Counter({ + name: 'gql_query_count_total', + help: 'Total GQL queries made', + registers: [gqlRegistry] +}); + +export const gqlQueryCount = new client.Counter({ + name: 'gql_query_count', + help: 'GQL queries made', + labelNames: ['name'] as const, + registers: [gqlRegistry] +}); + +// Export metrics on a server +const app: Application = express(); + +export const startGQLMetricsServer = async (config: Config): Promise => { + if (!config.metrics || !config.metrics.gql) { + log('GQL metrics disabled. To enable add GQL metrics host and port.'); + return; + } + + assert(config.metrics.host, 'Missing config for metrics host'); + assert(config.metrics.gql.port, 'Missing config for gql metrics port'); + + app.get('/metrics', async (req, res) => { + res.setHeader('Content-Type', gqlRegistry.contentType); + const metrics = await gqlRegistry.metrics(); + res.send(metrics); + }); + + app.listen(config.metrics.gql.port, config.metrics.host, () => { + log(`GQL Metrics exposed at http://${config.metrics.host}:${config.metrics.gql.port}/metrics`); + }); +}; diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index 37585c3b..39f1ce04 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -79,6 +79,7 @@ export class JobRunner { } async _pruneChain (job: any, syncStatus: SyncStatusInterface): Promise { + console.time('time:job-runner#_pruneChain'); const { pruneBlockHeight } = job.data; log(`Processing chain pruning at ${pruneBlockHeight}`); @@ -119,6 +120,8 @@ export class JobRunner { // Update the canonical block in the SyncStatus. await this._indexer.updateSyncStatusCanonicalBlock(newCanonicalBlockHash, pruneBlockHeight); } + + console.timeEnd('time:job-runner#_pruneChain'); } async _indexBlock (job: any, syncStatus: SyncStatusInterface): Promise { @@ -145,6 +148,7 @@ export class JobRunner { throw new Error(message); } + console.time('time:job-runner#_indexBlock-get-block-progress-entities'); let [parentBlock, blockProgress] = await this._indexer.getBlockProgressEntities( { blockHash: In([parentHash, blockHash]) @@ -155,6 +159,7 @@ export class JobRunner { } } ); + console.timeEnd('time:job-runner#_indexBlock-get-block-progress-entities'); // Check if parent block has been processed yet, if not, push a high priority job to process that first and abort. // However, don't go beyond the `latestCanonicalBlockHash` from SyncStatus as we have to assume the reorg can't be that deep. @@ -209,7 +214,9 @@ export class JobRunner { throw new Error(message); } else { // Remove the unknown events of the parent block if it is marked complete. + console.time('time:job-runner#_indexBlock-remove-unknown-events'); await this._indexer.removeUnknownEvents(parentBlock); + console.timeEnd('time:job-runner#_indexBlock-remove-unknown-events'); } } else { blockProgress = parentBlock; @@ -220,7 +227,9 @@ export class JobRunner { // Delay required to process block. await wait(jobDelayInMilliSecs); + console.time('time:job-runner#_indexBlock-fetch-block-events'); blockProgress = await this._indexer.fetchBlockEvents({ cid, blockHash, blockNumber, parentHash, blockTimestamp: timestamp }); + console.timeEnd('time:job-runner#_indexBlock-fetch-block-events'); } if (this._indexer.processBlock) { diff --git a/packages/util/src/misc.ts b/packages/util/src/misc.ts index 1e5d44ae..241e6852 100644 --- a/packages/util/src/misc.ts +++ b/packages/util/src/misc.ts @@ -191,7 +191,9 @@ export const getFullBlock = async (ethClient: EthClient, ethProvider: providers. // TODO: Calculate size from rlp encoded data. // Get block info from JSON RPC API provided by ipld-eth-server. const provider = ethProvider as providers.JsonRpcProvider; + console.time('time:misc#getFullBlock-eth_getBlockByHash'); const { size } = await provider.send('eth_getBlockByHash', [blockHash, false]); + console.timeEnd('time:misc#getFullBlock-eth_getBlockByHash'); return { headerId: fullBlock.id, diff --git a/yarn.lock b/yarn.lock index 4432cbdd..ca22cd34 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5147,6 +5147,19 @@ copy-descriptor@^0.1.0: resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= +copyfiles@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/copyfiles/-/copyfiles-2.4.1.tgz#d2dcff60aaad1015f09d0b66e7f0f1c5cd3c5da5" + integrity sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg== + dependencies: + glob "^7.0.5" + minimatch "^3.0.3" + mkdirp "^1.0.4" + noms "0.0.0" + through2 "^2.0.1" + untildify "^4.0.0" + yargs "^16.1.0" + core-js-pure@^3.0.1: version "3.13.0" resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.13.0.tgz#9d267fb47d1d7046cfbc05e7b67bb235b6735355" @@ -7463,6 +7476,18 @@ glob@^7.0.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, gl once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + global-dirs@^2.0.1: version "2.1.0" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.1.0.tgz#e9046a49c806ff04d6c1825e196c8f0091e8df4d" @@ -10003,6 +10028,13 @@ minimatch@5.0.1: dependencies: brace-expansion "^2.0.1" +minimatch@^3.0.3, minimatch@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + minimist-options@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" @@ -10514,6 +10546,14 @@ nodemon@^2.0.7: undefsafe "^2.0.3" update-notifier "^4.1.0" +noms@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/noms/-/noms-0.0.0.tgz#da8ebd9f3af9d6760919b27d9cdc8092a7332859" + integrity sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow== + dependencies: + inherits "^2.0.1" + readable-stream "~1.0.31" + nopt@^4.0.1: version "4.0.3" resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" @@ -12069,7 +12109,7 @@ readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@~1.0.15: +readable-stream@~1.0.15, readable-stream@~1.0.31: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" integrity sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw= @@ -13443,7 +13483,7 @@ thenify-all@^1.0.0: dependencies: any-promise "^1.0.0" -through2@^2.0.0, through2@^2.0.3: +through2@^2.0.0, through2@^2.0.1, through2@^2.0.3: version "2.0.5" resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd" integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ== @@ -14008,6 +14048,11 @@ unset-value@^1.0.0: has-value "^0.3.1" isobject "^3.0.0" +untildify@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" + integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== + upath@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/upath/-/upath-2.0.1.tgz#50c73dea68d6f6b990f51d279ce6081665d61a8b"