cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2024-11-19 20:36:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement query for multiple entities and nested relation fields in eden-watcher (#166)

Browse Source 
* Implement query for multiple entities in eden-watcher

* Implement nested relation queries

* Implement GQL query params first, skip, orderBy, orderDirection

* Add blockNumber index to subgraph entities

* Add logs for timing eth-calls and storage calls

* Add prometheus metrics to monitor GQL queries

* Fix default limit and order of 1-N related field in GQL entitiy query

* Add timer logs for block processing

* Run transpiled js in all watchers

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
This commit is contained in:
nikugogoi 2022-09-01 14:17:43 +05:30 committed by GitHub
parent 97e88ab5f0
commit 8af7417df6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
60 changed files with 777 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/codegen/src/entity.ts
View File

@ -209,11 +209,15 @@ export class Entity {
entityObject.imports.push(
{
toImport: new Set(['Entity', 'PrimaryColumn', 'Column']),
toImport: new Set(['Entity', 'PrimaryColumn', 'Column', 'Index']),
from: 'typeorm'
}
);
entityObject.indexOn.push({
columns: ['blockNumber']
});
// Add common columns.
entityObject.columns.push({
name: 'id',

2
packages/codegen/src/templates/config-template.handlebars
View File

@ -27,6 +27,8 @@
[metrics]
host = "127.0.0.1"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"

6
packages/codegen/src/templates/indexer-template.handlebars
View File

@ -430,7 +430,7 @@ export class Indexer implements IPLDIndexerInterface {
async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
const relations = this._relationsMap.get(entity) || {};
const data = await this._graphWatcher.getEntity(entity, id, relations, block);
const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
return data;
}
@ -704,6 +704,7 @@ export class Indexer implements IPLDIndexerInterface {
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
@ -725,6 +726,7 @@ export class Indexer implements IPLDIndexerInterface {
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
@ -816,8 +818,10 @@ export class Indexer implements IPLDIndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {

13
packages/codegen/src/templates/package-template.handlebars
View File

@ -6,9 +6,13 @@
"main": "dist/index.js",
"scripts": {
"lint": "eslint .",
"build": "tsc",
"server": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"build": "yarn clean && tsc && yarn copy-assets",
"clean": "rm -rf ./dist",
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
"server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
"server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
"job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
"fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
"reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@ -65,6 +69,7 @@
"eslint-plugin-promise": "^5.1.0",
"eslint-plugin-standard": "^5.0.0",
"ts-node": "^10.0.0",
"typescript": "^4.3.2"
"typescript": "^4.3.2",
"copyfiles": "^2.4.1"
}
}

17
packages/codegen/src/templates/resolvers-template.handlebars
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, BlockHeight, StateKind } from '@vulcanize/util';
import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -68,6 +68,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
{{~#each this.params}}, {{this.name}}: {{this.type~}} {{/each}} }): Promise<ValueResult> => {
log('{{this.name}}', blockHash, contractAddress
{{~#each this.params}}, {{this.name~}} {{/each}});
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('{{this.name}}').inc(1);
return indexer.{{this.name}}(blockHash, contractAddress
{{~#each this.params}}, {{this.name~}} {{/each}});
},
@ -77,6 +80,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
{{~#each subgraphQueries}}
{{this.queryName}}: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('{{this.queryName}}', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('{{this.queryName}}').inc(1);
return indexer.getSubgraphEntity({{this.entityName}}, id, block);
},
@ -84,6 +89,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
{{/each}}
events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => {
log('events', blockHash, contractAddress, name);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('events').inc(1);
const block = await indexer.getBlockProgress(blockHash);
if (!block || !block.isComplete) {
@ -96,6 +103,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => {
log('eventsInRange', fromBlockNumber, toBlockNumber);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('eventsInRange').inc(1);
const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
if (expected !== actual) {
@ -108,6 +117,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getStateByCID: async (_: any, { cid }: { cid: string }) => {
log('getStateByCID', cid);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getStateByCID').inc(1);
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
@ -116,6 +127,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
@ -124,6 +137,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getSyncStatus: async () => {
log('getSyncStatus');
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getSyncStatus').inc(1);
return indexer.getSyncStatus();
}

8
packages/codegen/src/templates/server-template.handlebars
View File

@ -14,7 +14,7 @@ import debug from 'debug';
import 'graphql-import-node';
import { createServer } from 'http';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}
@ -45,10 +45,10 @@ export const main = async (): Promise<any> => {
const db = new Database(config.database);
await db.init();
{{#if (subgraphPath)}}
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}
@ -98,6 +98,8 @@ export const main = async (): Promise<any> => {
log(`Server is listening on host ${host} port ${port}`);
});
startGQLMetricsServer(config);
return { app, server };
};

4
packages/codegen/src/templates/tsconfig-template.handlebars
View File

@ -12,9 +12,9 @@
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */

2
packages/eden-watcher/environments/local.toml
View File

@ -25,6 +25,8 @@
[metrics]
host = "127.0.0.1"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"

17
packages/eden-watcher/package.json
View File

@ -6,9 +6,13 @@
"main": "dist/index.js",
"scripts": {
"lint": "eslint .",
"build": "tsc",
"server": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"build": "yarn clean && tsc && yarn copy-assets",
"clean": "rm -rf ./dist",
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
"server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
"server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
"job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
"fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
"reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@ -32,13 +36,14 @@
"@apollo/client": "^3.3.19",
"@ethersproject/providers": "^5.4.4",
"@ipld/dag-cbor": "^6.0.12",
"@vulcanize/graph-node": "^0.1.0",
"@vulcanize/ipld-eth-client": "^0.1.0",
"@vulcanize/solidity-mapper": "^0.1.0",
"@vulcanize/util": "^0.1.0",
"@vulcanize/graph-node": "^0.1.0",
"apollo-server-express": "^2.25.0",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",
"decimal.js": "^10.3.1",
"ethers": "^5.4.4",
"express": "^4.17.1",
"graphql": "^15.5.0",
@ -46,8 +51,7 @@
"json-bigint": "^1.0.0",
"reflect-metadata": "^0.1.13",
"typeorm": "^0.2.32",
"yargs": "^17.0.1",
"decimal.js": "^10.3.1"
"yargs": "^17.0.1"
},
"devDependencies": {
"@ethersproject/abi": "^5.3.0",
@ -55,6 +59,7 @@
"@types/yargs": "^17.0.0",
"@typescript-eslint/eslint-plugin": "^4.25.0",
"@typescript-eslint/parser": "^4.25.0",
"copyfiles": "^2.4.1",
"eslint": "^7.27.0",
"eslint-config-semistandard": "^15.0.1",
"eslint-config-standard": "^16.0.3",

3
packages/eden-watcher/src/entity/Account.ts
View File

@ -2,11 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Account {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Block.ts
View File

@ -2,10 +2,11 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Block {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Claim.ts
View File

@ -2,11 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Claim {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Distribution.ts
View File

@ -2,11 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Distribution {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Distributor.ts
View File

@ -2,9 +2,10 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
@Entity()
@Index(['blockNumber'])
export class Distributor {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Epoch.ts
View File

@ -2,12 +2,13 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import Decimal from 'decimal.js';
import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Epoch {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Network.ts
View File

@ -2,11 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintArrayTransformer, bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Network {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Producer.ts
View File

@ -2,10 +2,11 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Producer {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/ProducerEpoch.ts
View File

@ -2,12 +2,13 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import Decimal from 'decimal.js';
import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class ProducerEpoch {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/ProducerRewardCollectorChange.ts
View File

@ -2,10 +2,11 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class ProducerRewardCollectorChange {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/ProducerSet.ts
View File

@ -2,9 +2,10 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
@Entity()
@Index(['blockNumber'])
export class ProducerSet {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/ProducerSetChange.ts
View File

@ -2,7 +2,7 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
enum ProducerSetChangeType {
@ -11,6 +11,7 @@ enum ProducerSetChangeType {
}
@Entity()
@Index(['blockNumber'])
export class ProducerSetChange {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/RewardSchedule.ts
View File

@ -2,9 +2,10 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
@Entity()
@Index(['blockNumber'])
export class RewardSchedule {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/RewardScheduleEntry.ts
View File

@ -2,10 +2,11 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class RewardScheduleEntry {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Slash.ts
View File

@ -2,11 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Slash {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Slot.ts
View File

@ -2,12 +2,13 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import Decimal from 'decimal.js';
import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Slot {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/SlotClaim.ts
View File

@ -2,12 +2,13 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import Decimal from 'decimal.js';
import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class SlotClaim {
@PrimaryColumn('varchar')
id!: string;

3
packages/eden-watcher/src/entity/Staker.ts
View File

@ -2,10 +2,11 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Staker {
@PrimaryColumn('varchar')
id!: string;

12
packages/eden-watcher/src/indexer.ts
View File

@ -357,13 +357,15 @@ export class Indexer implements IPLDIndexerInterface {
}
async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
const relations = this._relationsMap.get(entity) || {};
const data = await this._graphWatcher.getEntity(entity, id, relations, block);
const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
return data;
}
async getSubgraphEntities<Entity> (entity: new () => Entity, block: BlockHeight, where: { [key: string]: any } = {}, queryOptions: QueryOptions = {}): Promise<any[]> {
return this._graphWatcher.getEntities(entity, this._relationsMap, block, where, queryOptions);
}
async triggerIndexingOnEvent (event: Event): Promise<void> {
const resultEvent = this.getResultEvent(event);
@ -956,6 +958,7 @@ export class Indexer implements IPLDIndexerInterface {
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
@ -977,6 +980,7 @@ export class Indexer implements IPLDIndexerInterface {
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
@ -1068,8 +1072,10 @@ export class Indexer implements IPLDIndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {

100
packages/eden-watcher/src/resolvers.ts
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { BlockHeight, StateKind } from '@vulcanize/util';
import { BlockHeight, OrderDirection, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -79,114 +79,204 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
Query: {
producer: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('producer', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('producer').inc(1);
return indexer.getSubgraphEntity(Producer, id, block);
},
producers: async (_: any, { block = {}, first, skip }: { block: BlockHeight, first: number, skip: number }) => {
log('producers', block, first, skip);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('producers').inc(1);
return indexer.getSubgraphEntities(
Producer,
block,
{},
{ limit: first, skip }
);
},
producerSet: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('producerSet', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('producerSet').inc(1);
return indexer.getSubgraphEntity(ProducerSet, id, block);
},
producerSetChange: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('producerSetChange', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('producerSetChange').inc(1);
return indexer.getSubgraphEntity(ProducerSetChange, id, block);
},
producerRewardCollectorChange: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('producerRewardCollectorChange', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('producerRewardCollectorChange').inc(1);
return indexer.getSubgraphEntity(ProducerRewardCollectorChange, id, block);
},
rewardScheduleEntry: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('rewardScheduleEntry', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('rewardScheduleEntry').inc(1);
return indexer.getSubgraphEntity(RewardScheduleEntry, id, block);
},
rewardSchedule: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('rewardSchedule', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('rewardSchedule').inc(1);
return indexer.getSubgraphEntity(RewardSchedule, id, block);
},
producerEpoch: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('producerEpoch', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('producerEpoch').inc(1);
return indexer.getSubgraphEntity(ProducerEpoch, id, block);
},
block: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('block', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('block').inc(1);
return indexer.getSubgraphEntity(Block, id, block);
},
blocks: async (_: any, { block = {}, where, first, skip, orderBy, orderDirection }: { block: BlockHeight, where: { [key: string]: any }, first: number, skip: number, orderBy: string, orderDirection: OrderDirection }) => {
log('blocks', block, where, first, skip, orderBy, orderDirection);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('blocks').inc(1);
return indexer.getSubgraphEntities(
Block,
block,
where,
{ limit: first, skip, orderBy, orderDirection }
);
},
epoch: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('epoch', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('epoch').inc(1);
return indexer.getSubgraphEntity(Epoch, id, block);
},
epoches: async (_: any, { block = {}, where, first, skip }: { block: BlockHeight, where: { [key: string]: any }, first: number, skip: number }) => {
log('epoches', block, where, first, skip);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('epoches').inc(1);
return indexer.getSubgraphEntities(
Epoch,
block,
where,
{ limit: first, skip }
);
},
slotClaim: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('slotClaim', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('slotClaim').inc(1);
return indexer.getSubgraphEntity(SlotClaim, id, block);
},
slot: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('slot', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('slot').inc(1);
return indexer.getSubgraphEntity(Slot, id, block);
},
staker: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('staker', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('staker').inc(1);
return indexer.getSubgraphEntity(Staker, id, block);
},
stakers: async (_: any, { block = {}, where, first, skip, orderBy, orderDirection }: { block: BlockHeight, where: { [key: string]: any }, first: number, skip: number, orderBy: string, orderDirection: OrderDirection }) => {
log('stakers', block, where, first, skip, orderBy, orderDirection);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('stakers').inc(1);
return indexer.getSubgraphEntities(
Staker,
block,
where,
{ limit: first, skip, orderBy, orderDirection }
);
},
network: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('network', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('network').inc(1);
return indexer.getSubgraphEntity(Network, id, block);
},
distributor: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('distributor', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('distributor').inc(1);
return indexer.getSubgraphEntity(Distributor, id, block);
},
distribution: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('distribution', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('distribution').inc(1);
return indexer.getSubgraphEntity(Distribution, id, block);
},
claim: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('claim', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('claim').inc(1);
return indexer.getSubgraphEntity(Claim, id, block);
},
slash: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('slash', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('slash').inc(1);
return indexer.getSubgraphEntity(Slash, id, block);
},
account: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('account', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('account').inc(1);
return indexer.getSubgraphEntity(Account, id, block);
},
events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => {
log('events', blockHash, contractAddress, name);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('events').inc(1);
const block = await indexer.getBlockProgress(blockHash);
if (!block || !block.isComplete) {
@ -199,6 +289,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => {
log('eventsInRange', fromBlockNumber, toBlockNumber);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('eventsInRange').inc(1);
const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
if (expected !== actual) {
@ -211,6 +303,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getStateByCID: async (_: any, { cid }: { cid: string }) => {
log('getStateByCID', cid);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getStateByCID').inc(1);
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
@ -219,6 +313,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
@ -227,6 +323,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getSyncStatus: async () => {
log('getSyncStatus');
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getSyncStatus').inc(1);
return indexer.getSyncStatus();
}

34
packages/eden-watcher/src/schema.gql
View File

@ -218,10 +218,16 @@ type SyncStatus {
latestCanonicalBlockNumber: Int!
}
enum OrderDirection {
asc
desc
}
type Query {
events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
producer(id: String!, block: Block_height): Producer!
producers(block: Block_height, first: Int = 100, skip: Int = 0): [Producer!]!
producerSet(id: String!, block: Block_height): ProducerSet!
producerSetChange(id: String!, block: Block_height): ProducerSetChange!
producerRewardCollectorChange(id: String!, block: Block_height): ProducerRewardCollectorChange!
@ -229,10 +235,13 @@ type Query {
rewardSchedule(id: String!, block: Block_height): RewardSchedule!
producerEpoch(id: String!, block: Block_height): ProducerEpoch!
block(id: String!, block: Block_height): Block!
blocks(where: Block_filter, block: Block_height, orderBy: Block_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Block!]!
epoch(id: String!, block: Block_height): Epoch!
epoches(where: Epoch_filter, block: Block_height, first: Int = 100, skip: Int = 0): [Epoch!]!
slotClaim(id: String!, block: Block_height): SlotClaim!
slot(id: String!, block: Block_height): Slot!
staker(id: String!, block: Block_height): Staker!
stakers(where: Staker_filter, block: Block_height, orderBy: Staker_orderBy, orderDirection: OrderDirection, first: Int = 100, skip: Int = 0): [Staker!]!
network(id: String!, block: Block_height): Network!
distributor(id: String!, block: Block_height): Distributor!
distribution(id: String!, block: Block_height): Distribution!
@ -311,6 +320,18 @@ type Block {
size: BigInt
}
input Block_filter {
fromActiveProducer: Boolean
number_gte: BigInt
number_lte: BigInt
timestamp_lte: BigInt
}
enum Block_orderBy {
number
timestamp
}
type Epoch {
id: ID!
finalized: Boolean!
@ -323,6 +344,11 @@ type Epoch {
producerRewards: [ProducerEpoch!]!
}
input Epoch_filter {
epochNumber_gte: BigInt
epochNumber_lte: BigInt
}
type ProducerEpoch {
id: ID!
address: Bytes!
@ -361,6 +387,14 @@ type Staker {
rank: BigInt
}
input Staker_filter {
rank_gte: BigInt
}
enum Staker_orderBy {
rank
}
type Network {
id: ID!
slot0: Slot

4
packages/eden-watcher/src/server.ts
View File

@ -14,7 +14,7 @@ import debug from 'debug';
import 'graphql-import-node';
import { createServer } from 'http';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util';
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
import { createResolvers } from './resolvers';
@ -92,6 +92,8 @@ export const main = async (): Promise<any> => {
log(`Server is listening on host ${host} port ${port}`);
});
startGQLMetricsServer(config);
return { app, server };
};

2
packages/eden-watcher/tsconfig.json
View File

@ -12,7 +12,7 @@
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */

2
packages/erc20-watcher/src/indexer.ts
View File

@ -450,8 +450,10 @@ export class Indexer implements IndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {

13
packages/erc721-watcher/package.json
View File

@ -6,9 +6,13 @@
"main": "dist/index.js",
"scripts": {
"lint": "eslint .",
"build": "tsc",
"server": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"build": "yarn clean && tsc && yarn copy-assets",
"clean": "rm -rf ./dist",
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
"server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
"server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
"job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
"fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
"reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@ -70,6 +74,7 @@
"eslint-plugin-standard": "^5.0.0",
"ts-node": "^10.0.0",
"typescript": "^4.3.2",
"hardhat": "^2.3.0"
"hardhat": "^2.3.0",
"copyfiles": "^2.4.1"
}
}

4
packages/erc721-watcher/src/indexer.ts
View File

@ -989,6 +989,7 @@ export class Indexer implements IPLDIndexerInterface {
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
@ -1010,6 +1011,7 @@ export class Indexer implements IPLDIndexerInterface {
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
@ -1101,8 +1103,10 @@ export class Indexer implements IPLDIndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {

2
packages/erc721-watcher/tsconfig.json
View File

@ -12,7 +12,7 @@
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */

278
packages/graph-node/src/database.ts
View File

@ -4,6 +4,7 @@
import assert from 'assert';
import {
Brackets,
Connection,
ConnectionOptions,
FindOneOptions,
@ -14,7 +15,9 @@ import {
import {
BlockHeight,
BlockProgressInterface,
Database as BaseDatabase
Database as BaseDatabase,
QueryOptions,
Where
} from '@vulcanize/util';
import { Block, fromEntityValue, toEntityValue } from './utils';
@ -92,7 +95,7 @@ export class Database {
return entities.map((entity: any) => entity.id);
}
async getEntityWithRelations<Entity> (entity: (new () => Entity) | string, id: string, relations: { [key: string]: any }, block: BlockHeight = {}): Promise<Entity | undefined> {
async getEntityWithRelations<Entity> (entity: (new () => Entity), id: string, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight = {}): Promise<Entity | undefined> {
const queryRunner = this._conn.createQueryRunner();
let { hash: blockHash, number: blockNumber } = block;
@ -124,75 +127,9 @@ export class Database {
entityData = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions);
}
// Get relational fields
if (entityData) {
// Populate relational fields.
// TODO: Implement query for nested relations.
const relationQueryPromises = Object.entries(relations).map(async ([field, data]) => {
assert(entityData);
const { entity: relatedEntity, isArray, isDerived, field: derivedField } = data;
const repo = queryRunner.manager.getRepository(relatedEntity);
let selectQueryBuilder = repo.createQueryBuilder('entity');
if (isDerived) {
// For derived relational field.
selectQueryBuilder = selectQueryBuilder.where(`entity.${derivedField} = :id`, { id: entityData.id });
if (isArray) {
selectQueryBuilder = selectQueryBuilder.distinctOn(['entity.id'])
.orderBy('entity.id')
.limit(DEFAULT_LIMIT);
} else {
selectQueryBuilder = selectQueryBuilder.limit(1);
}
} else {
if (isArray) {
// For one to many relational field.
selectQueryBuilder = selectQueryBuilder.where('entity.id IN (:...ids)', { ids: entityData[field] })
.distinctOn(['entity.id'])
.orderBy('entity.id')
.limit(DEFAULT_LIMIT);
// Subquery example if distinctOn is not performant.
//
// SELECT c.*
// FROM
// categories c,
// (
// SELECT id, MAX(block_number) as block_number
// FROM categories
// WHERE
// id IN ('nature', 'tech', 'issues')
// AND
// block_number <= 127
// GROUP BY id
// ) a
// WHERE
// c.id = a.id AND c.block_number = a.block_number
} else {
// For one to one relational field.
selectQueryBuilder = selectQueryBuilder.where('entity.id = :id', { id: entityData[field] })
.limit(1);
}
selectQueryBuilder = selectQueryBuilder.addOrderBy('entity.block_number', 'DESC');
}
if (blockNumber) {
selectQueryBuilder = selectQueryBuilder.andWhere(
'entity.block_number <= :blockNumber',
{ blockNumber }
);
}
if (isArray) {
entityData[field] = await selectQueryBuilder.getMany();
} else {
entityData[field] = await selectQueryBuilder.getOne();
}
});
await Promise.all(relationQueryPromises);
[entityData] = await this.loadRelations(block, relationsMap, entity, [entityData], 1);
}
return entityData;
@ -201,6 +138,207 @@ export class Database {
}
}
async getEntities<Entity> (entity: new () => Entity, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight, where: Where = {}, queryOptions: QueryOptions = {}, depth = 1): Promise<Entity[]> {
const queryRunner = this._conn.createQueryRunner();
try {
const repo = queryRunner.manager.getRepository(entity);
const { tableName } = repo.metadata;
let subQuery = repo.createQueryBuilder('subTable')
.select('subTable.id', 'id')
.addSelect('MAX(subTable.block_number)', 'block_number')
.addFrom('block_progress', 'blockProgress')
.where('subTable.block_hash = blockProgress.block_hash')
.andWhere('blockProgress.is_pruned = :isPruned', { isPruned: false })
.groupBy('subTable.id');
if (block.hash) {
const { canonicalBlockNumber, blockHashes } = await this._baseDatabase.getFrothyRegion(queryRunner, block.hash);
subQuery = subQuery
.andWhere(new Brackets(qb => {
qb.where('subTable.block_hash IN (:...blockHashes)', { blockHashes })
.orWhere('subTable.block_number <= :canonicalBlockNumber', { canonicalBlockNumber });
}));
}
if (block.number) {
subQuery = subQuery.andWhere('subTable.block_number <= :blockNumber', { blockNumber: block.number });
}
let selectQueryBuilder = repo.createQueryBuilder(tableName)
.innerJoin(
`(${subQuery.getQuery()})`,
'latestEntities',
`${tableName}.id = "latestEntities"."id" AND ${tableName}.block_number = "latestEntities"."block_number"`
)
.setParameters(subQuery.getParameters());
selectQueryBuilder = this._baseDatabase.buildQuery(repo, selectQueryBuilder, where);
if (queryOptions.orderBy) {
selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, queryOptions);
}
selectQueryBuilder = this._baseDatabase.orderQuery(repo, selectQueryBuilder, { ...queryOptions, orderBy: 'id' });
if (queryOptions.skip) {
selectQueryBuilder = selectQueryBuilder.offset(queryOptions.skip);
}
if (queryOptions.limit) {
selectQueryBuilder = selectQueryBuilder.limit(queryOptions.limit);
}
const entities = await selectQueryBuilder.getMany();
if (!entities.length) {
return [];
}
return this.loadRelations(block, relationsMap, entity, entities, depth);
} finally {
await queryRunner.release();
}
}
async loadRelations<Entity> (block: BlockHeight, relationsMap: Map<any, { [key: string]: any }>, entity: new () => Entity, entities: Entity[], depth: number): Promise<Entity[]> {
// Only support two-level nesting of relations
if (depth > 2) {
return entities;
}
const relations = relationsMap.get(entity);
if (relations === undefined) {
return entities;
}
const relationPromises = Object.entries(relations).map(async ([field, data]) => {
const { entity: relationEntity, isArray, isDerived, field: foreignKey } = data;
if (isDerived) {
const where: Where = {
[foreignKey]: [{
value: entities.map((entity: any) => entity.id),
not: false,
operator: 'in'
}]
};
const relatedEntities = await this.getEntities(
relationEntity,
relationsMap,
block,
where,
{},
depth + 1
);
const relatedEntitiesMap = relatedEntities.reduce((acc: {[key:string]: any[]}, entity: any) => {
// Related entity might be loaded with data.
const parentEntityId = entity[foreignKey].id ?? entity[foreignKey];
if (!acc[parentEntityId]) {
acc[parentEntityId] = [];
}
if (acc[parentEntityId].length < DEFAULT_LIMIT) {
acc[parentEntityId].push(entity);
}
return acc;
}, {});
entities.forEach((entity: any) => {
if (relatedEntitiesMap[entity.id]) {
entity[field] = relatedEntitiesMap[entity.id];
} else {
entity[field] = [];
}
});
return;
}
if (isArray) {
const relatedIds = entities.reduce((acc: Set<string>, entity: any) => {
entity[field].forEach((relatedEntityId: string) => acc.add(relatedEntityId));
return acc;
}, new Set());
const where: Where = {
id: [{
value: Array.from(relatedIds),
not: false,
operator: 'in'
}]
};
const relatedEntities = await this.getEntities(
relationEntity,
relationsMap,
block,
where,
{},
depth + 1
);
entities.forEach((entity: any) => {
const relatedEntityIds: Set<string> = entity[field].reduce((acc: Set<string>, id: string) => {
acc.add(id);
return acc;
}, new Set());
entity[field] = [];
relatedEntities.forEach((relatedEntity: any) => {
if (relatedEntityIds.has(relatedEntity.id) && entity[field].length < DEFAULT_LIMIT) {
entity[field].push(relatedEntity);
}
});
});
return;
}
// field is neither an array nor derivedFrom
const where: Where = {
id: [{
value: entities.map((entity: any) => entity[field]),
not: false,
operator: 'in'
}]
};
const relatedEntities = await this.getEntities(
relationEntity,
relationsMap,
block,
where,
{},
depth + 1
);
const relatedEntitiesMap = relatedEntities.reduce((acc: {[key:string]: any}, entity: any) => {
acc[entity.id] = entity;
return acc;
}, {});
entities.forEach((entity: any) => {
if (relatedEntitiesMap[entity[field]]) {
entity[field] = relatedEntitiesMap[entity[field]];
}
});
});
await Promise.all(relationPromises);
return entities;
}
async saveEntity (entity: string, data: any): Promise<void> {
const repo = this._conn.getRepository(entity);

8
packages/graph-node/src/loader.ts
View File

@ -74,7 +74,9 @@ export const instantiate = async (
const entityId = __getString(id);
assert(context.block);
console.time(`time:loader#index.store.get-db-${entityName}`);
const entityData = await database.getEntity(entityName, entityId, context.block.blockHash);
console.timeEnd(`time:loader#index.store.get-db-${entityName}`);
if (!entityData) {
return null;
@ -95,7 +97,9 @@ export const instantiate = async (
assert(context.block);
let dbData = await database.fromGraphEntity(instanceExports, context.block, entityName, entityInstance);
console.time(`time:loader#index.store.set-db-${entityName}`);
await database.saveEntity(entityName, dbData);
console.timeEnd(`time:loader#index.store.set-db-${entityName}`);
// Resolve any field name conflicts in the dbData for auto-diff.
dbData = resolveEntityFieldConflicts(dbData);
@ -206,7 +210,9 @@ export const instantiate = async (
assert(context.block);
// TODO: Check for function overloading.
console.time(`time:loader#ethereum.call-${functionName}`);
let result = await contract[functionName](...functionParams, { blockTag: context.block.blockHash });
console.timeEnd(`time:loader#ethereum.call-${functionName}`);
// Using function signature does not work.
const { outputs } = contract.interface.getFunction(functionName);
@ -279,6 +285,7 @@ export const instantiate = async (
assert(storageLayout);
assert(context.block);
console.time(`time:loader#ethereum.storageValue-${variableString}`);
const result = await indexer.getStorageValue(
storageLayout,
context.block.blockHash,
@ -286,6 +293,7 @@ export const instantiate = async (
variableString,
...mappingKeyValues
);
console.timeEnd(`time:loader#ethereum.storageValue-${variableString}`);
const storageValueType = getStorageValueType(storageLayout, variableString, mappingKeyValues);

49
packages/graph-node/src/watcher.ts
View File

@ -11,11 +11,11 @@ import { ContractInterface, utils, providers } from 'ethers';
import { ResultObject } from '@vulcanize/assemblyscript/lib/loader';
import { EthClient } from '@vulcanize/ipld-eth-client';
import { IndexerInterface, getFullBlock, BlockHeight, ServerConfig, getFullTransaction } from '@vulcanize/util';
import { IndexerInterface, getFullBlock, BlockHeight, ServerConfig, getFullTransaction, QueryOptions } from '@vulcanize/util';
import { createBlock, createEvent, getSubgraphConfig, resolveEntityFieldConflicts, Transaction } from './utils';
import { Context, GraphData, instantiate } from './loader';
import { Database } from './database';
import { Database, DEFAULT_LIMIT } from './database';
const log = debug('vulcanize:graph-watcher');
@ -248,14 +248,55 @@ export class GraphWatcher {
this._indexer = indexer;
}
async getEntity<Entity> (entity: new () => Entity, id: string, relations: { [key: string]: any }, block?: BlockHeight): Promise<any> {
async getEntity<Entity> (entity: new () => Entity, id: string, relationsMap: Map<any, { [key: string]: any }>, block?: BlockHeight): Promise<any> {
// Get entity from the database.
const result = await this._database.getEntityWithRelations(entity, id, relations, block) as any;
const result = await this._database.getEntityWithRelations(entity, id, relationsMap, block);
// Resolve any field name conflicts in the entity result.
return resolveEntityFieldConflicts(result);
}
async getEntities<Entity> (entity: new () => Entity, relationsMap: Map<any, { [key: string]: any }>, block: BlockHeight, where: { [key: string]: any } = {}, queryOptions: QueryOptions): Promise<any> {
where = Object.entries(where).reduce((acc: { [key: string]: any }, [fieldWithSuffix, value]) => {
const [field, ...suffix] = fieldWithSuffix.split('_');
if (!acc[field]) {
acc[field] = [];
}
const filter = {
value,
not: false,
operator: 'equals'
};
let operator = suffix.shift();
if (operator === 'not') {
filter.not = true;
operator = suffix.shift();
}
if (operator) {
filter.operator = operator;
}
acc[field].push(filter);
return acc;
}, {});
if (!queryOptions.limit) {
queryOptions.limit = DEFAULT_LIMIT;
}
// Get entities from the database.
const entities = await this._database.getEntities(entity, relationsMap, block, where, queryOptions);
// Resolve any field name conflicts in the entity result.
return entities.map(entity => resolveEntityFieldConflicts(entity));
}
/**
* Method to reinstantiate WASM instance for specified dataSource.
* @param dataSourceName

13
packages/graph-test-watcher/package.json
View File

@ -6,9 +6,13 @@
"main": "dist/index.js",
"scripts": {
"lint": "eslint .",
"build": "tsc",
"server": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"build": "yarn clean && tsc && yarn copy-assets",
"clean": "rm -rf ./dist",
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
"server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
"server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
"job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
"fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
"reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@ -66,6 +70,7 @@
"eslint-plugin-promise": "^5.1.0",
"eslint-plugin-standard": "^5.0.0",
"ts-node": "^10.0.0",
"typescript": "^4.3.2"
"typescript": "^4.3.2",
"copyfiles": "^2.4.1"
}
}

3
packages/graph-test-watcher/src/entity/Author.ts
View File

@ -2,12 +2,13 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import Decimal from 'decimal.js';
import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Author {
@PrimaryColumn('varchar')
id!: string;

3
packages/graph-test-watcher/src/entity/Blog.ts
View File

@ -2,7 +2,7 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintArrayTransformer } from '@vulcanize/util';
@ -12,6 +12,7 @@ enum BlogType {
}
@Entity()
@Index(['blockNumber'])
export class Blog {
@PrimaryColumn('varchar')
id!: string;

3
packages/graph-test-watcher/src/entity/Category.ts
View File

@ -2,11 +2,12 @@
// Copyright 2021 Vulcanize, Inc.
//
import { Entity, PrimaryColumn, Column } from 'typeorm';
import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
@Entity()
@Index(['blockNumber'])
export class Category {
@PrimaryColumn('varchar')
id!: string;

8
packages/graph-test-watcher/src/indexer.ts
View File

@ -376,9 +376,7 @@ export class Indexer implements IPLDIndexerInterface {
}
async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block: BlockHeight): Promise<Entity | undefined> {
const relations = this._relationsMap.get(entity) || {};
const data = await this._graphWatcher.getEntity(entity, id, relations, block);
const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
return data;
}
@ -663,6 +661,7 @@ export class Indexer implements IPLDIndexerInterface {
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
@ -684,6 +683,7 @@ export class Indexer implements IPLDIndexerInterface {
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
@ -775,8 +775,10 @@ export class Indexer implements IPLDIndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {

2
packages/graph-test-watcher/tsconfig.json
View File

@ -12,7 +12,7 @@
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */

30
packages/ipld-eth-client/src/eth-client.ts
View File

@ -40,7 +40,9 @@ export class EthClient {
async getStorageAt ({ blockHash, contract, slot }: { blockHash: string, contract: string, slot: string }): Promise<{ value: string, proof: { data: string } }> {
slot = `0x${padKey(slot)}`;
console.time(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`);
const result = await this._getCachedOrFetch('getStorageAt', { blockHash, contract, slot });
console.timeEnd(`time:eth-client#getStorageAt-${JSON.stringify({ blockHash, contract, slot })}`);
const { getStorageAt: { value, cid, ipldBlock } } = result;
return {
@ -62,46 +64,64 @@ export class EthClient {
}
async getBlockWithTransactions ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<any> {
return this._graphqlClient.query(
console.time(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
const result = await this._graphqlClient.query(
ethQueries.getBlockWithTransactions,
{
blockNumber: blockNumber?.toString(),
blockHash
}
);
console.timeEnd(`time:eth-client#getBlockWithTransactions-${JSON.stringify({ blockNumber, blockHash })}`);
return result;
}
async getBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<any> {
return this._graphqlClient.query(
console.time(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
const result = await this._graphqlClient.query(
ethQueries.getBlocks,
{
blockNumber: blockNumber?.toString(),
blockHash
}
);
console.timeEnd(`time:eth-client#getBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
return result;
}
async getFullBlocks ({ blockNumber, blockHash }: { blockNumber?: number, blockHash?: string }): Promise<any> {
return this._graphqlClient.query(
console.time(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
const result = await this._graphqlClient.query(
ethQueries.getFullBlocks,
{
blockNumber: blockNumber?.toString(),
blockHash
}
);
console.timeEnd(`time:eth-client#getFullBlocks-${JSON.stringify({ blockNumber, blockHash })}`);
return result;
}
async getFullTransaction (txHash: string): Promise<any> {
return this._graphqlClient.query(
console.time(`time:eth-client#getFullTransaction-${txHash}`);
const result = this._graphqlClient.query(
ethQueries.getFullTransaction,
{
txHash
}
);
console.timeEnd(`time:eth-client#getFullTransaction-${txHash}`);
return result;
}
async getBlockByHash (blockHash?: string): Promise<any> {
console.time(`time:eth-client#getBlockByHash-${blockHash}`);
const result = await this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash });
console.timeEnd(`time:eth-client#getBlockByHash-${blockHash}`);
return {
block: {
@ -113,7 +133,9 @@ export class EthClient {
}
async getLogs (vars: Vars): Promise<any> {
console.time(`time:eth-client#getLogs-${JSON.stringify(vars)}`);
const result = await this._getCachedOrFetch('getLogs', vars);
console.timeEnd(`time:eth-client#getLogs-${JSON.stringify(vars)}`);
const {
getLogs
} = result;

2
packages/mobymask-watcher/environments/local.toml
View File

@ -22,6 +22,8 @@
[metrics]
host = "127.0.0.1"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"

13
packages/mobymask-watcher/package.json
View File

@ -6,9 +6,13 @@
"main": "dist/index.js",
"scripts": {
"lint": "eslint .",
"build": "tsc",
"server": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"build": "yarn clean && tsc && yarn copy-assets",
"clean": "rm -rf ./dist",
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
"server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
"server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
"job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
"fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
"reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@ -62,6 +66,7 @@
"eslint-plugin-promise": "^5.1.0",
"eslint-plugin-standard": "^5.0.0",
"ts-node": "^10.0.0",
"typescript": "^4.3.2"
"typescript": "^4.3.2",
"copyfiles": "^2.4.1"
}
}

4
packages/mobymask-watcher/src/indexer.ts
View File

@ -733,6 +733,7 @@ export class Indexer implements IPLDIndexerInterface {
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
@ -754,6 +755,7 @@ export class Indexer implements IPLDIndexerInterface {
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
@ -845,8 +847,10 @@ export class Indexer implements IPLDIndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {

29
packages/mobymask-watcher/src/resolvers.ts
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, StateKind } from '@vulcanize/util';
import { ValueResult, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -60,31 +60,48 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
Query: {
multiNonce: (_: any, { blockHash, contractAddress, key0, key1 }: { blockHash: string, contractAddress: string, key0: string, key1: bigint }): Promise<ValueResult> => {
log('multiNonce', blockHash, contractAddress, key0, key1);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('multiNonce').inc(1);
return indexer.multiNonce(blockHash, contractAddress, key0, key1);
},
_owner: (_: any, { blockHash, contractAddress }: { blockHash: string, contractAddress: string }): Promise<ValueResult> => {
log('_owner', blockHash, contractAddress);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('_owner').inc(1);
return indexer._owner(blockHash, contractAddress);
},
isRevoked: (_: any, { blockHash, contractAddress, key0 }: { blockHash: string, contractAddress: string, key0: string }): Promise<ValueResult> => {
log('isRevoked', blockHash, contractAddress, key0);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('isRevoked').inc(1);
return indexer.isRevoked(blockHash, contractAddress, key0);
},
isPhisher: (_: any, { blockHash, contractAddress, key0 }: { blockHash: string, contractAddress: string, key0: string }): Promise<ValueResult> => {
log('isPhisher', blockHash, contractAddress, key0);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('isPhisher').inc(1);
return indexer.isPhisher(blockHash, contractAddress, key0);
},
isMember: (_: any, { blockHash, contractAddress, key0 }: { blockHash: string, contractAddress: string, key0: string }): Promise<ValueResult> => {
log('isMember', blockHash, contractAddress, key0);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('isMember').inc(1);
return indexer.isMember(blockHash, contractAddress, key0);
},
events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => {
log('events', blockHash, contractAddress, name);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('events').inc(1);
const block = await indexer.getBlockProgress(blockHash);
if (!block || !block.isComplete) {
@ -97,6 +114,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => {
log('eventsInRange', fromBlockNumber, toBlockNumber);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('eventsInRange').inc(1);
const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);
return events.map(event => indexer.getResultEvent(event));
@ -104,6 +123,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getStateByCID: async (_: any, { cid }: { cid: string }) => {
log('getStateByCID', cid);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getStateByCID').inc(1);
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
@ -112,6 +133,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
@ -120,12 +143,16 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getSyncStatus: async () => {
log('getSyncStatus');
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getSyncStatus').inc(1);
return indexer.getSyncStatus();
},
latestBlock: async () => {
log('latestBlock');
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('latestBlock').inc(1);
return indexer.getLatestBlock();
}

4
packages/mobymask-watcher/src/server.ts
View File

@ -14,7 +14,7 @@ import debug from 'debug';
import 'graphql-import-node';
import { createServer } from 'http';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util';
import { createResolvers } from './resolvers';
import { Indexer } from './indexer';
@ -83,6 +83,8 @@ export const main = async (): Promise<any> => {
log(`Server is listening on host ${host} port ${port}`);
});
startGQLMetricsServer(config);
return { app, server };
};

2
packages/mobymask-watcher/tsconfig.json
View File

@ -12,7 +12,7 @@
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */

1
packages/util/index.ts
View File

@ -19,3 +19,4 @@ export * from './src/ipld-database';
export * from './src/ipfs';
export * from './src/index-block';
export * from './src/metrics';
export * from './src/gql-metrics';

8
packages/util/src/common.ts
View File

@ -67,9 +67,7 @@ export const processBlockByNumber = async (
});
if (!blocks.length) {
console.time('time:common#processBlockByNumber-ipld-eth-server');
blocks = await indexer.getBlocks({ blockNumber });
console.timeEnd('time:common#processBlockByNumber-ipld-eth-server');
}
if (blocks.length) {
@ -135,7 +133,7 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block
log(`Processing events batch from index ${events[0].index} to ${events[0].index + events.length - 1}`);
}
console.time('time:common#processBacthEvents-processing_events_batch');
console.time('time:common#processBatchEvents-processing_events_batch');
for (let event of events) {
// Process events in loop
@ -189,7 +187,7 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block
block = await indexer.updateBlockProgress(block, event.index);
}
console.timeEnd('time:common#processBacthEvents-processing_events_batch');
console.timeEnd('time:common#processBatchEvents-processing_events_batch');
}
if (indexer.processBlockAfterEvents) {
@ -199,5 +197,7 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block
}
block.isComplete = true;
console.time('time:common#processBatchEvents-updateBlockProgress');
await indexer.updateBlockProgress(block, block.lastProcessedEventIndex);
console.timeEnd('time:common#processBatchEvents-updateBlockProgress');
};

5
packages/util/src/config.ts
View File

@ -56,9 +56,14 @@ export interface UpstreamConfig {
}
}
export interface GQLMetricsConfig {
port: number;
}
export interface MetricsConfig {
host: string;
port: number;
gql: GQLMetricsConfig;
}
export interface Config {

124
packages/util/src/database.ts
View File

@ -4,7 +4,6 @@
import assert from 'assert';
import {
Brackets,
Connection,
ConnectionOptions,
createConnection,
@ -24,9 +23,6 @@ import { BlockProgressInterface, ContractInterface, EventInterface, SyncStatusIn
import { MAX_REORG_DEPTH, UNKNOWN_EVENT_NAME } from './constants';
import { blockProgressCount, eventCount } from './metrics';
const DEFAULT_LIMIT = 100;
const DEFAULT_SKIP = 0;
const OPERATOR_MAP = {
equals: '=',
gt: '>',
@ -207,13 +203,21 @@ export class Database {
.innerJoinAndSelect('event.block', 'block')
.where('block.block_hash = :blockHash AND block.is_pruned = false', { blockHash });
queryBuilder = this._buildQuery(repo, queryBuilder, where, queryOptions);
queryBuilder = this.buildQuery(repo, queryBuilder, where);
if (queryOptions.orderBy) {
queryBuilder = this.orderQuery(repo, queryBuilder, queryOptions);
}
queryBuilder.addOrderBy('event.id', 'ASC');
const { limit = DEFAULT_LIMIT, skip = DEFAULT_SKIP } = queryOptions;
if (queryOptions.skip) {
queryBuilder = queryBuilder.offset(queryOptions.skip);
}
queryBuilder = queryBuilder.offset(skip)
.limit(limit);
if (queryOptions.limit) {
queryBuilder = queryBuilder.limit(queryOptions.limit);
}
return queryBuilder.getMany();
}
@ -388,56 +392,6 @@ export class Database {
return event;
}
async getModelEntities<Entity> (queryRunner: QueryRunner, entity: new () => Entity, block: BlockHeight, where: Where = {}, queryOptions: QueryOptions = {}, relations: Relation[] = []): Promise<Entity[]> {
const repo = queryRunner.manager.getRepository(entity);
const { tableName } = repo.metadata;
let subQuery = repo.createQueryBuilder('subTable')
.select('MAX(subTable.block_number)')
.where(`subTable.id = ${tableName}.id`);
if (block.hash) {
const { canonicalBlockNumber, blockHashes } = await this.getFrothyRegion(queryRunner, block.hash);
subQuery = subQuery
.andWhere(new Brackets(qb => {
qb.where('subTable.block_hash IN (:...blockHashes)', { blockHashes })
.orWhere('subTable.block_number <= :canonicalBlockNumber', { canonicalBlockNumber });
}));
}
if (block.number) {
subQuery = subQuery.andWhere('subTable.block_number <= :blockNumber', { blockNumber: block.number });
}
let selectQueryBuilder = repo.createQueryBuilder(tableName)
.where(`${tableName}.block_number IN (${subQuery.getQuery()})`)
.setParameters(subQuery.getParameters());
relations.forEach(relation => {
let alias, property;
if (typeof relation === 'string') {
[, alias] = relation.split('.');
property = relation;
} else {
alias = relation.alias;
property = relation.property;
}
selectQueryBuilder = selectQueryBuilder.leftJoinAndSelect(property, alias);
});
selectQueryBuilder = this._buildQuery(repo, selectQueryBuilder, where, queryOptions);
const { limit = DEFAULT_LIMIT, skip = DEFAULT_SKIP } = queryOptions;
selectQueryBuilder = selectQueryBuilder.skip(skip)
.take(limit);
return selectQueryBuilder.getMany();
}
async getFrothyEntity<Entity> (queryRunner: QueryRunner, repo: Repository<Entity>, data: { blockHash: string, id: string }): Promise<{ blockHash: string, blockNumber: number, id: string }> {
// Hierarchical query for getting the entity in the frothy region.
const heirerchicalQuery = `
@ -596,16 +550,21 @@ export class Database {
eventCount.set(this._eventCount);
}
_buildQuery<Entity> (repo: Repository<Entity>, selectQueryBuilder: SelectQueryBuilder<Entity>, where: Where = {}, queryOptions: QueryOptions = {}): SelectQueryBuilder<Entity> {
const { tableName } = repo.metadata;
buildQuery<Entity> (repo: Repository<Entity>, selectQueryBuilder: SelectQueryBuilder<Entity>, where: Where = {}): SelectQueryBuilder<Entity> {
Object.entries(where).forEach(([field, filters]) => {
filters.forEach((filter, index) => {
// Form the where clause.
let { not, operator, value } = filter;
const columnMetadata = repo.metadata.findColumnWithPropertyName(field);
assert(columnMetadata);
let whereClause = `${tableName}.${columnMetadata.propertyAliasName} `;
let whereClause = `"${selectQueryBuilder.alias}"."${columnMetadata.databaseName}" `;
if (columnMetadata.relationMetadata) {
// For relation fields, use the id column.
const idColumn = columnMetadata.relationMetadata.joinColumns.find(column => column.referencedColumn?.propertyName === 'id');
assert(idColumn);
whereClause = `"${selectQueryBuilder.alias}"."${idColumn.databaseName}" `;
}
if (not) {
if (operator === 'equals') {
@ -617,9 +576,7 @@ export class Database {
whereClause += `${OPERATOR_MAP[operator]} `;
if (['contains', 'starts'].some(el => el === operator)) {
whereClause += '%:';
} else if (operator === 'in') {
if (operator === 'in') {
whereClause += '(:...';
} else {
// Convert to string type value as bigint type throws error in query.
@ -631,9 +588,7 @@ export class Database {
const variableName = `${field}${index}`;
whereClause += variableName;
if (['contains', 'ends'].some(el => el === operator)) {
whereClause += '%';
} else if (operator === 'in') {
if (operator === 'in') {
whereClause += ')';
if (!value.length) {
@ -641,18 +596,35 @@ export class Database {
}
}
if (['contains', 'starts'].some(el => el === operator)) {
value = `%${value}`;
}
if (['contains', 'ends'].some(el => el === operator)) {
value += '%';
}
selectQueryBuilder = selectQueryBuilder.andWhere(whereClause, { [variableName]: value });
});
});
const { orderBy, orderDirection } = queryOptions;
if (orderBy) {
const columnMetadata = repo.metadata.findColumnWithPropertyName(orderBy);
assert(columnMetadata);
selectQueryBuilder = selectQueryBuilder.orderBy(`${tableName}.${columnMetadata.propertyAliasName}`, orderDirection === 'desc' ? 'DESC' : 'ASC');
}
return selectQueryBuilder;
}
orderQuery<Entity> (
repo: Repository<Entity>,
selectQueryBuilder: SelectQueryBuilder<Entity>,
orderOptions: { orderBy?: string, orderDirection?: string }
): SelectQueryBuilder<Entity> {
const { orderBy, orderDirection } = orderOptions;
assert(orderBy);
const columnMetadata = repo.metadata.findColumnWithPropertyName(orderBy);
assert(columnMetadata);
return selectQueryBuilder.addOrderBy(
`${selectQueryBuilder.alias}.${columnMetadata.propertyAliasName}`,
orderDirection === 'desc' ? 'DESC' : 'ASC'
);
}
}

51
packages/util/src/gql-metrics.ts Normal file
View File

@ -0,0 +1,51 @@
//
// Copyright 2022 Vulcanize, Inc.
//
import * as client from 'prom-client';
import express, { Application } from 'express';
import debug from 'debug';
import assert from 'assert';
import { Config } from './config';
const log = debug('vulcanize:gql-metrics');
const gqlRegistry = new client.Registry();
// Create custom metrics
export const gqlTotalQueryCount = new client.Counter({
name: 'gql_query_count_total',
help: 'Total GQL queries made',
registers: [gqlRegistry]
});
export const gqlQueryCount = new client.Counter({
name: 'gql_query_count',
help: 'GQL queries made',
labelNames: ['name'] as const,
registers: [gqlRegistry]
});
// Export metrics on a server
const app: Application = express();
export const startGQLMetricsServer = async (config: Config): Promise<void> => {
if (!config.metrics || !config.metrics.gql) {
log('GQL metrics disabled. To enable add GQL metrics host and port.');
return;
}
assert(config.metrics.host, 'Missing config for metrics host');
assert(config.metrics.gql.port, 'Missing config for gql metrics port');
app.get('/metrics', async (req, res) => {
res.setHeader('Content-Type', gqlRegistry.contentType);
const metrics = await gqlRegistry.metrics();
res.send(metrics);
});
app.listen(config.metrics.gql.port, config.metrics.host, () => {
log(`GQL Metrics exposed at http://${config.metrics.host}:${config.metrics.gql.port}/metrics`);
});
};

9
packages/util/src/job-runner.ts
View File

@ -79,6 +79,7 @@ export class JobRunner {
}
async _pruneChain (job: any, syncStatus: SyncStatusInterface): Promise<void> {
console.time('time:job-runner#_pruneChain');
const { pruneBlockHeight } = job.data;
log(`Processing chain pruning at ${pruneBlockHeight}`);
@ -119,6 +120,8 @@ export class JobRunner {
// Update the canonical block in the SyncStatus.
await this._indexer.updateSyncStatusCanonicalBlock(newCanonicalBlockHash, pruneBlockHeight);
}
console.timeEnd('time:job-runner#_pruneChain');
}
async _indexBlock (job: any, syncStatus: SyncStatusInterface): Promise<void> {
@ -145,6 +148,7 @@ export class JobRunner {
throw new Error(message);
}
console.time('time:job-runner#_indexBlock-get-block-progress-entities');
let [parentBlock, blockProgress] = await this._indexer.getBlockProgressEntities(
{
blockHash: In([parentHash, blockHash])
@ -155,6 +159,7 @@ export class JobRunner {
}
}
);
console.timeEnd('time:job-runner#_indexBlock-get-block-progress-entities');
// Check if parent block has been processed yet, if not, push a high priority job to process that first and abort.
// However, don't go beyond the `latestCanonicalBlockHash` from SyncStatus as we have to assume the reorg can't be that deep.
@ -209,7 +214,9 @@ export class JobRunner {
throw new Error(message);
} else {
// Remove the unknown events of the parent block if it is marked complete.
console.time('time:job-runner#_indexBlock-remove-unknown-events');
await this._indexer.removeUnknownEvents(parentBlock);
console.timeEnd('time:job-runner#_indexBlock-remove-unknown-events');
}
} else {
blockProgress = parentBlock;
@ -220,7 +227,9 @@ export class JobRunner {
// Delay required to process block.
await wait(jobDelayInMilliSecs);
console.time('time:job-runner#_indexBlock-fetch-block-events');
blockProgress = await this._indexer.fetchBlockEvents({ cid, blockHash, blockNumber, parentHash, blockTimestamp: timestamp });
console.timeEnd('time:job-runner#_indexBlock-fetch-block-events');
}
if (this._indexer.processBlock) {

2
packages/util/src/misc.ts
View File

@ -191,7 +191,9 @@ export const getFullBlock = async (ethClient: EthClient, ethProvider: providers.
// TODO: Calculate size from rlp encoded data.
// Get block info from JSON RPC API provided by ipld-eth-server.
const provider = ethProvider as providers.JsonRpcProvider;
console.time('time:misc#getFullBlock-eth_getBlockByHash');
const { size } = await provider.send('eth_getBlockByHash', [blockHash, false]);
console.timeEnd('time:misc#getFullBlock-eth_getBlockByHash');
return {
headerId: fullBlock.id,

49
yarn.lock
View File

@ -5147,6 +5147,19 @@ copy-descriptor@^0.1.0:
resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d"
integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=
copyfiles@^2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/copyfiles/-/copyfiles-2.4.1.tgz#d2dcff60aaad1015f09d0b66e7f0f1c5cd3c5da5"
integrity sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg==
dependencies:
glob "^7.0.5"
minimatch "^3.0.3"
mkdirp "^1.0.4"
noms "0.0.0"
through2 "^2.0.1"
untildify "^4.0.0"
yargs "^16.1.0"
core-js-pure@^3.0.1:
version "3.13.0"
resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.13.0.tgz#9d267fb47d1d7046cfbc05e7b67bb235b6735355"
@ -7463,6 +7476,18 @@ glob@^7.0.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, gl
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.0.5:
version "7.2.3"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.1.1"
once "^1.3.0"
path-is-absolute "^1.0.0"
global-dirs@^2.0.1:
version "2.1.0"
resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-2.1.0.tgz#e9046a49c806ff04d6c1825e196c8f0091e8df4d"
@ -10003,6 +10028,13 @@ minimatch@5.0.1:
dependencies:
brace-expansion "^2.0.1"
minimatch@^3.0.3, minimatch@^3.1.1:
version "3.1.2"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
dependencies:
brace-expansion "^1.1.7"
minimist-options@4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"
@ -10514,6 +10546,14 @@ nodemon@^2.0.7:
undefsafe "^2.0.3"
update-notifier "^4.1.0"
noms@0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/noms/-/noms-0.0.0.tgz#da8ebd9f3af9d6760919b27d9cdc8092a7332859"
integrity sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==
dependencies:
inherits "^2.0.1"
readable-stream "~1.0.31"
nopt@^4.0.1:
version "4.0.3"
resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
@ -12069,7 +12109,7 @@ readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.0.6, readable
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
readable-stream@~1.0.15:
readable-stream@~1.0.15, readable-stream@~1.0.31:
version "1.0.34"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c"
integrity sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=
@ -13443,7 +13483,7 @@ thenify-all@^1.0.0:
dependencies:
any-promise "^1.0.0"
through2@^2.0.0, through2@^2.0.3:
through2@^2.0.0, through2@^2.0.1, through2@^2.0.3:
version "2.0.5"
resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
@ -14008,6 +14048,11 @@ unset-value@^1.0.0:
has-value "^0.3.1"
isobject "^3.0.0"
untildify@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==
upath@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/upath/-/upath-2.0.1.tgz#50c73dea68d6f6b990f51d279ce6081665d61a8b"