cerc-io/watcher-ts
11
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2026-09-08 00:44:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement query for multiple entities and nested relation fields in eden-watcher (#166)

Browse Source 
* Implement query for multiple entities in eden-watcher

* Implement nested relation queries

* Implement GQL query params first, skip, orderBy, orderDirection

* Add blockNumber index to subgraph entities

* Add logs for timing eth-calls and storage calls

* Add prometheus metrics to monitor GQL queries

* Fix default limit and order of 1-N related field in GQL entitiy query

* Add timer logs for block processing

* Run transpiled js in all watchers

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
This commit is contained in:
nabarun
2022-09-01 14:17:43 +05:30
committed by GitHub
co-authored by prathamesh
parent 97e88ab5f0
commit 8af7417df6
60 changed files with 777 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/codegen/src/entity.ts
+5 -1
View File
@@ -209,11 +209,15 @@ export class Entity {
entityObject.imports.push(
{
toImport: new Set(['Entity', 'PrimaryColumn', 'Column']),
toImport: new Set(['Entity', 'PrimaryColumn', 'Column', 'Index']),
from: 'typeorm'
}
);
entityObject.indexOn.push({
columns: ['blockNumber']
});
// Add common columns.
entityObject.columns.push({
name: 'id',
packages/codegen/src/templates/config-template.handlebars
+2
View File
@@ -27,6 +27,8 @@
[metrics]
host = "127.0.0.1"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
packages/codegen/src/templates/indexer-template.handlebars
+5 -1
View File
@@ -430,7 +430,7 @@ export class Indexer implements IPLDIndexerInterface {
async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
const relations = this._relationsMap.get(entity) || {};
const data = await this._graphWatcher.getEntity(entity, id, relations, block);
const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
return data;
}
@@ -704,6 +704,7 @@ export class Indexer implements IPLDIndexerInterface {
const blockPromise = this._ethClient.getBlockByHash(blockHash);
let logs: any[];
console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
if (this._serverConfig.filterLogs) {
const watchedContracts = this._baseIndexer.getWatchedContracts();
@@ -725,6 +726,7 @@ export class Indexer implements IPLDIndexerInterface {
} else {
({ logs } = await this._ethClient.getLogs({ blockHash }));
}
console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
let [
{ block },
@@ -816,8 +818,10 @@ export class Indexer implements IPLDIndexerInterface {
parentHash: block.parent.hash
};
console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
await dbTx.commitTransaction();
console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
return blockProgress;
} catch (error) {
packages/codegen/src/templates/package-template.handlebars
+9 -4
View File
@@ -6,9 +6,13 @@
"main": "dist/index.js",
"scripts": {
"lint": "eslint .",
"build": "tsc",
"server": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"build": "yarn clean && tsc && yarn copy-assets",
"clean": "rm -rf ./dist",
"copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
"server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
"server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
"job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
"job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
"watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
"fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
"reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@@ -65,6 +69,7 @@
"eslint-plugin-promise": "^5.1.0",
"eslint-plugin-standard": "^5.0.0",
"ts-node": "^10.0.0",
"typescript": "^4.3.2"
"typescript": "^4.3.2",
"copyfiles": "^2.4.1"
}
}
packages/codegen/src/templates/resolvers-template.handlebars
+16 -1
View File
@@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, BlockHeight, StateKind } from '@vulcanize/util';
import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@@ -68,6 +68,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
{{~#each this.params}}, {{this.name}}: {{this.type~}} {{/each}} }): Promise<ValueResult> => {
log('{{this.name}}', blockHash, contractAddress
{{~#each this.params}}, {{this.name~}} {{/each}});
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('{{this.name}}').inc(1);
return indexer.{{this.name}}(blockHash, contractAddress
{{~#each this.params}}, {{this.name~}} {{/each}});
},
@@ -77,6 +80,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
{{~#each subgraphQueries}}
{{this.queryName}}: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
log('{{this.queryName}}', id, block);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('{{this.queryName}}').inc(1);
return indexer.getSubgraphEntity({{this.entityName}}, id, block);
},
@@ -84,6 +89,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
{{/each}}
events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => {
log('events', blockHash, contractAddress, name);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('events').inc(1);
const block = await indexer.getBlockProgress(blockHash);
if (!block || !block.isComplete) {
@@ -96,6 +103,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => {
log('eventsInRange', fromBlockNumber, toBlockNumber);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('eventsInRange').inc(1);
const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
if (expected !== actual) {
@@ -108,6 +117,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getStateByCID: async (_: any, { cid }: { cid: string }) => {
log('getStateByCID', cid);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getStateByCID').inc(1);
const ipldBlock = await indexer.getIPLDBlockByCid(cid);
@@ -116,6 +127,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
@@ -124,6 +137,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
getSyncStatus: async () => {
log('getSyncStatus');
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getSyncStatus').inc(1);
return indexer.getSyncStatus();
}
packages/codegen/src/templates/server-template.handlebars
+5 -3
View File
@@ -14,7 +14,7 @@ import debug from 'debug';
import 'graphql-import-node';
import { createServer } from 'http';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}
@@ -45,10 +45,10 @@ export const main = async (): Promise<any> => {
const db = new Database(config.database);
await db.init();
{{#if (subgraphPath)}}
const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();
const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}
@@ -98,6 +98,8 @@ export const main = async (): Promise<any> => {
log(`Server is listening on host ${host} port ${port}`);
});
startGQLMetricsServer(config);
return { app, server };
};
packages/codegen/src/templates/tsconfig-template.handlebars
+2 -2
View File
@@ -12,9 +12,9 @@
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
// "declaration": true, /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
"sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "dist", /* Redirect output structure to the directory. */
"outDir": "dist", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */