diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index a9d7492b..4a957287 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -194,7 +194,7 @@ export class Schema { // Get type composer object for return type from the schema composer. type: this._composer.getAnyTC(subgraphType).NonNull, args: { - id: 'String!', + id: 'ID!', block: 'Block_height' } }; diff --git a/packages/codegen/src/templates/reset-state-template.handlebars b/packages/codegen/src/templates/reset-state-template.handlebars index 1705cf85..a8437612 100644 --- a/packages/codegen/src/templates/reset-state-template.handlebars +++ b/packages/codegen/src/templates/reset-state-template.handlebars @@ -92,7 +92,7 @@ export const handler = async (argv: any): Promise => { const ipldStatus = await indexer.getIPLDStatus(); - if(ipldStatus) { + if (ipldStatus) { if (ipldStatus.latestHooksBlockNumber > blockProgress.blockNumber) { await indexer.updateIPLDStatusHooksBlock(blockProgress.blockNumber, true); } diff --git a/packages/graph-node/assembly/tsconfig.json b/packages/graph-node/assembly/tsconfig.json index 062482b9..9c0f2d02 100644 --- a/packages/graph-node/assembly/tsconfig.json +++ b/packages/graph-node/assembly/tsconfig.json @@ -1,5 +1,8 @@ { - "extends": "@vulcanize/assemblyscript/std/assembly.json", + "extends": "@graphprotocol/graph-ts/tsconfig", + "compilerOptions": { + "types": ["@graphprotocol/graph-ts"] + }, "include": [ "./**/*.ts" ] diff --git a/packages/graph-node/bin/compare-blocks b/packages/graph-node/bin/compare-blocks new file mode 100755 index 00000000..7bfa4ab5 --- /dev/null +++ b/packages/graph-node/bin/compare-blocks @@ -0,0 +1,9 @@ +#! /usr/bin/env node + +const { main } = require('../dist/cli/compare/compare-blocks') + +main().catch(err => { + console.log(err); +}).finally(() => { + process.exit(0); +}); diff --git a/packages/graph-node/environments/compare-cli-config.toml b/packages/graph-node/environments/compare-cli-config.toml index 29098e42..78dd811f 100644 --- a/packages/graph-node/environments/compare-cli-config.toml +++ b/packages/graph-node/environments/compare-cli-config.toml @@ -4,3 +4,12 @@ [queries] queryDir = "../graph-test-watcher/src/gql/queries" + names = [] + +[cache] + endpoint = "gqlEndpoint1" + + [cache.config] + name = "subgraph-requests" + enabled = true + deleteOnStart = false diff --git a/packages/graph-node/src/cli/compare/client.ts b/packages/graph-node/src/cli/compare/client.ts index 9d0d6568..4d112187 100644 --- a/packages/graph-node/src/cli/compare/client.ts +++ b/packages/graph-node/src/cli/compare/client.ts @@ -7,32 +7,58 @@ import fs from 'fs'; import path from 'path'; import { gql } from '@apollo/client/core'; -import { GraphQLClient, GraphQLConfig } from '@vulcanize/ipld-eth-client'; +import { GraphQLClient, Config } from '@vulcanize/ipld-eth-client'; +import { Cache } from '@vulcanize/cache'; export class Client { - _config: GraphQLConfig; + _config: Config; _graphqlClient: GraphQLClient; _queryDir: string; + _cache: Cache | undefined; - constructor (config: GraphQLConfig, queryDir: string) { + constructor (config: Config, queryDir: string) { this._config = config; this._queryDir = path.resolve(process.cwd(), queryDir); - const { gqlEndpoint } = config; + const { gqlEndpoint, cache } = config; assert(gqlEndpoint, 'Missing gql endpoint'); this._graphqlClient = new GraphQLClient(config); + + this._cache = cache; } - async getEntity ({ blockHash, queryName, id }: { blockHash: string, queryName: string, id: string }): Promise { + async getResult (queryName: string, params: { [key: string]: any }): Promise { + return this._getCachedOrFetch(queryName, params); + } + + async _getCachedOrFetch (queryName: string, params: {[key: string]: any}): Promise { + const keyObj = { + queryName, + params + }; + + // Check if request cached in db, if cache is enabled. + if (this._cache) { + const [value, found] = await this._cache.get(keyObj) || [undefined, false]; + if (found) { + return value; + } + } + const entityQuery = fs.readFileSync(path.resolve(this._queryDir, `${queryName}.gql`), 'utf8'); - return this._graphqlClient.query( + // Result not cached or cache disabled, need to perform an upstream GQL query. + const result = await this._graphqlClient.query( gql(entityQuery), - { - id, - blockHash - } + params ); + + // Cache the result and return it, if cache is enabled. + if (this._cache) { + await this._cache.put(keyObj, result); + } + + return result; } } diff --git a/packages/graph-node/src/cli/compare/compare-blocks.ts b/packages/graph-node/src/cli/compare/compare-blocks.ts new file mode 100644 index 00000000..d205328b --- /dev/null +++ b/packages/graph-node/src/cli/compare/compare-blocks.ts @@ -0,0 +1,80 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import yargs from 'yargs'; +import 'reflect-metadata'; +import debug from 'debug'; + +import { compareQuery, Config, getClients, getConfig } from './utils'; + +const log = debug('vulcanize:compare-blocks'); + +export const main = async (): Promise => { + const argv = await yargs.parserConfiguration({ + 'parse-numbers': false + }).options({ + configFile: { + alias: 'cf', + type: 'string', + demandOption: true, + describe: 'Configuration file path (toml)' + }, + queryDir: { + alias: 'qf', + type: 'string', + describe: 'Path to queries directory' + }, + startBlock: { + type: 'number', + demandOption: true, + describe: 'Start block number' + }, + endBlock: { + type: 'number', + demandOption: true, + describe: 'End block number' + }, + rawJson: { + alias: 'j', + type: 'boolean', + describe: 'Whether to print out raw diff object', + default: false + } + }).argv; + + const config: Config = await getConfig(argv.configFile); + + const { startBlock, endBlock, rawJson, queryDir } = argv; + const queryNames = config.queries.names; + let diffFound = false; + + const clients = await getClients(config, queryDir); + + for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) { + const block = { number: blockNumber }; + console.time(`time:compare-block-${blockNumber}`); + + for (const queryName of queryNames) { + try { + log(`At block ${blockNumber} for query ${queryName}:`); + const resultDiff = await compareQuery(clients, queryName, { block }, rawJson); + + if (resultDiff) { + diffFound = true; + log('Results mismatch:', resultDiff); + } else { + log('Results match.'); + } + } catch (err: any) { + log('Error:', err.message); + } + } + + console.timeEnd(`time:compare-block-${blockNumber}`); + } + + if (diffFound) { + process.exit(1); + } +}; diff --git a/packages/graph-node/src/cli/compare/compare-entity.ts b/packages/graph-node/src/cli/compare/compare-entity.ts index e1ee016f..8af46023 100644 --- a/packages/graph-node/src/cli/compare/compare-entity.ts +++ b/packages/graph-node/src/cli/compare/compare-entity.ts @@ -4,28 +4,8 @@ import yargs from 'yargs'; import 'reflect-metadata'; -import path from 'path'; -import toml from 'toml'; -import fs from 'fs-extra'; -import assert from 'assert'; -import util from 'util'; -import { diffString, diff } from 'json-diff'; -import { Client } from './client'; - -interface EndpointConfig { - gqlEndpoint1: string; - gqlEndpoint2: string; -} - -interface QueryConfig { - queryDir: string; -} - -interface Config { - endpoints: EndpointConfig; - queries: QueryConfig; -} +import { compareQuery, Config, getClients, getConfig } from './utils'; export const main = async (): Promise => { const argv = await yargs.parserConfiguration({ @@ -45,8 +25,11 @@ export const main = async (): Promise => { blockHash: { alias: 'b', type: 'string', - demandOption: true, - describe: 'Blockhash' + describe: 'Block hash' + }, + blockNumber: { + type: 'number', + describe: 'Block number' }, queryName: { alias: 'q', @@ -57,7 +40,6 @@ export const main = async (): Promise => { entityId: { alias: 'i', type: 'string', - demandOption: true, describe: 'Id of the entity to be queried' }, rawJson: { @@ -70,75 +52,21 @@ export const main = async (): Promise => { const config: Config = await getConfig(argv.configFile); - const { client1, client2 } = await getClients(config, argv.queryDir); - const queryName = argv.queryName; const id = argv.entityId; const blockHash = argv.blockHash; - const result1 = await client1.getEntity({ blockHash, queryName, id }); - const result2 = await client2.getEntity({ blockHash, queryName, id }); + const block = { + number: argv.blockNumber, + hash: blockHash + }; - // Getting the diff of two result objects. - let resultDiff; - if (argv.rawJson) { - resultDiff = diff(result1, result2); + const clients = await getClients(config, argv.queryDir); - if (resultDiff) { - // Use util.inspect to extend depth limit in the output. - resultDiff = util.inspect(diff(result1, result2), false, null); - } - } else { - resultDiff = diffString(result1, result2); - } + const resultDiff = await compareQuery(clients, queryName, { id, block }, argv.rawJson); if (resultDiff) { console.log(resultDiff); process.exit(1); } }; - -async function getConfig (configFile: string): Promise { - const configFilePath = path.resolve(configFile); - const fileExists = await fs.pathExists(configFilePath); - if (!fileExists) { - throw new Error(`Config file not found: ${configFilePath}`); - } - - const config = toml.parse(await fs.readFile(configFilePath, 'utf8')); - - return config; -} - -async function getClients (config: Config, queryDir?: string): Promise<{ - client1: Client, - client2: Client -}> { - assert(config.endpoints, 'Missing endpoints config'); - - const gqlEndpoint1 = config.endpoints.gqlEndpoint1; - const gqlEndpoint2 = config.endpoints.gqlEndpoint2; - - assert(gqlEndpoint1, 'Missing endpoint one'); - assert(gqlEndpoint2, 'Missing endpoint two'); - - if (!queryDir) { - assert(config.queries, 'Missing queries config'); - queryDir = config.queries.queryDir; - } - - assert(queryDir, 'Query directory not provided'); - - const client1 = new Client({ - gqlEndpoint: gqlEndpoint1 - }, queryDir); - - const client2 = new Client({ - gqlEndpoint: gqlEndpoint2 - }, queryDir); - - return { - client1, - client2 - }; -} diff --git a/packages/graph-node/src/cli/compare/utils.ts b/packages/graph-node/src/cli/compare/utils.ts new file mode 100644 index 00000000..8754c531 --- /dev/null +++ b/packages/graph-node/src/cli/compare/utils.ts @@ -0,0 +1,122 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +import assert from 'assert'; +import util from 'util'; +import path from 'path'; +import toml from 'toml'; +import fs from 'fs-extra'; +import { diffString, diff } from 'json-diff'; + +import { Config as CacheConfig, getCache } from '@vulcanize/cache'; + +import { Client } from './client'; + +interface EndpointConfig { + gqlEndpoint1: string; + gqlEndpoint2: string; +} + +interface QueryConfig { + queryDir: string; + names: string[]; +} + +export interface Config { + endpoints: EndpointConfig; + queries: QueryConfig; + cache: { + endpoint: string; + config: CacheConfig; + } +} + +export const getConfig = async (configFile: string): Promise => { + const configFilePath = path.resolve(configFile); + const fileExists = await fs.pathExists(configFilePath); + + if (!fileExists) { + throw new Error(`Config file not found: ${configFilePath}`); + } + + const config = toml.parse(await fs.readFile(configFilePath, 'utf8')); + + if (config.queries.queryDir) { + // Resolve path from config file path. + const configFileDir = path.dirname(configFilePath); + config.queries.queryDir = path.resolve(configFileDir, config.queries.queryDir); + } + + return config; +}; + +export const compareQuery = async ( + clients: { + client1: Client, + client2: Client + }, + queryName: string, + params: { [key: string]: any }, + rawJson: boolean +): Promise => { + const { client1, client2 } = clients; + + const result2 = await client2.getResult(queryName, params); + const result1 = await client1.getResult(queryName, params); + + // Getting the diff of two result objects. + let resultDiff; + + if (rawJson) { + resultDiff = diff(result1, result2); + + if (resultDiff) { + // Use util.inspect to extend depth limit in the output. + resultDiff = util.inspect(diff(result1, result2), false, null); + } + } else { + resultDiff = diffString(result1, result2); + } + + return resultDiff; +}; + +export const getClients = async (config: Config, queryDir?: string):Promise<{ + client1: Client, + client2: Client +}> => { + assert(config.endpoints, 'Missing endpoints config'); + + const { + endpoints: { gqlEndpoint1, gqlEndpoint2 }, + cache: { endpoint, config: cacheConfig } + } = config; + + assert(gqlEndpoint1, 'Missing endpoint one'); + assert(gqlEndpoint2, 'Missing endpoint two'); + + if (!queryDir) { + assert(config.queries, 'Missing queries config'); + queryDir = config.queries.queryDir; + } + + assert(queryDir, 'Query directory not provided'); + assert(cacheConfig, 'Cache config not provided'); + const cache = await getCache(cacheConfig); + + const client1 = new Client({ + gqlEndpoint: gqlEndpoint1, + cache: endpoint === 'gqlEndpoint1' ? cache : undefined + }, queryDir); + + const client2 = new Client({ + gqlEndpoint: gqlEndpoint2, + cache: endpoint === 'gqlEndpoint2' ? cache : undefined + }, queryDir); + + return { + client1, + client2 + }; +}; diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts index 7a7cb5d3..00956410 100644 --- a/packages/ipld-eth-client/src/eth-client.ts +++ b/packages/ipld-eth-client/src/eth-client.ts @@ -11,7 +11,7 @@ import ethQueries from './eth-queries'; import { padKey } from './utils'; import { GraphQLClient, GraphQLConfig } from './graphql-client'; -interface Config extends GraphQLConfig { +export interface Config extends GraphQLConfig { cache: Cache | undefined; }