diff --git a/packages/eden-watcher/src/indexer.ts b/packages/eden-watcher/src/indexer.ts index b6c0d2e1..2ae50c18 100644 --- a/packages/eden-watcher/src/indexer.ts +++ b/packages/eden-watcher/src/indexer.ts @@ -585,6 +585,10 @@ export class Indexer implements IPLDIndexerInterface { return this._entityTypesMap; } + getRelationsMap (): Map { + return this._relationsMap; + } + _populateEntityTypesMap (): void { this._entityTypesMap.set( 'Producer', diff --git a/packages/eden-watcher/test/queries/account.gql b/packages/eden-watcher/test/queries/account.gql new file mode 100644 index 00000000..697f66db --- /dev/null +++ b/packages/eden-watcher/test/queries/account.gql @@ -0,0 +1,13 @@ +query account($id: String!, $block: Block_height){ + account(id: $id, block: $block){ + id + totalClaimed + totalSlashed + claims{ + id + } + slashes{ + id + } + } +} diff --git a/packages/eden-watcher/test/queries/block.gql b/packages/eden-watcher/test/queries/block.gql new file mode 100644 index 00000000..ab1c9ae4 --- /dev/null +++ b/packages/eden-watcher/test/queries/block.gql @@ -0,0 +1,20 @@ +query block($id: String!, $block: Block_height){ + block(id: $id, block: $block){ + id + fromActiveProducer + hash + parentHash + unclesHash + author + stateRoot + transactionsRoot + receiptsRoot + number + gasUsed + gasLimit + timestamp + difficulty + totalDifficulty + size + } +} diff --git a/packages/eden-watcher/test/queries/claim.gql b/packages/eden-watcher/test/queries/claim.gql new file mode 100644 index 00000000..cdd67794 --- /dev/null +++ b/packages/eden-watcher/test/queries/claim.gql @@ -0,0 +1,12 @@ +query claim($id: String!, $block: Block_height){ + claim(id: $id, block: $block){ + id + timestamp + index + account{ + id + } + totalEarned + claimed + } +} diff --git a/packages/eden-watcher/test/queries/distribution.gql b/packages/eden-watcher/test/queries/distribution.gql new file mode 100644 index 00000000..b838cca2 --- /dev/null +++ b/packages/eden-watcher/test/queries/distribution.gql @@ -0,0 +1,12 @@ +query distribution($id: String!, $block: Block_height){ + distribution(id: $id, block: $block){ + id + distributor{ + id + } + timestamp + distributionNumber + merkleRoot + metadataURI + } +} diff --git a/packages/eden-watcher/test/queries/distributor.gql b/packages/eden-watcher/test/queries/distributor.gql new file mode 100644 index 00000000..dfeeb6fc --- /dev/null +++ b/packages/eden-watcher/test/queries/distributor.gql @@ -0,0 +1,8 @@ +query distributor($id: String!, $block: Block_height){ + distributor(id: $id, block: $block){ + id + currentDistribution{ + id + } + } +} diff --git a/packages/eden-watcher/test/queries/epoch.gql b/packages/eden-watcher/test/queries/epoch.gql new file mode 100644 index 00000000..63ffd458 --- /dev/null +++ b/packages/eden-watcher/test/queries/epoch.gql @@ -0,0 +1,19 @@ +query epoch($id: String!, $block: Block_height){ + epoch(id: $id, block: $block){ + id + finalized + epochNumber + startBlock{ + id + } + endBlock{ + id + } + producerBlocks + allBlocks + producerBlocksRatio + producerRewards{ + id + } + } +} diff --git a/packages/eden-watcher/test/queries/network.gql b/packages/eden-watcher/test/queries/network.gql new file mode 100644 index 00000000..19a287d6 --- /dev/null +++ b/packages/eden-watcher/test/queries/network.gql @@ -0,0 +1,20 @@ +query network($id: String!, $block: Block_height){ + network(id: $id, block: $block){ + id + slot0{ + id + } + slot1{ + id + } + slot2{ + id + } + stakers{ + id + } + numStakers + totalStaked + stakedPercentiles + } +} diff --git a/packages/eden-watcher/test/queries/producer.gql b/packages/eden-watcher/test/queries/producer.gql new file mode 100644 index 00000000..47fbe7ee --- /dev/null +++ b/packages/eden-watcher/test/queries/producer.gql @@ -0,0 +1,10 @@ +query producer($id: String!, $block: Block_height){ + producer(id: $id, block: $block){ + id + active + rewardCollector + rewards + confirmedBlocks + pendingEpochBlocks + } +} diff --git a/packages/eden-watcher/test/queries/producerEpoch.gql b/packages/eden-watcher/test/queries/producerEpoch.gql new file mode 100644 index 00000000..83d21988 --- /dev/null +++ b/packages/eden-watcher/test/queries/producerEpoch.gql @@ -0,0 +1,12 @@ +query producerEpoch($id: String!, $block: Block_height){ + producerEpoch(id: $id, block: $block){ + id + address + epoch{ + id + } + totalRewards + blocksProduced + blocksProducedRatio + } +} diff --git a/packages/eden-watcher/test/queries/producerRewardCollectorChange.gql b/packages/eden-watcher/test/queries/producerRewardCollectorChange.gql new file mode 100644 index 00000000..249c419c --- /dev/null +++ b/packages/eden-watcher/test/queries/producerRewardCollectorChange.gql @@ -0,0 +1,8 @@ +query producerRewardCollectorChange($id: String!, $block: Block_height){ + producerRewardCollectorChange(id: $id, block: $block){ + id + blockNumber + producer + rewardCollector + } +} diff --git a/packages/eden-watcher/test/queries/producerSet.gql b/packages/eden-watcher/test/queries/producerSet.gql new file mode 100644 index 00000000..6ca0e145 --- /dev/null +++ b/packages/eden-watcher/test/queries/producerSet.gql @@ -0,0 +1,8 @@ +query producerSet($id: String!, $block: Block_height){ + producerSet(id: $id, block: $block){ + id + producers{ + id + } + } +} diff --git a/packages/eden-watcher/test/queries/producerSetChange.gql b/packages/eden-watcher/test/queries/producerSetChange.gql new file mode 100644 index 00000000..2e79f638 --- /dev/null +++ b/packages/eden-watcher/test/queries/producerSetChange.gql @@ -0,0 +1,8 @@ +query producerSetChange($id: String!, $block: Block_height){ + producerSetChange(id: $id, block: $block){ + id + blockNumber + producer + changeType + } +} diff --git a/packages/eden-watcher/test/queries/rewardSchedule.gql b/packages/eden-watcher/test/queries/rewardSchedule.gql new file mode 100644 index 00000000..0d69aeb0 --- /dev/null +++ b/packages/eden-watcher/test/queries/rewardSchedule.gql @@ -0,0 +1,17 @@ +query rewardSchedule($id: String!, $block: Block_height){ + rewardSchedule(id: $id, block: $block){ + id + rewardScheduleEntries{ + id + } + lastEpoch{ + id + } + pendingEpoch{ + id + } + activeRewardScheduleEntry{ + id + } + } +} diff --git a/packages/eden-watcher/test/queries/rewardScheduleEntry.gql b/packages/eden-watcher/test/queries/rewardScheduleEntry.gql new file mode 100644 index 00000000..ecf4aee2 --- /dev/null +++ b/packages/eden-watcher/test/queries/rewardScheduleEntry.gql @@ -0,0 +1,8 @@ +query rewardScheduleEntry($id: String!, $block: Block_height){ + rewardScheduleEntry(id: $id, block: $block){ + id + startTime + epochDuration + rewardsPerEpoch + } +} diff --git a/packages/eden-watcher/test/queries/slash.gql b/packages/eden-watcher/test/queries/slash.gql new file mode 100644 index 00000000..db0c611c --- /dev/null +++ b/packages/eden-watcher/test/queries/slash.gql @@ -0,0 +1,10 @@ +query slash($id: String!, $block: Block_height){ + slash(id: $id, block: $block){ + id + timestamp + account{ + id + } + slashed + } +} diff --git a/packages/eden-watcher/test/queries/slot.gql b/packages/eden-watcher/test/queries/slot.gql new file mode 100644 index 00000000..f570f03b --- /dev/null +++ b/packages/eden-watcher/test/queries/slot.gql @@ -0,0 +1,15 @@ +query slot($id: String!, $block: Block_height){ + slot(id: $id, block: $block){ + id + owner + delegate + winningBid + oldBid + startTime + expirationTime + taxRatePerDay + claims{ + id + } + } +} diff --git a/packages/eden-watcher/test/queries/slotClaim.gql b/packages/eden-watcher/test/queries/slotClaim.gql new file mode 100644 index 00000000..425664df --- /dev/null +++ b/packages/eden-watcher/test/queries/slotClaim.gql @@ -0,0 +1,14 @@ +query slotClaim($id: String!, $block: Block_height){ + slotClaim(id: $id, block: $block){ + id + slot{ + id + } + owner + winningBid + oldBid + startTime + expirationTime + taxRatePerDay + } +} diff --git a/packages/eden-watcher/test/queries/staker.gql b/packages/eden-watcher/test/queries/staker.gql new file mode 100644 index 00000000..bb4cdd51 --- /dev/null +++ b/packages/eden-watcher/test/queries/staker.gql @@ -0,0 +1,7 @@ +query staker($id: String!, $block: Block_height){ + staker(id: $id, block: $block){ + id + staked + rank + } +} diff --git a/packages/graph-node/README.md b/packages/graph-node/README.md index 66f08124..68c78415 100644 --- a/packages/graph-node/README.md +++ b/packages/graph-node/README.md @@ -75,9 +75,9 @@ ./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10 ``` - * For comparing entities after fetching ids from one of the endpoints and then querying individually by ids: + * For comparing entities after fetching updated entity ids from watcher database: - * Set the `idsEndpoint` to choose which endpoint the ids should be fetched from. + * Set the watcher config file path and entities directory. ```toml [endpoints] @@ -90,7 +90,20 @@ "author", "blog" ] - idsEndpoint = "gqlEndpoint1" + + [watcher] + configPath = "../../graph-test-watcher/environments/local.toml" + entitiesDir = "../../graph-test-watcher/dist/entity/*" + ``` + + * To verify diff IPLD state generated at each block, set the watcher endpoint and `verifyState` flag to true + + ```toml + [watcher] + configPath = "../../graph-test-watcher/environments/local.toml" + entitiesDir = "../../graph-test-watcher/dist/entity/*" + endpoint = "gqlEndpoint2" + verifyState = true ``` * Run the CLI with `fetch-ids` flag set to true:\ diff --git a/packages/graph-node/environments/compare-cli-config.toml b/packages/graph-node/environments/compare-cli-config.toml index 0cf851ac..18abe269 100644 --- a/packages/graph-node/environments/compare-cli-config.toml +++ b/packages/graph-node/environments/compare-cli-config.toml @@ -6,6 +6,13 @@ queryDir = "../graph-test-watcher/src/gql/queries" names = [] idsEndpoint = "gqlEndpoint1" + blockDelayInMs = 250 + +[watcher] + configpath = "../../graph-test-watcher/environments/local.toml" + entitiesDir = "../../graph-test-watcher/src/entity" + endpoint = "gqlEndpoint2" + verifyState = true [cache] endpoint = "gqlEndpoint1" diff --git a/packages/graph-node/package.json b/packages/graph-node/package.json index feea870b..9d84e8ee 100644 --- a/packages/graph-node/package.json +++ b/packages/graph-node/package.json @@ -58,6 +58,7 @@ "js-yaml": "^4.1.0", "json-bigint": "^1.0.0", "json-diff": "^0.5.4", + "omit-deep": "^0.3.0", "pluralize": "^8.0.0", "reflect-metadata": "^0.1.13", "toml": "^3.0.0", diff --git a/packages/graph-node/src/cli/compare/compare-blocks.ts b/packages/graph-node/src/cli/compare/compare-blocks.ts index 77fd85cb..c4f460d6 100644 --- a/packages/graph-node/src/cli/compare/compare-blocks.ts +++ b/packages/graph-node/src/cli/compare/compare-blocks.ts @@ -5,10 +5,17 @@ import yargs from 'yargs'; import 'reflect-metadata'; import debug from 'debug'; +import path from 'path'; import assert from 'assert'; +import { SnakeNamingStrategy } from 'typeorm-naming-strategies'; +import _ from 'lodash'; +import omitDeep from 'omit-deep'; +import { getConfig as getWatcherConfig, wait } from '@vulcanize/util'; +import { GraphQLClient } from '@vulcanize/ipld-eth-client'; -import { compareQuery, Config, getClients, getConfig } from './utils'; -import { Client } from './client'; +import { compareObjects, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils'; +import { Database } from '../../database'; +import { getSubgraphConfig } from '../../utils'; const log = debug('vulcanize:compare-blocks'); @@ -50,48 +57,109 @@ export const main = async (): Promise => { } }).argv; - const config: Config = await getConfig(argv.configFile); - - const { startBlock, endBlock, rawJson, queryDir, fetchIds } = argv; + const { startBlock, endBlock, rawJson, queryDir, fetchIds, configFile } = argv; + const config: Config = await getConfig(configFile); + const snakeNamingStrategy = new SnakeNamingStrategy(); + const clients = await getClients(config, queryDir); const queryNames = config.queries.names; let diffFound = false; + let blockDelay = wait(0); + let subgraphContracts: string[] = []; + let db: Database | undefined, subgraphGQLClient: GraphQLClient | undefined; - const clients = await getClients(config, queryDir); + if (config.watcher) { + const watcherConfigPath = path.resolve(path.dirname(configFile), config.watcher.configPath); + const entitiesDir = path.resolve(path.dirname(configFile), config.watcher.entitiesDir); + const watcherConfig = await getWatcherConfig(watcherConfigPath); + db = new Database(watcherConfig.database, entitiesDir); + await db.init(); + + if (config.watcher.verifyState) { + const { dataSources } = await getSubgraphConfig(watcherConfig.server.subgraphPath); + subgraphContracts = dataSources.map((dataSource: any) => dataSource.source.address); + const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string; + subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint }); + } + } for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) { const block = { number: blockNumber }; + let updatedEntityIds: string[][] = []; + let ipldStateByBlock = {}; console.time(`time:compare-block-${blockNumber}`); - for (const queryName of queryNames) { + if (fetchIds) { + // Fetch entity ids updated at block. + console.time(`time:fetch-updated-ids-${blockNumber}`); + + const updatedEntityIdPromises = queryNames.map( + queryName => { + assert(db); + + return db.getEntityIdsAtBlockNumber( + blockNumber, + snakeNamingStrategy.tableName(queryName, '') + ); + } + ); + + updatedEntityIds = await Promise.all(updatedEntityIdPromises); + console.timeEnd(`time:fetch-updated-ids-${blockNumber}`); + } + + if (config.watcher.verifyState) { + assert(db); + const [block] = await db?.getBlocksAtHeight(blockNumber, false); + assert(subgraphGQLClient); + ipldStateByBlock = await getIPLDStateByBlock(subgraphGQLClient, subgraphContracts, block.blockHash); + } + + await blockDelay; + for (const [index, queryName] of queryNames.entries()) { try { log(`At block ${blockNumber} for query ${queryName}:`); + let resultDiff = ''; if (fetchIds) { - const { idsEndpoint } = config.queries; - assert(idsEndpoint, 'Specify endpoint for fetching ids when fetchId is true'); - const client = Object.values(clients).find(client => client.endpoint === config.endpoints[idsEndpoint]); - assert(client); - const ids = await client.getIds(queryName, blockNumber); + for (const id of updatedEntityIds[index]) { + const { diff, result1: result } = await compareQuery( + clients, + queryName, + { block, id }, + rawJson + ); - for (const id of ids) { - const isDiff = await compareAndLog(clients, queryName, { block, id }, rawJson); + if (config.watcher.verifyState) { + await checkEntityInIPLDState(ipldStateByBlock, queryName, result, id, rawJson); + } - if (isDiff) { - diffFound = isDiff; + if (diff) { + resultDiff = diff; } } } else { - const isDiff = await compareAndLog(clients, queryName, { block }, rawJson); + ({ diff: resultDiff } = await compareQuery( + clients, + queryName, + { block }, + rawJson + )); + } - if (isDiff) { - diffFound = isDiff; - } + if (resultDiff) { + log('Results mismatch:', resultDiff); + diffFound = true; + } else { + log('Results match.'); } } catch (err: any) { log('Error:', err.message); } } + // Set delay between requests for a block. + blockDelay = wait(config.queries.blockDelayInMs || 0); + console.timeEnd(`time:compare-block-${blockNumber}`); } @@ -100,24 +168,21 @@ export const main = async (): Promise => { } }; -const compareAndLog = async ( - clients: { client1: Client, client2: Client }, +const checkEntityInIPLDState = async ( + ipldState: {[key: string]: any}, queryName: string, - params: { [key: string]: any }, + entityResult: {[key: string]: any}, + id: string, rawJson: boolean -): Promise => { - const resultDiff = await compareQuery( - clients, - queryName, - params, - rawJson - ); +) => { + const entityName = _.startCase(queryName); + const ipldEntity = ipldState[entityName][id]; - if (resultDiff) { - log('Results mismatch:', resultDiff); - return true; + // Filter __typename key in GQL result. + const resultEntity = omitDeep(entityResult[queryName], '__typename'); + const diff = compareObjects(ipldEntity, resultEntity, rawJson); + + if (diff) { + log('Results mismatch for IPLD state:', diff); } - - log('Results match.'); - return false; }; diff --git a/packages/graph-node/src/cli/compare/compare-entity.ts b/packages/graph-node/src/cli/compare/compare-entity.ts index 18c570f2..95dc8a9c 100644 --- a/packages/graph-node/src/cli/compare/compare-entity.ts +++ b/packages/graph-node/src/cli/compare/compare-entity.ts @@ -4,9 +4,12 @@ import yargs from 'yargs'; import 'reflect-metadata'; +import debug from 'debug'; import { compareQuery, Config, getClients, getConfig } from './utils'; +const log = debug('vulcanize:compare-entity'); + export const main = async (): Promise => { const argv = await yargs.parserConfiguration({ 'parse-numbers': false @@ -62,10 +65,10 @@ export const main = async (): Promise => { const clients = await getClients(config, argv.queryDir); - const resultDiff = await compareQuery(clients, queryName, { id, block }, argv.rawJson); + const { diff } = await compareQuery(clients, queryName, { id, block }, argv.rawJson); - if (resultDiff) { - console.log(resultDiff); + if (diff) { + log(diff); process.exit(1); } }; diff --git a/packages/graph-node/src/cli/compare/utils.ts b/packages/graph-node/src/cli/compare/utils.ts index f77ff788..96f9bf45 100644 --- a/packages/graph-node/src/cli/compare/utils.ts +++ b/packages/graph-node/src/cli/compare/utils.ts @@ -8,25 +8,43 @@ import path from 'path'; import toml from 'toml'; import fs from 'fs-extra'; import { diffString, diff } from 'json-diff'; +import _ from 'lodash'; import { Config as CacheConfig, getCache } from '@vulcanize/cache'; +import { GraphQLClient } from '@vulcanize/ipld-eth-client'; +import { gql } from '@apollo/client/core'; import { Client } from './client'; +const IPLD_STATE_QUERY = ` +query getState($blockHash: String!, $contractAddress: String!, $kind: String){ + getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){ + data + } +} +`; + interface EndpointConfig { gqlEndpoint1: string; gqlEndpoint2: string; + requestDelayInMs: number; } interface QueryConfig { queryDir: string; names: string[]; - idsEndpoint: keyof EndpointConfig; + blockDelayInMs: number; } export interface Config { endpoints: EndpointConfig; queries: QueryConfig; + watcher: { + configPath: string; + entitiesDir: string; + verifyState: boolean; + endpoint: keyof EndpointConfig; + } cache: { endpoint: keyof EndpointConfig; config: CacheConfig; @@ -52,6 +70,12 @@ export const getConfig = async (configFile: string): Promise => { return config; }; +interface CompareResult { + diff: string, + result1: any, + result2: any +} + export const compareQuery = async ( clients: { client1: Client, @@ -60,27 +84,22 @@ export const compareQuery = async ( queryName: string, params: { [key: string]: any }, rawJson: boolean -): Promise => { +): Promise => { const { client1, client2 } = clients; - const result2 = await client2.getResult(queryName, params); - const result1 = await client1.getResult(queryName, params); + const [result1, result2] = await Promise.all([ + client1.getResult(queryName, params), + client2.getResult(queryName, params) + ]); // Getting the diff of two result objects. - let resultDiff; + const resultDiff = compareObjects(result1, result2, rawJson); - if (rawJson) { - resultDiff = diff(result1, result2); - - if (resultDiff) { - // Use util.inspect to extend depth limit in the output. - resultDiff = util.inspect(diff(result1, result2), false, null); - } - } else { - resultDiff = diffString(result1, result2); - } - - return resultDiff; + return { + diff: resultDiff, + result1, + result2 + }; }; export const getClients = async (config: Config, queryDir?: string):Promise<{ @@ -121,3 +140,40 @@ export const getClients = async (config: Config, queryDir?: string):Promise<{ client2 }; }; + +export const getBlockIPLDState = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}> => { + const contractIPLDStates: {[key: string]: any}[] = await Promise.all(contracts.map(async contract => { + const { getState } = await client.query( + gql(IPLD_STATE_QUERY), + { + blockHash, + contractAddress: contract, + kind: 'diff' + } + ); + + if (getState) { + const data = JSON.parse(getState.data); + return data.state; + } + + return {}; + })); + + return contractIPLDStates.reduce((acc, state) => _.merge(acc, state)); +}; + +export const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => { + if (rawJson) { + const diffObj = diff(obj1, obj2); + + if (diffObj) { + // Use util.inspect to extend depth limit in the output. + return util.inspect(diffObj, false, null); + } + + return ''; + } else { + return diffString(obj1, obj2); + } +}; diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index f2b192b0..048326b7 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -7,11 +7,13 @@ import { Connection, ConnectionOptions, FindOneOptions, - LessThanOrEqual + LessThanOrEqual, + Repository } from 'typeorm'; import { BlockHeight, + BlockProgressInterface, Database as BaseDatabase } from '@vulcanize/util'; @@ -77,6 +79,19 @@ export class Database { } } + async getEntityIdsAtBlockNumber (blockNumber: number, tableName: string): Promise { + const repo = this._conn.getRepository(tableName); + + const entities = await repo.find({ + select: ['id'], + where: { + blockNumber + } + }); + + return entities.map((entity: any) => entity.id); + } + async getEntityWithRelations (entity: (new () => Entity) | string, id: string, relations: { [key: string]: any }, block: BlockHeight = {}): Promise { const queryRunner = this._conn.createQueryRunner(); let { hash: blockHash, number: blockNumber } = block; @@ -265,4 +280,10 @@ export class Database { return acc; }, {}); } + + async getBlocksAtHeight (height: number, isPruned: boolean) { + const repo: Repository = this._conn.getRepository('block_progress'); + + return this._baseDatabase.getBlocksAtHeight(repo, height, isPruned); + } } diff --git a/packages/graph-node/src/loader.ts b/packages/graph-node/src/loader.ts index 26e5a48d..10fb24cf 100644 --- a/packages/graph-node/src/loader.ts +++ b/packages/graph-node/src/loader.ts @@ -10,7 +10,6 @@ import { Contract, ContractInterface } from 'ethers'; -import JSONbig from 'json-bigint'; import BN from 'bn.js'; import debug from 'debug'; @@ -26,12 +25,11 @@ import { resolveEntityFieldConflicts, getEthereumTypes, jsonFromBytes, - getStorageValueType + getStorageValueType, + jsonBigIntStringReplacer } from './utils'; import { Database } from './database'; -const JSONbigString = JSONbig({ storeAsString: true }); - // Endianness of BN used in bigInt store host API. // Negative bigInt is being stored in wasm in 2's compliment, 'le' representation. // (for eg. bigInt.fromString(negativeI32Value)) @@ -104,14 +102,39 @@ export const instantiate = async ( // Prepare the diff data. const diffData: any = { state: {} }; + assert(indexer.getRelationsMap); + + const result = Array.from(indexer.getRelationsMap().entries()) + .find(([key]) => key.name === entityName); + + if (result) { + // Update dbData if relations exist. + const [_, relations] = result; + + // Update relation fields for diff data to be similar to GQL query entities. + Object.entries(relations).forEach(([relation, { isArray, isDerived }]) => { + if (isDerived || !dbData[relation]) { + // Field is not present in dbData for derived relations + return; + } + + if (isArray) { + dbData[relation] = dbData[relation] + .map((id: string) => ({ id })) + .sort((a: any, b: any) => a.id.localeCompare(b.id)); + } else { + dbData[relation] = { id: dbData[relation] }; + } + }); + } // JSON stringify and parse data for handling unknown types when encoding. // For example, decimal.js values are converted to string in the diff data. diffData.state[entityName] = { - // Using JSONbigString to store bigints as string values to be encoded by IPLD dag-cbor. + // Using custom replacer to store bigints as string values to be encoded by IPLD dag-cbor. // TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode. // https://github.com/rvagg/cborg#type-encoders - [dbData.id]: JSONbigString.parse(JSONbigString.stringify(dbData)) + [dbData.id]: JSON.parse(JSON.stringify(dbData, jsonBigIntStringReplacer)) }; // Create an auto-diff. diff --git a/packages/graph-node/src/types/common/main.d.ts b/packages/graph-node/src/types/common/main.d.ts new file mode 100644 index 00000000..d638963f --- /dev/null +++ b/packages/graph-node/src/types/common/main.d.ts @@ -0,0 +1,5 @@ +// +// Copyright 2022 Vulcanize, Inc. +// + +declare module 'omit-deep'; diff --git a/packages/graph-node/src/types/common/package.json b/packages/graph-node/src/types/common/package.json new file mode 100644 index 00000000..5861d0f0 --- /dev/null +++ b/packages/graph-node/src/types/common/package.json @@ -0,0 +1,6 @@ +{ + "name": "common", + "version": "0.1.0", + "license": "AGPL-3.0", + "typings": "main.d.ts" +} diff --git a/packages/graph-node/src/utils.ts b/packages/graph-node/src/utils.ts index ede17b6c..6c6f991d 100644 --- a/packages/graph-node/src/utils.ts +++ b/packages/graph-node/src/utils.ts @@ -798,3 +798,11 @@ const getEthereumType = (storageTypes: StorageLayout['types'], type: string, map return utils.ParamType.from(label); }; + +export const jsonBigIntStringReplacer = (_: string, value: any) => { + if (typeof value === 'bigint') { + return value.toString(); + } + + return value; +}; diff --git a/packages/graph-node/tsconfig.json b/packages/graph-node/tsconfig.json index 65e490f4..d6824857 100644 --- a/packages/graph-node/tsconfig.json +++ b/packages/graph-node/tsconfig.json @@ -52,7 +52,7 @@ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ - // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 33264cbe..b1f0cda6 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -106,6 +106,7 @@ export interface IndexerInterface { cacheContract?: (contract: ContractInterface) => void; watchContract?: (address: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise getEntityTypesMap?: () => Map + getRelationsMap?: () => Map createDiffStaged?: (contractAddress: string, blockHash: string, data: any) => Promise processInitialState?: (contractAddress: string, blockHash: string) => Promise processStateCheckpoint?: (contractAddress: string, blockHash: string) => Promise diff --git a/yarn.lock b/yarn.lock index d45839e0..4432cbdd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8596,7 +8596,7 @@ is-plain-obj@^2.0.0, is-plain-obj@^2.1.0: resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== -is-plain-object@^2.0.3, is-plain-object@^2.0.4: +is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== @@ -10817,6 +10817,14 @@ oboe@2.1.4: dependencies: http-https "^1.0.0" +omit-deep@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/omit-deep/-/omit-deep-0.3.0.tgz#21c8af3499bcadd29651a232cbcacbc52445ebec" + integrity sha512-Lbl/Ma59sss2b15DpnWnGmECBRL8cRl/PjPbPMVW+Y8zIQzRrwMaI65Oy6HvxyhYeILVKBJb2LWeG81bj5zbMg== + dependencies: + is-plain-object "^2.0.1" + unset-value "^0.1.1" + on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" @@ -13984,6 +13992,14 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= +unset-value@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-0.1.2.tgz#506810b867f27c2a5a6e9b04833631f6de58d310" + integrity sha512-yhv5I4TsldLdE3UcVQn0hD2T5sNCPv4+qm/CTUpRKIpwthYRIipsAPdsrNpOI79hPQa0rTTeW22Fq6JWRcTgNg== + dependencies: + has-value "^0.3.1" + isobject "^3.0.0" + unset-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559"