cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2025-03-15 23:39:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update compare CLI to verify IPLD block meta data (#179)

Browse Source 
* Update compare CLI to compare IPLD block meta data

* Include init and checkpoints in IPLD blocks verification

* Accomodate changes in codegen
This commit is contained in:
prathamesh0 2022-09-12 17:44:53 +05:30 committed by GitHub
parent 996f68a390
commit 8960f67f1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 154 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/codegen/src/templates/resolvers-template.handlebars
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -125,7 +125,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);

4
packages/eden-watcher/src/resolvers.ts
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { BlockHeight, OrderDirection, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -311,7 +311,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);

4
packages/erc721-watcher/src/resolvers.ts
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, BlockHeight, StateKind } from '@cerc-io/util';
import { ValueResult, BlockHeight } from '@cerc-io/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -166,7 +166,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);

21
packages/graph-node/src/cli/compare/compare-blocks.ts
View File

@ -12,7 +12,7 @@ import _ from 'lodash';
import { getConfig as getWatcherConfig, wait } from '@cerc-io/util';
import { GraphQLClient } from '@cerc-io/ipld-eth-client';
import { checkEntityInIPLDState, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils';
import { checkEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig } from './utils';
import { Database } from '../../database';
import { getSubgraphConfig } from '../../utils';
@ -64,6 +64,7 @@ export const main = async (): Promise<void> => {
let diffFound = false;
let blockDelay = wait(0);
let subgraphContracts: string[] = [];
const contractLatestStateCIDMap: Map<string, string> = new Map();
let db: Database | undefined, subgraphGQLClient: GraphQLClient | undefined;
if (config.watcher) {
@ -79,6 +80,10 @@ export const main = async (): Promise<void> => {
const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string;
subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint });
}
subgraphContracts.forEach(subgraphContract => {
contractLatestStateCIDMap.set(subgraphContract, '');
});
}
for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) {
@ -110,7 +115,18 @@ export const main = async (): Promise<void> => {
assert(db);
const [block] = await db?.getBlocksAtHeight(blockNumber, false);
assert(subgraphGQLClient);
ipldStateByBlock = await getIPLDStateByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
// Check meta data for each IPLD block found
contractIPLDsByBlock.flat().forEach(contractIPLD => {
const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
if (ipldMetaDataDiff) {
log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
diffFound = true;
}
});
ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
}
await blockDelay;
@ -166,6 +182,7 @@ export const main = async (): Promise<void> => {
}
} catch (err: any) {
log('Error:', err.message);
log('Error:', err);
}
}

148
packages/graph-node/src/cli/compare/utils.ts
View File

@ -21,6 +21,14 @@ import { DEFAULT_LIMIT } from '../../database';
const IPLD_STATE_QUERY = `
query getState($blockHash: String!, $contractAddress: String!, $kind: String){
getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){
block {
cid
number
hash
}
contractAddress
cid
kind
data
}
}
@ -150,43 +158,131 @@ export const getClients = async (config: Config, queryDir?: string):Promise<{
};
};
export const getBlockIPLDState = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}> => {
const contractIPLDStates: {[key: string]: any}[] = await Promise.all(contracts.map(async contract => {
export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
// Fetch IPLD states for all contracts
return Promise.all(contracts.map(async contract => {
const { getState } = await client.query(
gql(IPLD_STATE_QUERY),
{
blockHash,
contractAddress: contract,
kind: 'diff'
contractAddress: contract
}
);
if (getState) {
const data = JSON.parse(getState.data);
const stateIPLDs = [];
// Apply default limit on array type relation fields.
Object.values(data.state)
.forEach((idEntityMap: any) => {
Object.values(idEntityMap)
.forEach((entity: any) => {
Object.values(entity)
.forEach(fieldValue => {
if (
Array.isArray(fieldValue) &&
fieldValue.length &&
fieldValue[0].id
) {
fieldValue.splice(DEFAULT_LIMIT);
}
});
});
});
// If 'checkpoint' is found at the same block, fetch 'diff' as well
if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) {
// Check if 'init' present at the same block
const { getState: getInitState } = await client.query(
gql(IPLD_STATE_QUERY),
{
blockHash,
contractAddress: contract,
kind: 'init'
}
);
return data.state;
if (getInitState && getInitState.block.hash === blockHash) {
// Append the 'init' IPLD block to the result
stateIPLDs.push(getInitState);
}
// Check if 'diff' present at the same block
const { getState: getDiffState } = await client.query(
gql(IPLD_STATE_QUERY),
{
blockHash,
contractAddress: contract,
kind: 'diff'
}
);
if (getDiffState && getDiffState.block.hash === blockHash) {
// Append the 'diff' IPLD block to the result
stateIPLDs.push(getDiffState);
}
}
return {};
// Append the IPLD block to the result
stateIPLDs.push(getState);
return stateIPLDs;
}));
};
export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map<string, string>, rawJson: boolean) => {
// Return if IPLD for a contract not found
if (!contractIPLD) {
return;
}
const { contractAddress, cid, kind, block } = contractIPLD;
let parentCID = contractLatestStateCIDMap.get(contractAddress);
// If CID is same as the parent CID, skip the check
if (cid === parentCID) {
return;
}
// Update the parent CID in the map
contractLatestStateCIDMap.set(contractAddress, cid);
// Actual meta data from the GQL result
const data = JSON.parse(contractIPLD.data);
// If parentCID not initialized (is empty at start)
// Take the expected parentCID from the actual data itself
if (parentCID === '') {
parentCID = data.meta.parent['/'];
}
// Expected meta data
const expectedMetaData = {
id: contractAddress,
kind,
parent: {
'/': parentCID
},
ethBlock: {
cid: {
'/': block.cid
},
num: block.number
}
};
return compareObjects(expectedMetaData, data.meta, rawJson);
};
export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => {
const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => {
if (!contractIPLD) {
return {};
}
const data = JSON.parse(contractIPLD.data);
// Apply default limit on array type relation fields.
Object.values(data.state)
.forEach((idEntityMap: any) => {
Object.values(idEntityMap)
.forEach((entity: any) => {
Object.values(entity)
.forEach(fieldValue => {
if (
Array.isArray(fieldValue) &&
fieldValue.length &&
fieldValue[0].id
) {
fieldValue.splice(DEFAULT_LIMIT);
}
});
});
});
return data.state;
});
return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
};
@ -219,6 +315,8 @@ export const checkEntityInIPLDState = async (
return diff;
};
// obj1: expected
// obj2: actual
const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => {
if (rawJson) {
const diffObj = diff(obj1, obj2);

2
packages/graph-node/src/loader.ts
View File

@ -15,7 +15,7 @@ import debug from 'debug';
import { BaseProvider } from '@ethersproject/providers';
import loader from '@vulcanize/assemblyscript/lib/loader';
import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp, jsonBigIntStringReplacer } from '@cerc-io/util';
import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp } from '@cerc-io/util';
import { TypeId, Level } from './types';
import {

2
packages/graph-node/tsconfig.json
View File

@ -11,7 +11,7 @@
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es5", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"target": "ES2019", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */

4
packages/graph-test-watcher/src/resolvers.ts
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, BlockHeight, StateKind, jsonBigIntStringReplacer } from '@cerc-io/util';
import { ValueResult, BlockHeight, jsonBigIntStringReplacer } from '@cerc-io/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -122,7 +122,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);

4
packages/mobymask-watcher/src/resolvers.ts
View File

@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';
import { ValueResult, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';
import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';
import { Indexer } from './indexer';
import { EventWatcher } from './events';
@ -131,7 +131,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},
getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);