mirror of
https://github.com/cerc-io/watcher-ts
synced 2025-07-31 12:02:07 +00:00
Add an option to run compare CLI in batched intervals (#202)
This commit is contained in:
prathamesh0
GitHub
parent
5ffe13b723
commit
45111e68ac
@ -45,6 +45,16 @@ export const main = async (): Promise<void> => {
|
|||||||
demandOption: true,
|
demandOption: true,
|
||||||
describe: 'End block number'
|
describe: 'End block number'
|
||||||
},
|
},
|
||||||
|
batchSize: {
|
||||||
|
type: 'number',
|
||||||
|
default: 1,
|
||||||
|
describe: 'No. of blocks to be compared in an interval (default 1)'
|
||||||
|
},
|
||||||
|
interval: {
|
||||||
|
type: 'number',
|
||||||
|
default: 1,
|
||||||
|
describe: 'Block interval (default 1)'
|
||||||
|
},
|
||||||
rawJson: {
|
rawJson: {
|
||||||
alias: 'j',
|
alias: 'j',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
@ -63,7 +73,7 @@ export const main = async (): Promise<void> => {
|
|||||||
}
|
}
|
||||||
}).argv;
|
}).argv;
|
||||||
|
|
||||||
const { startBlock, endBlock, rawJson, queryDir, fetchIds, configFile, timeDiff } = argv;
|
const { startBlock, endBlock, batchSize, interval, rawJson, queryDir, fetchIds, configFile, timeDiff } = argv;
|
||||||
const config: Config = await getConfig(configFile);
|
const config: Config = await getConfig(configFile);
|
||||||
const snakeNamingStrategy = new SnakeNamingStrategy();
|
const snakeNamingStrategy = new SnakeNamingStrategy();
|
||||||
const clients = await getClients(config, timeDiff, queryDir);
|
const clients = await getClients(config, timeDiff, queryDir);
|
||||||
@ -93,132 +103,135 @@ export const main = async (): Promise<void> => {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) {
|
for (let bathchStart = startBlock; bathchStart <= endBlock; bathchStart += interval) {
|
||||||
const block = { number: blockNumber };
|
const batchEnd = bathchStart + batchSize;
|
||||||
const updatedEntityIds: { [entityName: string]: string[] } = {};
|
for (let blockNumber = bathchStart; blockNumber < batchEnd && blockNumber <= endBlock; blockNumber++) {
|
||||||
const updatedEntities: Set<string> = new Set();
|
const block = { number: blockNumber };
|
||||||
let ipldStateByBlock = {};
|
const updatedEntityIds: { [entityName: string]: string[] } = {};
|
||||||
assert(db);
|
const updatedEntities: Set<string> = new Set();
|
||||||
console.time(`time:compare-block-${blockNumber}`);
|
let ipldStateByBlock = {};
|
||||||
|
|
||||||
if (fetchIds) {
|
|
||||||
// Fetch entity ids updated at block.
|
|
||||||
console.time(`time:fetch-updated-ids-${blockNumber}`);
|
|
||||||
|
|
||||||
for (const entityName of Object.values(queryNames)) {
|
|
||||||
updatedEntityIds[entityName] = await db.getEntityIdsAtBlockNumber(
|
|
||||||
blockNumber,
|
|
||||||
snakeNamingStrategy.tableName(entityName, '')
|
|
||||||
);
|
|
||||||
}
|
|
||||||
console.timeEnd(`time:fetch-updated-ids-${blockNumber}`);
|
|
||||||
} else {
|
|
||||||
for (const entityName of Object.values(queryNames)) {
|
|
||||||
const isUpdated = await db.isEntityUpdatedAtBlockNumber(
|
|
||||||
blockNumber,
|
|
||||||
snakeNamingStrategy.tableName(entityName, '')
|
|
||||||
);
|
|
||||||
|
|
||||||
if (isUpdated) {
|
|
||||||
updatedEntities.add(entityName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.watcher.verifyState) {
|
|
||||||
assert(db);
|
assert(db);
|
||||||
const [block] = await db.getBlocksAtHeight(blockNumber, false);
|
console.time(`time:compare-block-${blockNumber}`);
|
||||||
assert(subgraphGQLClient);
|
|
||||||
const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
|
|
||||||
|
|
||||||
// Check meta data for each IPLD block found
|
if (fetchIds) {
|
||||||
contractIPLDsByBlock.flat().forEach(contractIPLD => {
|
// Fetch entity ids updated at block.
|
||||||
const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
|
console.time(`time:fetch-updated-ids-${blockNumber}`);
|
||||||
if (ipldMetaDataDiff) {
|
|
||||||
log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
|
for (const entityName of Object.values(queryNames)) {
|
||||||
diffFound = true;
|
updatedEntityIds[entityName] = await db.getEntityIdsAtBlockNumber(
|
||||||
|
blockNumber,
|
||||||
|
snakeNamingStrategy.tableName(entityName, '')
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
console.timeEnd(`time:fetch-updated-ids-${blockNumber}`);
|
||||||
|
} else {
|
||||||
|
for (const entityName of Object.values(queryNames)) {
|
||||||
|
const isUpdated = await db.isEntityUpdatedAtBlockNumber(
|
||||||
|
blockNumber,
|
||||||
|
snakeNamingStrategy.tableName(entityName, '')
|
||||||
|
);
|
||||||
|
|
||||||
ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
|
if (isUpdated) {
|
||||||
}
|
updatedEntities.add(entityName);
|
||||||
|
|
||||||
await blockDelay;
|
|
||||||
for (const [queryName, entityName] of Object.entries(queryNames)) {
|
|
||||||
try {
|
|
||||||
log(`At block ${blockNumber} for query ${queryName}:`);
|
|
||||||
let resultDiff = '';
|
|
||||||
|
|
||||||
if (fetchIds) {
|
|
||||||
const queryLimit = config.queries.queryLimits[queryName];
|
|
||||||
|
|
||||||
if (queryLimit) {
|
|
||||||
// Take only last `queryLimit` entity ids to compare in GQL.
|
|
||||||
const idsLength = updatedEntityIds[entityName].length;
|
|
||||||
updatedEntityIds[entityName].splice(0, idsLength - queryLimit);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const id of updatedEntityIds[entityName]) {
|
|
||||||
const { diff, result1: result } = await compareQuery(
|
|
||||||
clients,
|
|
||||||
queryName,
|
|
||||||
{ block, id },
|
|
||||||
rawJson,
|
|
||||||
timeDiff
|
|
||||||
);
|
|
||||||
|
|
||||||
if (config.watcher.verifyState) {
|
|
||||||
const ipldDiff = await checkGQLEntityInIPLDState(ipldStateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields);
|
|
||||||
|
|
||||||
if (ipldDiff) {
|
|
||||||
log('Results mismatch for IPLD state:', ipldDiff);
|
|
||||||
diffFound = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (diff) {
|
|
||||||
resultDiff = diff;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (updatedEntities.has(entityName)) {
|
|
||||||
let result;
|
|
||||||
|
|
||||||
({ diff: resultDiff, result1: result } = await compareQuery(
|
|
||||||
clients,
|
|
||||||
queryName,
|
|
||||||
{ block },
|
|
||||||
rawJson,
|
|
||||||
timeDiff
|
|
||||||
));
|
|
||||||
|
|
||||||
if (config.watcher.verifyState) {
|
|
||||||
const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
|
|
||||||
|
|
||||||
if (ipldDiff) {
|
|
||||||
log('Results mismatch for IPLD state:', ipldDiff);
|
|
||||||
diffFound = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (resultDiff) {
|
|
||||||
log('Results mismatch:', resultDiff);
|
|
||||||
diffFound = true;
|
|
||||||
} else {
|
|
||||||
log('Results match.');
|
|
||||||
}
|
|
||||||
} catch (err: any) {
|
|
||||||
log('Error:', err.message);
|
|
||||||
log('Error:', JSON.stringify(err, null, 2));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (config.watcher.verifyState) {
|
||||||
|
assert(db);
|
||||||
|
const [block] = await db.getBlocksAtHeight(blockNumber, false);
|
||||||
|
assert(subgraphGQLClient);
|
||||||
|
const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
|
||||||
|
|
||||||
|
// Check meta data for each IPLD block found
|
||||||
|
contractIPLDsByBlock.flat().forEach(contractIPLD => {
|
||||||
|
const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
|
||||||
|
if (ipldMetaDataDiff) {
|
||||||
|
log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
|
||||||
|
diffFound = true;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
|
||||||
|
}
|
||||||
|
|
||||||
|
await blockDelay;
|
||||||
|
for (const [queryName, entityName] of Object.entries(queryNames)) {
|
||||||
|
try {
|
||||||
|
log(`At block ${blockNumber} for query ${queryName}:`);
|
||||||
|
let resultDiff = '';
|
||||||
|
|
||||||
|
if (fetchIds) {
|
||||||
|
const queryLimit = config.queries.queryLimits[queryName];
|
||||||
|
|
||||||
|
if (queryLimit) {
|
||||||
|
// Take only last `queryLimit` entity ids to compare in GQL.
|
||||||
|
const idsLength = updatedEntityIds[entityName].length;
|
||||||
|
updatedEntityIds[entityName].splice(0, idsLength - queryLimit);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const id of updatedEntityIds[entityName]) {
|
||||||
|
const { diff, result1: result } = await compareQuery(
|
||||||
|
clients,
|
||||||
|
queryName,
|
||||||
|
{ block, id },
|
||||||
|
rawJson,
|
||||||
|
timeDiff
|
||||||
|
);
|
||||||
|
|
||||||
|
if (config.watcher.verifyState) {
|
||||||
|
const ipldDiff = await checkGQLEntityInIPLDState(ipldStateByBlock, entityName, result[queryName], id, rawJson, config.watcher.skipFields);
|
||||||
|
|
||||||
|
if (ipldDiff) {
|
||||||
|
log('Results mismatch for IPLD state:', ipldDiff);
|
||||||
|
diffFound = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (diff) {
|
||||||
|
resultDiff = diff;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (updatedEntities.has(entityName)) {
|
||||||
|
let result;
|
||||||
|
|
||||||
|
({ diff: resultDiff, result1: result } = await compareQuery(
|
||||||
|
clients,
|
||||||
|
queryName,
|
||||||
|
{ block },
|
||||||
|
rawJson,
|
||||||
|
timeDiff
|
||||||
|
));
|
||||||
|
|
||||||
|
if (config.watcher.verifyState) {
|
||||||
|
const ipldDiff = await checkGQLEntitiesInIPLDState(ipldStateByBlock, entityName, result[queryName], rawJson, config.watcher.skipFields);
|
||||||
|
|
||||||
|
if (ipldDiff) {
|
||||||
|
log('Results mismatch for IPLD state:', ipldDiff);
|
||||||
|
diffFound = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resultDiff) {
|
||||||
|
log('Results mismatch:', resultDiff);
|
||||||
|
diffFound = true;
|
||||||
|
} else {
|
||||||
|
log('Results match.');
|
||||||
|
}
|
||||||
|
} catch (err: any) {
|
||||||
|
log('Error:', err.message);
|
||||||
|
log('Error:', JSON.stringify(err, null, 2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set delay between requests for a block.
|
||||||
|
blockDelay = wait(config.queries.blockDelayInMs || 0);
|
||||||
|
|
||||||
|
console.timeEnd(`time:compare-block-${blockNumber}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set delay between requests for a block.
|
|
||||||
blockDelay = wait(config.queries.blockDelayInMs || 0);
|
|
||||||
|
|
||||||
console.timeEnd(`time:compare-block-${blockNumber}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (diffFound) {
|
if (diffFound) {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user