cerc-io/watcher-ts
16
1
Fork 0
mirror of https://github.com/cerc-io/watcher-ts synced 2024-11-19 12:26:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

CLI to compare eden-watcher entities and fix mapping code (#149)

Browse Source 
* Make sumStaked variable local in eden network mapping

* Implement compare CLI to fetch and query by ids

* Set filterLogs to true for eden-watcher

* Use varchar for bigint array type in eden-watcher

* Store subgraph entities by id in IPLD state

* Store bigint vales as string in IPLD state

* Update eden watcher hook to store single Block entity in IPLD checkpoint

* Fix entity enum type property

* Fix parsing big numbers in event params

* Fix event bigint params parsing in all watchers

* Set default limit to query result and process block after events
This commit is contained in:
nikugogoi 2022-08-08 15:58:13 +05:30 committed by GitHub
parent 6990eb892a
commit 1a903fccc6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 366 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
packages/codegen/src/templates/indexer-template.handlebars
View File

@ -52,6 +52,7 @@ import { {{subgraphEntity.className}} } from './entity/{{subgraphEntity.classNam
{{/each}}
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
{{#each contracts as | contract |}}
const KIND_{{capitalize contract.contractName}} = '{{contract.contractKind}}';
@ -177,8 +178,8 @@ export class Indexer implements IPLDIndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSONbig.parse(event.eventInfo);
const { tx, eventSignature } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -288,7 +289,7 @@ export class Indexer implements IPLDIndexerInterface {
{{/if}}
await this._db.{{query.saveQueryName}}({ blockHash, blockNumber, contractAddress
{{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbig.stringify(result.proof) });
{{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbigNative.stringify(result.proof) });
{{#if query.stateVariableType}}
{{#if (compare query.stateVariableType 'Mapping')}}
@ -441,13 +442,15 @@ export class Indexer implements IPLDIndexerInterface {
async processBlock (blockHash: string, blockNumber: number): Promise<void> {
// Call a function to create initial state for contracts.
await this._baseIndexer.createInit(this, blockHash, blockNumber);
{{#if (subgraphPath)}}
// Call subgraph handler for block.
await this._graphWatcher.handleBlock(blockHash);
{{/if}}
}
{{#if (subgraphPath)}}
async processBlockAfterEvents (blockHash: string): Promise<void> {
// Call subgraph handler for block.
await this._graphWatcher.handleBlock(blockHash);
}
{{/if}}
parseEventNameAndArgs (kind: string, logObj: any): any {
const { topics, data } = logObj;
@ -770,10 +773,10 @@ export class Indexer implements IPLDIndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

8
packages/eden-watcher/environments/local.toml
View File

@ -10,13 +10,13 @@
checkpointInterval = 2000
# IPFS API address (can be taken from the output on running the IPFS daemon).
ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
# ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"
subgraphPath = "../graph-node/test/subgraph/eden"
wasmRestartBlocksInterval = 20
# Boolean to filter logs by contract.
filterLogs = false
filterLogs = true
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
@ -38,8 +38,8 @@
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
rpcProviderEndpoint = "http://127.0.0.1:8081"
gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
rpcProviderEndpoint = "http://127.0.0.1:8082"
blockDelayInMilliSecs = 2000
[upstream.cache]

3
packages/eden-watcher/src/entity/Network.ts
View File

@ -35,6 +35,7 @@ export class Network {
@Column('numeric', { transformer: bigintTransformer })
totalStaked!: bigint;
@Column('numeric', { transformer: bigintArrayTransformer, array: true })
// https://github.com/brianc/node-postgres/issues/1943#issuecomment-520500053
@Column('varchar', { transformer: bigintArrayTransformer, array: true })
stakedPercentiles!: bigint[];
}

4
packages/eden-watcher/src/entity/ProducerSetChange.ts
View File

@ -6,8 +6,8 @@ import { Entity, PrimaryColumn, Column } from 'typeorm';
import { bigintTransformer } from '@vulcanize/util';
enum ProducerSetChangeType {
Added,
Removed
Added = 'Added',
Removed = 'Removed'
}
@Entity()

4
packages/eden-watcher/src/gql/queries/account.gql
View File

@ -1,5 +1,5 @@
query account($id: String!, $blockHash: Bytes!){
account(id: $id, block: { hash: $blockHash }){
query account($id: String!, $block: Block_height){
account(id: $id, block: $block){
id
totalClaimed
totalSlashed

4
packages/eden-watcher/src/gql/queries/block.gql
View File

@ -1,5 +1,5 @@
query block($id: String!, $blockHash: Bytes!){
block(id: $id, block: { hash: $blockHash }){
query block($id: String!, $block: Block_height){
block(id: $id, block: $block){
id
fromActiveProducer
hash

4
packages/eden-watcher/src/gql/queries/claim.gql
View File

@ -1,5 +1,5 @@
query claim($id: String!, $blockHash: Bytes!){
claim(id: $id, block: { hash: $blockHash }){
query claim($id: String!, $block: Block_height){
claim(id: $id, block: $block){
id
timestamp
index

4
packages/eden-watcher/src/gql/queries/distribution.gql
View File

@ -1,5 +1,5 @@
query distribution($id: String!, $blockHash: Bytes!){
distribution(id: $id, block: { hash: $blockHash }){
query distribution($id: String!, $block: Block_height){
distribution(id: $id, block: $block){
id
distributor{
id

4
packages/eden-watcher/src/gql/queries/distributor.gql
View File

@ -1,5 +1,5 @@
query distributor($id: String!, $blockHash: Bytes!){
distributor(id: $id, block: { hash: $blockHash }){
query distributor($id: String!, $block: Block_height){
distributor(id: $id, block: $block){
id
currentDistribution{
id

4
packages/eden-watcher/src/gql/queries/epoch.gql
View File

@ -1,5 +1,5 @@
query epoch($id: String!, $blockHash: Bytes!){
epoch(id: $id, block: { hash: $blockHash }){
query epoch($id: String!, $block: Block_height){
epoch(id: $id, block: $block){
id
finalized
epochNumber

4
packages/eden-watcher/src/gql/queries/network.gql
View File

@ -1,5 +1,5 @@
query network($id: String!, $blockHash: Bytes!){
network(id: $id, block: { hash: $blockHash }){
query network($id: String!, $block: Block_height){
network(id: $id, block: $block){
id
slot0{
id

4
packages/eden-watcher/src/gql/queries/producer.gql
View File

@ -1,5 +1,5 @@
query producer($id: String!, $blockHash: Bytes!){
producer(id: $id, block: { hash: $blockHash }){
query producer($id: String!, $block: Block_height){
producer(id: $id, block: $block){
id
active
rewardCollector

4
packages/eden-watcher/src/gql/queries/producerEpoch.gql
View File

@ -1,5 +1,5 @@
query producerEpoch($id: String!, $blockHash: Bytes!){
producerEpoch(id: $id, block: { hash: $blockHash }){
query producerEpoch($id: String!, $block: Block_height){
producerEpoch(id: $id, block: $block){
id
address
epoch{

4
packages/eden-watcher/src/gql/queries/producerRewardCollectorChange.gql
View File

@ -1,5 +1,5 @@
query producerRewardCollectorChange($id: String!, $blockHash: Bytes!){
producerRewardCollectorChange(id: $id, block: { hash: $blockHash }){
query producerRewardCollectorChange($id: String!, $block: Block_height){
producerRewardCollectorChange(id: $id, block: $block){
id
blockNumber
producer

4
packages/eden-watcher/src/gql/queries/producerSet.gql
View File

@ -1,5 +1,5 @@
query producerSet($id: String!, $blockHash: Bytes!){
producerSet(id: $id, block: { hash: $blockHash }){
query producerSet($id: String!, $block: Block_height){
producerSet(id: $id, block: $block){
id
producers{
id

4
packages/eden-watcher/src/gql/queries/producerSetChange.gql
View File

@ -1,5 +1,5 @@
query producerSetChange($id: String!, $blockHash: Bytes!){
producerSetChange(id: $id, block: { hash: $blockHash }){
query producerSetChange($id: String!, $block: Block_height){
producerSetChange(id: $id, block: $block){
id
blockNumber
producer

4
packages/eden-watcher/src/gql/queries/rewardSchedule.gql
View File

@ -1,5 +1,5 @@
query rewardSchedule($id: String!, $blockHash: Bytes!){
rewardSchedule(id: $id, block: { hash: $blockHash }){
query rewardSchedule($id: String!, $block: Block_height){
rewardSchedule(id: $id, block: $block){
id
rewardScheduleEntries{
id

4
packages/eden-watcher/src/gql/queries/rewardScheduleEntry.gql
View File

@ -1,5 +1,5 @@
query rewardScheduleEntry($id: String!, $blockHash: Bytes!){
rewardScheduleEntry(id: $id, block: { hash: $blockHash }){
query rewardScheduleEntry($id: String!, $block: Block_height){
rewardScheduleEntry(id: $id, block: $block){
id
startTime
epochDuration

4
packages/eden-watcher/src/gql/queries/slash.gql
View File

@ -1,5 +1,5 @@
query slash($id: String!, $blockHash: Bytes!){
slash(id: $id, block: { hash: $blockHash }){
query slash($id: String!, $block: Block_height){
slash(id: $id, block: $block){
id
timestamp
account{

4
packages/eden-watcher/src/gql/queries/slot.gql
View File

@ -1,5 +1,5 @@
query slot($id: String!, $blockHash: Bytes!){
slot(id: $id, block: { hash: $blockHash }){
query slot($id: String!, $block: Block_height){
slot(id: $id, block: $block){
id
owner
delegate

4
packages/eden-watcher/src/gql/queries/slotClaim.gql
View File

@ -1,5 +1,5 @@
query slotClaim($id: String!, $blockHash: Bytes!){
slotClaim(id: $id, block: { hash: $blockHash }){
query slotClaim($id: String!, $block: Block_height){
slotClaim(id: $id, block: $block){
id
slot{
id

4
packages/eden-watcher/src/gql/queries/staker.gql
View File

@ -1,5 +1,5 @@
query staker($id: String!, $blockHash: Bytes!){
staker(id: $id, block: { hash: $blockHash }){
query staker($id: String!, $block: Block_height){
staker(id: $id, block: $block){
id
staked
rank

58
packages/eden-watcher/src/hooks.ts
View File

@ -2,7 +2,10 @@
// Copyright 2021 Vulcanize, Inc.
//
import { IPLDBlockInterface, StateKind } from '@vulcanize/util';
import assert from 'assert';
import * as codec from '@ipld/dag-cbor';
import _ from 'lodash';
import { Indexer, ResultEvent } from './indexer';
@ -51,9 +54,60 @@ export async function createStateCheckpoint (indexer: Indexer, contractAddress:
assert(blockHash);
assert(contractAddress);
// Use indexer.createStateCheckpoint() method to create a custom checkpoint.
// TODO: Pass blockProgress instead of blockHash to hook method.
const block = await indexer.getBlockProgress(blockHash);
assert(block);
return false;
// Fetch the latest 'checkpoint' | 'init' for the contract to fetch diffs after it.
let prevNonDiffBlock: IPLDBlockInterface;
let getDiffBlockNumber: number;
const checkpointBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Checkpoint, block.blockNumber);
if (checkpointBlock) {
const checkpointBlockNumber = checkpointBlock.block.blockNumber;
prevNonDiffBlock = checkpointBlock;
getDiffBlockNumber = checkpointBlockNumber;
// Update IPLD status map with the latest checkpoint info.
// Essential while importing state as checkpoint at the snapshot block is added by import-state CLI.
// (job-runner won't have the updated ipld status)
indexer.updateIPLDStatusMap(contractAddress, { checkpoint: checkpointBlockNumber });
} else {
// There should be an initial state at least.
const initBlock = await indexer.getLatestIPLDBlock(contractAddress, StateKind.Init);
assert(initBlock, 'No initial state found');
prevNonDiffBlock = initBlock;
// Take block number previous to initial state block to include any diff state at that block.
getDiffBlockNumber = initBlock.block.blockNumber - 1;
}
// Fetching all diff blocks after the latest 'checkpoint' | 'init'.
const diffBlocks = await indexer.getDiffIPLDBlocksByBlocknumber(contractAddress, getDiffBlockNumber);
const prevNonDiffBlockData = codec.decode(Buffer.from(prevNonDiffBlock.data)) as any;
const data = {
state: prevNonDiffBlockData.state
};
// Merge all diff blocks after previous checkpoint.
for (const diffBlock of diffBlocks) {
const diff = codec.decode(Buffer.from(diffBlock.data)) as any;
data.state = _.merge(data.state, diff.state);
}
// Check if Block entity exists.
if (data.state.Block) {
// Store only block entity at checkpoint height instead of all entities.
data.state.Block = {
[blockHash]: data.state.Block[blockHash]
};
}
await indexer.createStateCheckpoint(contractAddress, blockHash, data);
return true;
}
/**

23
packages/eden-watcher/src/indexer.ts
View File

@ -57,6 +57,7 @@ import { Account } from './entity/Account';
import { Slash } from './entity/Slash';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
const KIND_EDENNETWORK = 'EdenNetwork';
const KIND_MERKLEDISTRIBUTOR = 'EdenNetworkDistribution';
@ -176,8 +177,8 @@ export class Indexer implements IPLDIndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSONbig.parse(event.eventInfo);
const { tx, eventSignature } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -287,6 +288,10 @@ export class Indexer implements IPLDIndexerInterface {
return this._baseIndexer.getIPLDBlockByCid(cid);
}
async getDiffIPLDBlocksByBlocknumber (contractAddress: string, blockNumber: number): Promise<IPLDBlock[]> {
return this._db.getDiffIPLDBlocksByBlocknumber(contractAddress, blockNumber);
}
getIPLDData (ipldBlock: IPLDBlock): any {
return this._baseIndexer.getIPLDData(ipldBlock);
}
@ -370,13 +375,15 @@ export class Indexer implements IPLDIndexerInterface {
await this._baseIndexer.createInit(this, blockHash, blockNumber);
console.timeEnd('time:indexer#processBlock-init_state');
}
console.time('time:indexer#processBlock-mapping_code');
async processBlockAfterEvents (blockHash: string): Promise<void> {
console.time('time:indexer#processBlockAfterEvents-mapping_code');
// Call subgraph handler for block.
await this._graphWatcher.handleBlock(blockHash);
console.timeEnd('time:indexer#processBlock-mapping_code');
console.timeEnd('time:indexer#processBlockAfterEvents-mapping_code');
}
parseEventNameAndArgs (kind: string, logObj: any): any {
@ -1013,10 +1020,10 @@ export class Indexer implements IPLDIndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

17
packages/erc20-watcher/src/indexer.ts
View File

@ -23,6 +23,7 @@ import { BlockProgress } from './entity/BlockProgress';
import { Contract } from './entity/Contract';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
const ETH_CALL_MODE = 'eth_call';
@ -108,7 +109,7 @@ export class Indexer implements IndexerInterface {
}
// https://github.com/GoogleChromeLabs/jsbi/issues/30#issuecomment-521460510
log(JSONbig.stringify(result, null, 2));
log(JSONbigNative.stringify(result, null, 2));
return result;
}
@ -142,10 +143,10 @@ export class Indexer implements IndexerInterface {
result = await this._baseIndexer.getStorageValue(this._storageLayout, blockHash, token, '_balances', owner);
}
log(JSONbig.stringify(result, null, 2));
log(JSONbigNative.stringify(result, null, 2));
const { value, proof } = result;
await this._db.saveBalance({ blockHash, blockNumber, token, owner, value: BigInt(value), proof: JSONbig.stringify(proof) });
await this._db.saveBalance({ blockHash, blockNumber, token, owner, value: BigInt(value), proof: JSONbigNative.stringify(proof) });
return result;
}
@ -180,7 +181,7 @@ export class Indexer implements IndexerInterface {
// log(JSONbig.stringify(result, null, 2));
const { value, proof } = result;
await this._db.saveAllowance({ blockHash, blockNumber, token, owner, spender, value: BigInt(value), proof: JSONbig.stringify(proof) });
await this._db.saveAllowance({ blockHash, blockNumber, token, owner, spender, value: BigInt(value), proof: JSONbigNative.stringify(proof) });
return result;
}
@ -406,10 +407,10 @@ export class Indexer implements IndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

41
packages/erc721-watcher/src/indexer.ts
View File

@ -42,6 +42,7 @@ import { IPLDBlock } from './entity/IPLDBlock';
import { TransferCount } from './entity/TransferCount';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
const KIND_ERC721 = 'ERC721';
@ -134,8 +135,8 @@ export class Indexer implements IPLDIndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSONbig.parse(event.eventInfo);
const { tx, eventSignature } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -211,7 +212,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveSupportsInterface({ blockHash, blockNumber, contractAddress, interfaceId, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveSupportsInterface({ blockHash, blockNumber, contractAddress, interfaceId, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -242,7 +243,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveBalanceOf({ blockHash, blockNumber, contractAddress, owner, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveBalanceOf({ blockHash, blockNumber, contractAddress, owner, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -271,7 +272,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveOwnerOf({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveOwnerOf({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -300,7 +301,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveGetApproved({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveGetApproved({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -329,7 +330,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveIsApprovedForAll({ blockHash, blockNumber, contractAddress, owner, operator, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveIsApprovedForAll({ blockHash, blockNumber, contractAddress, owner, operator, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -358,7 +359,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -387,7 +388,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -416,7 +417,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveTokenURI({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveTokenURI({ blockHash, blockNumber, contractAddress, tokenId, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -477,7 +478,7 @@ export class Indexer implements IPLDIndexerInterface {
'_name'
);
await this._db._saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveName({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForElementaryType({}, '_name', result.value.toString());
@ -513,7 +514,7 @@ export class Indexer implements IPLDIndexerInterface {
'_symbol'
);
await this._db._saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveSymbol({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForElementaryType({}, '_symbol', result.value.toString());
@ -550,7 +551,7 @@ export class Indexer implements IPLDIndexerInterface {
key0
);
await this._db._saveOwners({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveOwners({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForMappingType({}, '_owners', [key0.toString()], result.value.toString());
@ -587,7 +588,7 @@ export class Indexer implements IPLDIndexerInterface {
key0
);
await this._db._saveBalances({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveBalances({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForMappingType({}, '_balances', [key0.toString()], result.value.toString());
@ -624,7 +625,7 @@ export class Indexer implements IPLDIndexerInterface {
key0
);
await this._db._saveTokenApprovals({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveTokenApprovals({ blockHash, blockNumber, contractAddress, key0, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForMappingType({}, '_tokenApprovals', [key0.toString()], result.value.toString());
@ -662,7 +663,7 @@ export class Indexer implements IPLDIndexerInterface {
key1
);
await this._db._saveOperatorApprovals({ blockHash, blockNumber, contractAddress, key0, key1, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveOperatorApprovals({ blockHash, blockNumber, contractAddress, key0, key1, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForMappingType({}, '_operatorApprovals', [key0.toString(), key1.toString()], result.value.toString());
@ -1057,10 +1058,10 @@ export class Indexer implements IPLDIndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

43
packages/graph-node/README.md
View File

@ -55,3 +55,46 @@
```
* The program will exit with code `1` if the query results are not equal.
* For comparing queries in a range of blocks:
* Config file should have the names of queries to be fired.
```toml
[queries]
queryDir = "../graph-test-watcher/src/gql/queries"
names = [
"author",
"blog"
]
```
* Run the CLI:
```bash
./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10
```
* For comparing entities after fetching ids from one of the endpoints and then querying individually by ids:
* Set the `idsEndpoint` to choose which endpoint the ids should be fetched from.
```toml
[endpoints]
gqlEndpoint1 = "http://localhost:8000/subgraphs/name/example1"
gqlEndpoint2 = "http://localhost:3008/graphql"
[queries]
queryDir = "../graph-test-watcher/src/gql/queries"
names = [
"author",
"blog"
]
idsEndpoint = "gqlEndpoint1"
```
* Run the CLI with `fetch-ids` flag set to true:\
```bash
./bin/compare-blocks --config-file environments/compare-cli-config.toml --start-block 1 --end-block 10 --fetch-ids
```

1
packages/graph-node/environments/compare-cli-config.toml
View File

@ -5,6 +5,7 @@
[queries]
queryDir = "../graph-test-watcher/src/gql/queries"
names = []
idsEndpoint = "gqlEndpoint1"
[cache]
endpoint = "gqlEndpoint1"

2
packages/graph-node/package.json
View File

@ -10,6 +10,7 @@
"@types/chai": "^4.2.18",
"@types/chai-spies": "^1.0.3",
"@types/js-yaml": "^4.0.4",
"@types/pluralize": "^0.0.29",
"@typescript-eslint/eslint-plugin": "^4.25.0",
"@typescript-eslint/parser": "^4.25.0",
"chai": "^4.3.4",
@ -57,6 +58,7 @@
"js-yaml": "^4.1.0",
"json-bigint": "^1.0.0",
"json-diff": "^0.5.4",
"pluralize": "^8.0.0",
"reflect-metadata": "^0.1.13",
"toml": "^3.0.0",
"typeorm": "^0.2.32",

42
packages/graph-node/src/cli/compare/client.ts
View File

@ -5,6 +5,7 @@
import assert from 'assert';
import fs from 'fs';
import path from 'path';
import pluralize from 'pluralize';
import { gql } from '@apollo/client/core';
import { GraphQLClient, Config } from '@vulcanize/ipld-eth-client';
@ -15,6 +16,7 @@ export class Client {
_graphqlClient: GraphQLClient;
_queryDir: string;
_cache: Cache | undefined;
_endpoint: string;
constructor (config: Config, queryDir: string) {
this._config = config;
@ -22,16 +24,56 @@ export class Client {
const { gqlEndpoint, cache } = config;
assert(gqlEndpoint, 'Missing gql endpoint');
this._endpoint = gqlEndpoint;
this._graphqlClient = new GraphQLClient(config);
this._cache = cache;
}
get endpoint () {
return this._endpoint;
}
async getResult (queryName: string, params: { [key: string]: any }): Promise<any> {
return this._getCachedOrFetch(queryName, params);
}
async getIds (queryName: string, blockNumber: number): Promise<string[]> {
const keyObj = { queryName, blockNumber };
if (this._cache) {
const [value, found] = await this._cache.get(keyObj) || [undefined, false];
if (found) {
return value;
}
}
const result = await this._graphqlClient.query(
gql(
`query($blockNumber: Int){
${pluralize(queryName)}(
block: { number: $blockNumber }
) {
id
}
}`
),
{
blockNumber
}
);
const ids = result[pluralize(queryName)].map((entity: { id: string }) => entity.id);
// Cache the result and return it, if cache is enabled.
if (this._cache) {
await this._cache.put(keyObj, ids);
}
return ids;
}
async _getCachedOrFetch (queryName: string, params: {[key: string]: any}): Promise<any> {
const keyObj = {
queryName,

55
packages/graph-node/src/cli/compare/compare-blocks.ts
View File

@ -5,8 +5,10 @@
import yargs from 'yargs';
import 'reflect-metadata';
import debug from 'debug';
import assert from 'assert';
import { compareQuery, Config, getClients, getConfig } from './utils';
import { Client } from './client';
const log = debug('vulcanize:compare-blocks');
@ -40,12 +42,17 @@ export const main = async (): Promise<void> => {
type: 'boolean',
describe: 'Whether to print out raw diff object',
default: false
},
fetchIds: {
type: 'boolean',
describe: 'Fetch ids and compare multiple entities',
default: false
}
}).argv;
const config: Config = await getConfig(argv.configFile);
const { startBlock, endBlock, rawJson, queryDir } = argv;
const { startBlock, endBlock, rawJson, queryDir, fetchIds } = argv;
const queryNames = config.queries.names;
let diffFound = false;
@ -58,13 +65,27 @@ export const main = async (): Promise<void> => {
for (const queryName of queryNames) {
try {
log(`At block ${blockNumber} for query ${queryName}:`);
const resultDiff = await compareQuery(clients, queryName, { block }, rawJson);
if (resultDiff) {
diffFound = true;
log('Results mismatch:', resultDiff);
if (fetchIds) {
const { idsEndpoint } = config.queries;
assert(idsEndpoint, 'Specify endpoint for fetching ids when fetchId is true');
const client = Object.values(clients).find(client => client.endpoint === config.endpoints[idsEndpoint]);
assert(client);
const ids = await client.getIds(queryName, blockNumber);
for (const id of ids) {
const isDiff = await compareAndLog(clients, queryName, { block, id }, rawJson);
if (isDiff) {
diffFound = isDiff;
}
}
} else {
log('Results match.');
const isDiff = await compareAndLog(clients, queryName, { block }, rawJson);
if (isDiff) {
diffFound = isDiff;
}
}
} catch (err: any) {
log('Error:', err.message);
@ -78,3 +99,25 @@ export const main = async (): Promise<void> => {
process.exit(1);
}
};
const compareAndLog = async (
clients: { client1: Client, client2: Client },
queryName: string,
params: { [key: string]: any },
rawJson: boolean
): Promise<boolean> => {
const resultDiff = await compareQuery(
clients,
queryName,
params,
rawJson
);
if (resultDiff) {
log('Results mismatch:', resultDiff);
return true;
}
log('Results match.');
return false;
};

3
packages/graph-node/src/cli/compare/compare-entity.ts
View File

@ -54,11 +54,10 @@ export const main = async (): Promise<void> => {
const queryName = argv.queryName;
const id = argv.entityId;
const blockHash = argv.blockHash;
const block = {
number: argv.blockNumber,
hash: blockHash
hash: argv.blockHash
};
const clients = await getClients(config, argv.queryDir);

3
packages/graph-node/src/cli/compare/utils.ts
View File

@ -21,13 +21,14 @@ interface EndpointConfig {
interface QueryConfig {
queryDir: string;
names: string[];
idsEndpoint: keyof EndpointConfig;
}
export interface Config {
endpoints: EndpointConfig;
queries: QueryConfig;
cache: {
endpoint: string;
endpoint: keyof EndpointConfig;
config: CacheConfig;
}
}

8
packages/graph-node/src/database.ts
View File

@ -17,6 +17,8 @@ import {
import { Block, fromEntityValue, toEntityValue } from './utils';
const DEFAULT_LIMIT = 100;
export class Database {
_config: ConnectionOptions
_conn!: Connection
@ -123,7 +125,8 @@ export class Database {
if (isArray) {
selectQueryBuilder = selectQueryBuilder.distinctOn(['entity.id'])
.orderBy('entity.id');
.orderBy('entity.id')
.limit(DEFAULT_LIMIT);
} else {
selectQueryBuilder = selectQueryBuilder.limit(1);
}
@ -132,7 +135,8 @@ export class Database {
// For one to many relational field.
selectQueryBuilder = selectQueryBuilder.where('entity.id IN (:...ids)', { ids: entityData[field] })
.distinctOn(['entity.id'])
.orderBy('entity.id');
.orderBy('entity.id')
.limit(DEFAULT_LIMIT);
// Subquery example if distinctOn is not performant.
//

9
packages/graph-node/src/loader.ts
View File

@ -29,6 +29,8 @@ import {
} from './utils';
import { Database } from './database';
const JSONbigString = JSONbig({ storeAsString: true });
// Endianness of BN used in bigInt store host API.
// Negative bigInt is being stored in wasm in 2's compliment, 'le' representation.
// (for eg. bigInt.fromString(negativeI32Value))
@ -104,7 +106,12 @@ export const instantiate = async (
// JSON stringify and parse data for handling unknown types when encoding.
// For example, decimal.js values are converted to string in the diff data.
diffData.state[entityName] = JSONbig.parse(JSONbig.stringify(dbData));
diffData.state[entityName] = {
// Using JSONbigString to store bigints as string values to be encoded by IPLD dag-cbor.
// TODO: Parse and store as native bigint by using Type encoders in IPLD dag-cbor encode.
// https://github.com/rvagg/cborg#type-encoders
[dbData.id]: JSONbigString.parse(JSONbigString.stringify(dbData))
};
// Create an auto-diff.
assert(indexer.createDiffStaged);

2
packages/graph-node/src/watcher.ts
View File

@ -123,7 +123,7 @@ export class GraphWatcher {
async handleEvent (eventData: any) {
const { contract, event, eventSignature, block, tx: { hash: txHash }, eventIndex } = eventData;
if (!this._context.block) {
if (!this._context.block || this._context.block.blockHash !== block.hash) {
this._context.block = await getFullBlock(this._ethClient, this._ethProvider, block.hash);
}

BIN
packages/graph-node/test/subgraph/eden/EdenNetwork/EdenNetwork.wasm
View File

Binary file not shown.

BIN
packages/graph-node/test/subgraph/eden/EdenNetworkGovernance/EdenNetworkGovernance.wasm
View File

Binary file not shown.

19
packages/graph-test-watcher/src/indexer.ts
View File

@ -46,6 +46,7 @@ import { Blog } from './entity/Blog';
import { Category } from './entity/Category';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
const KIND_EXAMPLE1 = 'Example1';
const KIND_FACTORY = 'Factory';
@ -167,8 +168,8 @@ export class Indexer implements IPLDIndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSONbig.parse(event.eventInfo);
const { tx, eventSignature } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -244,7 +245,7 @@ export class Indexer implements IPLDIndexerInterface {
const result: ValueResult = { value };
await this._db.saveGetMethod({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db.saveGetMethod({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) });
return result;
}
@ -275,7 +276,7 @@ export class Indexer implements IPLDIndexerInterface {
'_test'
);
await this._db._saveTest({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbig.stringify(result.proof) });
await this._db._saveTest({ blockHash, blockNumber, contractAddress, value: result.value, proof: JSONbigNative.stringify(result.proof) });
if (diff) {
const stateUpdate = updateStateForElementaryType({}, '_test', result.value.toString());
@ -406,7 +407,9 @@ export class Indexer implements IPLDIndexerInterface {
async processBlock (blockHash: string, blockNumber: number): Promise<void> {
// Call a function to create initial state for contracts.
await this._baseIndexer.createInit(this, blockHash, blockNumber);
}
async processBlockAfterEvents (blockHash: string): Promise<void> {
// Call subgraph handler for block.
await this._graphWatcher.handleBlock(blockHash);
}
@ -749,10 +752,10 @@ export class Indexer implements IPLDIndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

3
packages/lighthouse-watcher/src/indexer.ts
View File

@ -15,6 +15,7 @@ import lighthouseABI from './abi/Lighthouse.json';
export const UNKNOWN_EVENT_NAME = '__unknown__';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
export type ResultEvent = {
block: any;
@ -160,7 +161,7 @@ export class Indexer {
...eventInfo
},
proof: {
data: JSONbig.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

15
packages/mobymask-watcher/src/indexer.ts
View File

@ -47,6 +47,7 @@ import { _Owner } from './entity/_Owner';
import { MultiNonce } from './entity/MultiNonce';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
export const KIND_PHISHERREGISTRY = 'PhisherRegistry';
@ -139,8 +140,8 @@ export class Indexer implements IPLDIndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSONbig.parse(event.eventInfo);
const { tx, eventSignature } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventSignature } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -395,7 +396,7 @@ export class Indexer implements IPLDIndexerInterface {
contractAddress,
...mappingKeys,
value: result.value,
proof: result.proof ? JSONbig.stringify(result.proof) : null
proof: result.proof ? JSONbigNative.stringify(result.proof) : null
} as any;
}
@ -801,10 +802,10 @@ export class Indexer implements IPLDIndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

4
packages/solidity-mapper/src/storage.test.ts
View File

@ -1073,10 +1073,10 @@ describe('Get value from storage', () => {
before(async () => {
({ contract: testNestedArrays, storageLayout } = contracts.TestNestedArrays);
const transactions = [];
const transactions: Array<ContractTransaction> = [];
const addresses = generateDummyAddresses(7);
const transactionPromises = [];
const transactionPromises: Promise<ContractTransaction>[] = [];
// Set value for nestedStructArray.
for (let i = 0; i < 5; i++) {

11
packages/uni-info-watcher/src/indexer.ts
View File

@ -36,6 +36,7 @@ import { Tick } from './entity/Tick';
const SYNC_DELTA = 5;
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
export { OrderDirection, BlockHeight };
@ -69,8 +70,8 @@ export class Indexer implements IndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSON.parse(event.eventInfo);
const { tx, eventIndex } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx, eventIndex } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -394,9 +395,9 @@ export class Indexer implements IndexerInterface {
txHash: tx.hash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify(proof)
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify(proof)
});
}

13
packages/uni-watcher/src/indexer.ts
View File

@ -22,6 +22,7 @@ import { abi as nfpmABI, storageLayout as nfpmStorageLayout } from './artifacts/
import poolABI from './artifacts/pool.json';
const log = debug('vulcanize:indexer');
const JSONbigNative = JSONbig({ useNativeBigInt: true });
type ResultEvent = {
block: any;
@ -68,8 +69,8 @@ export class Indexer implements IndexerInterface {
getResultEvent (event: Event): ResultEvent {
const block = event.block;
const eventFields = JSON.parse(event.eventInfo);
const { tx } = JSON.parse(event.extraInfo);
const eventFields = JSONbigNative.parse(event.eventInfo);
const { tx } = JSONbigNative.parse(event.extraInfo);
return {
block: {
@ -521,10 +522,10 @@ export class Indexer implements IndexerInterface {
txHash,
contract,
eventName,
eventInfo: JSONbig.stringify(eventInfo),
extraInfo: JSONbig.stringify(extraInfo),
proof: JSONbig.stringify({
data: JSONbig.stringify({
eventInfo: JSONbigNative.stringify(eventInfo),
extraInfo: JSONbigNative.stringify(extraInfo),
proof: JSONbigNative.stringify({
data: JSONbigNative.stringify({
blockHash,
receiptCID,
log: {

4
packages/util/src/common.ts
View File

@ -191,4 +191,8 @@ export const processBatchEvents = async (indexer: IndexerInterface, block: Block
console.timeEnd('time:common#processBacthEvents-processing_events_batch');
}
if (indexer.processBlockAfterEvents) {
await indexer.processBlockAfterEvents(block.blockHash);
}
};

1
packages/util/src/types.ts
View File

@ -107,6 +107,7 @@ export interface IndexerInterface {
processInitialState?: (contractAddress: string, blockHash: string) => Promise<any>
processStateCheckpoint?: (contractAddress: string, blockHash: string) => Promise<boolean>
processBlock?: (blockHash: string, blockNumber: number) => Promise<void>
processBlockAfterEvents?: (blockHash: string) => Promise<void>
}
export interface IPLDIndexerInterface extends IndexerInterface {

10
yarn.lock
View File

@ -2581,6 +2581,11 @@
dependencies:
"@types/node" "*"
"@types/pluralize@^0.0.29":
version "0.0.29"
resolved "https://registry.yarnpkg.com/@types/pluralize/-/pluralize-0.0.29.tgz#6ffa33ed1fc8813c469b859681d09707eb40d03c"
integrity sha512-BYOID+l2Aco2nBik+iYS4SZX0Lf20KPILP5RGmM1IgzdwNdTs0eebiFriOPcej1sX9mLnSoiNte5zcFxssgpGA==
"@types/prettier@^2.1.1":
version "2.2.3"
resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.2.3.tgz#ef65165aea2924c9359205bf748865b8881753c0"
@ -11561,6 +11566,11 @@ please-upgrade-node@^3.2.0:
dependencies:
semver-compare "^1.0.0"
pluralize@^8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==
posix-character-classes@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"