diff --git a/packages/eden-watcher/src/indexer.ts b/packages/eden-watcher/src/indexer.ts index acd85d9c..b2bc15e2 100644 --- a/packages/eden-watcher/src/indexer.ts +++ b/packages/eden-watcher/src/indexer.ts @@ -31,6 +31,22 @@ import MerkleDistributorArtifacts from './artifacts/MerkleDistributor.json'; import DistributorGovernanceArtifacts from './artifacts/DistributorGovernance.json'; import { createInitialCheckpoint, handleEvent, createStateDiff, createStateCheckpoint } from './hooks'; import { IPFSClient } from './ipfs'; +import { ProducerSet } from './entity/ProducerSet'; +import { Producer } from './entity/Producer'; +import { RewardSchedule } from './entity/RewardSchedule'; +import { RewardScheduleEntry } from './entity/RewardScheduleEntry'; +import { Network } from './entity/Network'; +import { Staker } from './entity/Staker'; +import { ProducerEpoch } from './entity/ProducerEpoch'; +import { Epoch } from './entity/Epoch'; +import { Block } from './entity/Block'; +import { SlotClaim } from './entity/SlotClaim'; +import { Slot } from './entity/Slot'; +import { Distributor } from './entity/Distributor'; +import { Distribution } from './entity/Distribution'; +import { Claim } from './entity/Claim'; +import { Account } from './entity/Account'; +import { Slash } from './entity/Slash'; const log = debug('vulcanize:indexer'); @@ -117,6 +133,8 @@ export class Indexer implements IndexerInterface { _ipfsClient: IPFSClient + _relationsMap: Map + constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher: GraphWatcher) { assert(db); assert(ethClient); @@ -160,6 +178,9 @@ export class Indexer implements IndexerInterface { this._contractMap.set(KIND_DISTRIBUTORGOVERNANCE, new ethers.utils.Interface(DistributorGovernanceABI)); this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); + + this._relationsMap = new Map(); + this._populateRelationsMap(); } getResultEvent (event: Event): ResultEvent { @@ -527,7 +548,11 @@ export class Indexer implements IndexerInterface { } async getSubgraphEntity (entity: new () => Entity, id: string, blockHash: string): Promise { - return this._graphWatcher.getEntity(entity, id, blockHash); + const relations = this._relationsMap.get(entity) || {}; + + const data = await this._graphWatcher.getEntity(entity, id, blockHash, relations); + + return data; } async triggerIndexingOnEvent (event: Event): Promise { @@ -1097,6 +1122,87 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getAncestorAtDepth(blockHash, depth); } + _populateRelationsMap (): void { + // Needs to be generated by codegen. + this._relationsMap.set(ProducerSet, { + producers: { + entity: Producer, + isArray: true + } + }); + + this._relationsMap.set(RewardSchedule, { + rewardScheduleEntries: { + entity: RewardScheduleEntry, + isArray: true + }, + activeRewardScheduleEntry: { + entity: RewardScheduleEntry, + isArray: false + } + }); + + this._relationsMap.set(ProducerEpoch, { + epoch: { + entity: Epoch, + isArray: false + } + }); + + this._relationsMap.set(Epoch, { + startBlock: { + entity: Block, + isArray: false + }, + endBlock: { + entity: Block, + isArray: false + } + }); + + this._relationsMap.set(SlotClaim, { + slot: { + entity: Slot, + isArray: false + } + }); + + this._relationsMap.set(Network, { + stakers: { + entity: Staker, + isArray: true + } + }); + + this._relationsMap.set(Distributor, { + currentDistribution: { + entity: Distribution, + isArray: false + } + }); + + this._relationsMap.set(Distribution, { + distributor: { + entity: Distributor, + isArray: false + } + }); + + this._relationsMap.set(Claim, { + account: { + entity: Account, + isArray: false + } + }); + + this._relationsMap.set(Slash, { + account: { + entity: Account, + isArray: false + } + }); + } + async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial): Promise { assert(blockHash); let { block, logs } = await this._ethClient.getLogs({ blockHash }); diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index 70328750..35cfc834 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -73,6 +73,65 @@ export class Database { } } + async getEntityWithRelations (entity: (new () => Entity) | string, id: string, blockHash: string, relations: { [key: string]: any }): Promise { + const queryRunner = this._conn.createQueryRunner(); + + try { + const repo = queryRunner.manager.getRepository(entity); + + // Fetching blockHash for previous entity in frothy region. + const { blockHash: entityblockHash, blockNumber, id: frothyId } = await this._baseDatabase.getFrothyEntity(queryRunner, repo, { blockHash, id }); + + let selectQueryBuilder = repo.createQueryBuilder('entity'); + + if (frothyId) { + // If entity found in frothy region. + selectQueryBuilder = selectQueryBuilder.where('entity.block_hash = :entityblockHash', { entityblockHash }); + } else { + // If entity not in frothy region. + const canonicalBlockNumber = blockNumber + 1; + + selectQueryBuilder = selectQueryBuilder.innerJoinAndSelect('block_progress', 'block', 'block.block_hash = entity.block_hash') + .where('block.is_pruned = false') + .andWhere('entity.block_number <= :canonicalBlockNumber', { canonicalBlockNumber }) + .orderBy('entity.block_number', 'DESC') + .limit(1); + } + + selectQueryBuilder = selectQueryBuilder.andWhere('entity.id = :id', { id }); + + // TODO: Implement query for nested relations. + Object.entries(relations).forEach(([field, data], index) => { + const { entity: relatedEntity, isArray } = data; + const alias = `relatedEntity${index}`; + + if (isArray) { + // For one to many relational field. + selectQueryBuilder = selectQueryBuilder.leftJoinAndMapMany( + `entity.${field}`, + relatedEntity, + alias, + `${alias}.id IN (SELECT unnest(entity.${field})) AND ${alias}.block_number <= entity.block_number` + ) + .addOrderBy(`${alias}.block_number`, 'DESC'); + } else { + // For one to one relational field. + selectQueryBuilder = selectQueryBuilder.leftJoinAndMapOne( + `entity.${field}`, + relatedEntity, + alias, + `entity.${field} = ${alias}.id AND ${alias}.block_number <= entity.block_number` + ) + .addOrderBy(`${alias}.block_number`, 'DESC'); + } + }); + + return selectQueryBuilder.getOne(); + } finally { + await queryRunner.release(); + } + } + async saveEntity (entity: string, data: any): Promise { const repo = this._conn.getRepository(entity); diff --git a/packages/graph-node/src/watcher.ts b/packages/graph-node/src/watcher.ts index 3a431bdf..a1b21b50 100644 --- a/packages/graph-node/src/watcher.ts +++ b/packages/graph-node/src/watcher.ts @@ -184,9 +184,9 @@ export class GraphWatcher { this._indexer = indexer; } - async getEntity (entity: new () => Entity, id: string, blockHash: string): Promise { + async getEntity (entity: new () => Entity, id: string, blockHash: string, relations: { [key: string]: any }): Promise { // Get entity from the database. - const result = await this._database.getEntity(entity, id, blockHash) as any; + const result = await this._database.getEntityWithRelations(entity, id, blockHash, relations) as any; // Resolve any field name conflicts in the entity result. return resolveEntityFieldConflicts(result); diff --git a/packages/graph-node/test/subgraph/example1/generated/schema.ts b/packages/graph-node/test/subgraph/example1/generated/schema.ts index d4b490cf..7a218245 100644 --- a/packages/graph-node/test/subgraph/example1/generated/schema.ts +++ b/packages/graph-node/test/subgraph/example1/generated/schema.ts @@ -18,7 +18,6 @@ export class RelatedEntity extends Entity { this.set("id", Value.fromString(id)); this.set("paramBigInt", Value.fromBigInt(BigInt.zero())); - this.set("examples", Value.fromStringArray(new Array(0))); this.set("bigIntArray", Value.fromBigIntArray(new Array(0))); } @@ -57,15 +56,6 @@ export class RelatedEntity extends Entity { this.set("paramBigInt", Value.fromBigInt(value)); } - get examples(): Array { - let value = this.get("examples"); - return value!.toStringArray(); - } - - set examples(value: Array) { - this.set("examples", Value.fromStringArray(value)); - } - get bigIntArray(): Array { let value = this.get("bigIntArray"); return value!.toBigIntArray(); @@ -89,6 +79,7 @@ export class ExampleEntity extends Entity { this.set("paramEnum", Value.fromString("")); this.set("paramBigDecimal", Value.fromBigDecimal(BigDecimal.zero())); this.set("related", Value.fromString("")); + this.set("manyRelated", Value.fromStringArray(new Array(0))); } save(): void { @@ -188,4 +179,59 @@ export class ExampleEntity extends Entity { set related(value: string) { this.set("related", Value.fromString(value)); } + + get manyRelated(): Array { + let value = this.get("manyRelated"); + return value!.toStringArray(); + } + + set manyRelated(value: Array) { + this.set("manyRelated", Value.fromStringArray(value)); + } +} + +export class ManyRelatedEntity extends Entity { + constructor(id: string) { + super(); + this.set("id", Value.fromString(id)); + + this.set("count", Value.fromBigInt(BigInt.zero())); + } + + save(): void { + let id = this.get("id"); + assert(id != null, "Cannot save ManyRelatedEntity entity without an ID"); + if (id) { + assert( + id.kind == ValueKind.STRING, + "Cannot save ManyRelatedEntity entity with non-string ID. " + + 'Considering using .toHex() to convert the "id" to a string.' + ); + store.set("ManyRelatedEntity", id.toString(), this); + } + } + + static load(id: string): ManyRelatedEntity | null { + return changetype( + store.get("ManyRelatedEntity", id) + ); + } + + get id(): string { + let value = this.get("id"); + return value!.toString(); + } + + set id(value: string) { + this.set("id", Value.fromString(value)); + } + + get count(): BigInt { + let value = this.get("count"); + return value!.toBigInt(); + } + + set count(value: BigInt) { + this.set("count", Value.fromBigInt(value)); + } } diff --git a/packages/graph-node/test/subgraph/example1/schema.graphql b/packages/graph-node/test/subgraph/example1/schema.graphql index dd3a30a8..34bb5754 100644 --- a/packages/graph-node/test/subgraph/example1/schema.graphql +++ b/packages/graph-node/test/subgraph/example1/schema.graphql @@ -6,7 +6,6 @@ enum EnumType { type RelatedEntity @entity { id: ID! paramBigInt: BigInt! - examples: [ExampleEntity!]! bigIntArray: [BigInt!]! } @@ -20,4 +19,10 @@ type ExampleEntity @entity { paramEnum: EnumType! paramBigDecimal: BigDecimal! related: RelatedEntity! + manyRelated: [ManyRelatedEntity!]! +} + +type ManyRelatedEntity @entity { + id: ID! + count: BigInt! } diff --git a/packages/graph-node/test/subgraph/example1/src/mapping.ts b/packages/graph-node/test/subgraph/example1/src/mapping.ts index 64a96776..588b4f8d 100644 --- a/packages/graph-node/test/subgraph/example1/src/mapping.ts +++ b/packages/graph-node/test/subgraph/example1/src/mapping.ts @@ -4,7 +4,7 @@ import { Example1, Test } from '../generated/Example1/Example1'; -import { ExampleEntity, RelatedEntity } from '../generated/schema'; +import { ExampleEntity, ManyRelatedEntity, RelatedEntity } from '../generated/schema'; export function handleTest (event: Test): void { log.debug('event.address: {}', [event.address.toHexString()]); @@ -15,12 +15,12 @@ export function handleTest (event: Test): void { // Entities can be loaded from the store using a string ID; this ID // needs to be unique across all entities of the same type - let entity = ExampleEntity.load(event.transaction.hash.toHexString()); + let entity = ExampleEntity.load(event.transaction.from.toHex()); // Entities only exist after they have been saved to the store; // `null` checks allow to create entities on demand if (!entity) { - entity = new ExampleEntity(event.transaction.hash.toHexString()); + entity = new ExampleEntity(event.transaction.from.toHex()); // Entity fields can be set using simple assignments entity.count = BigInt.fromString('0'); @@ -37,10 +37,10 @@ export function handleTest (event: Test): void { entity.paramEnum = 'choice1'; entity.paramBigDecimal = BigDecimal.fromString('123'); - let relatedEntity = RelatedEntity.load(event.transaction.from.toHex()); + let relatedEntity = RelatedEntity.load(event.params.param1); if (!relatedEntity) { - relatedEntity = new RelatedEntity(event.transaction.from.toHex()); + relatedEntity = new RelatedEntity(event.params.param1); relatedEntity.paramBigInt = BigInt.fromString('123'); } @@ -48,14 +48,17 @@ export function handleTest (event: Test): void { bigIntArray.push(entity.count); relatedEntity.bigIntArray = bigIntArray; - const examples = relatedEntity.examples; - examples.push(entity.id); - relatedEntity.examples = examples; - relatedEntity.save(); - entity.related = relatedEntity.id; + const manyRelatedEntity = new ManyRelatedEntity(event.transaction.hash.toHexString()); + manyRelatedEntity.count = entity.count; + manyRelatedEntity.save(); + + const manyRelated = entity.manyRelated; + manyRelated.push(manyRelatedEntity.id); + entity.manyRelated = manyRelated; + // Entities can be written to the store with `.save()` entity.save(); diff --git a/packages/graph-test-watcher/src/entity/ExampleEntity.ts b/packages/graph-test-watcher/src/entity/ExampleEntity.ts index 4e873cc9..758cbb67 100644 --- a/packages/graph-test-watcher/src/entity/ExampleEntity.ts +++ b/packages/graph-test-watcher/src/entity/ExampleEntity.ts @@ -50,4 +50,7 @@ export class ExampleEntity { @Column('varchar') related!: string; + + @Column('varchar', { array: true }) + manyRelated!: string[] } diff --git a/packages/graph-test-watcher/src/entity/ManyRelatedEntity.ts b/packages/graph-test-watcher/src/entity/ManyRelatedEntity.ts new file mode 100644 index 00000000..e8fce624 --- /dev/null +++ b/packages/graph-test-watcher/src/entity/ManyRelatedEntity.ts @@ -0,0 +1,22 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +import { bigintTransformer } from '@vulcanize/util'; + +@Entity() +export class ManyRelatedEntity { + @PrimaryColumn('varchar') + id!: string; + + @PrimaryColumn('varchar', { length: 66 }) + blockHash!: string; + + @Column('integer') + blockNumber!: number; + + @Column('bigint', { transformer: bigintTransformer }) + count!: bigint; +} diff --git a/packages/graph-test-watcher/src/entity/RelatedEntity.ts b/packages/graph-test-watcher/src/entity/RelatedEntity.ts index 65511e02..2dbe958c 100644 --- a/packages/graph-test-watcher/src/entity/RelatedEntity.ts +++ b/packages/graph-test-watcher/src/entity/RelatedEntity.ts @@ -20,9 +20,6 @@ export class RelatedEntity { @Column('bigint', { transformer: bigintTransformer }) paramBigInt!: bigint; - @Column('varchar', { array: true }) - examples!: string[]; - @Column('bigint', { transformer: bigintArrayTransformer, array: true }) bigIntArray!: bigint[]; } diff --git a/packages/graph-test-watcher/src/indexer.ts b/packages/graph-test-watcher/src/indexer.ts index dd5f74cc..fb133be3 100644 --- a/packages/graph-test-watcher/src/indexer.ts +++ b/packages/graph-test-watcher/src/indexer.ts @@ -29,6 +29,9 @@ import { IPLDBlock } from './entity/IPLDBlock'; import artifacts from './artifacts/Example.json'; import { createInitialCheckpoint, handleEvent, createStateDiff, createStateCheckpoint } from './hooks'; import { IPFSClient } from './ipfs'; +import { ExampleEntity } from './entity/ExampleEntity'; +import { RelatedEntity } from './entity/RelatedEntity'; +import { ManyRelatedEntity } from './entity/ManyRelatedEntity'; const log = debug('vulcanize:indexer'); @@ -87,6 +90,8 @@ export class Indexer implements IndexerInterface { _ipfsClient: IPFSClient + _relationsMap: Map + constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher: GraphWatcher) { assert(db); assert(ethClient); @@ -111,6 +116,9 @@ export class Indexer implements IndexerInterface { this._contract = new ethers.utils.Interface(this._abi); this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); + + this._relationsMap = new Map(); + this._populateRelationsMap(); } getResultEvent (event: Event): ResultEvent { @@ -537,7 +545,11 @@ export class Indexer implements IndexerInterface { } async getSubgraphEntity (entity: new () => Entity, id: string, blockHash: string): Promise { - return this._graphWatcher.getEntity(entity, id, blockHash); + const relations = this._relationsMap.get(entity) || {}; + + const data = await this._graphWatcher.getEntity(entity, id, blockHash, relations); + + return data; } async triggerIndexingOnEvent (event: Event): Promise { @@ -708,6 +720,20 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getAncestorAtDepth(blockHash, depth); } + _populateRelationsMap (): void { + // Needs to be generated by codegen. + this._relationsMap.set(ExampleEntity, { + related: { + entity: RelatedEntity, + isArray: false + }, + manyRelated: { + entity: ManyRelatedEntity, + isArray: true + } + }); + } + async _fetchAndSaveEvents ({ cid: blockCid, blockHash }: DeepPartial): Promise { assert(blockHash); let { block, logs } = await this._ethClient.getLogs({ blockHash }); diff --git a/packages/graph-test-watcher/src/resolvers.ts b/packages/graph-test-watcher/src/resolvers.ts index a9fbc1b9..2b097f19 100644 --- a/packages/graph-test-watcher/src/resolvers.ts +++ b/packages/graph-test-watcher/src/resolvers.ts @@ -13,6 +13,7 @@ import { EventWatcher } from './events'; import { ExampleEntity } from './entity/ExampleEntity'; import { RelatedEntity } from './entity/RelatedEntity'; +import { ManyRelatedEntity } from './entity/ManyRelatedEntity'; const log = debug('vulcanize:resolver'); @@ -56,16 +57,22 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch return indexer._test(blockHash, contractAddress); }, + exampleEntity: async (_: any, { id, blockHash }: { id: string, blockHash: string }): Promise => { + log('exampleEntity', id, blockHash); + + return indexer.getSubgraphEntity(ExampleEntity, id, blockHash); + }, + relatedEntity: async (_: any, { id, blockHash }: { id: string, blockHash: string }): Promise => { log('relatedEntity', id, blockHash); return indexer.getSubgraphEntity(RelatedEntity, id, blockHash); }, - exampleEntity: async (_: any, { id, blockHash }: { id: string, blockHash: string }): Promise => { - log('exampleEntity', id, blockHash); + manyRelatedEntity: async (_: any, { id, blockHash }: { id: string, blockHash: string }): Promise => { + log('relatedEntity', id, blockHash); - return indexer.getSubgraphEntity(ExampleEntity, id, blockHash); + return indexer.getSubgraphEntity(ManyRelatedEntity, id, blockHash); }, events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => { diff --git a/packages/graph-test-watcher/src/schema.gql b/packages/graph-test-watcher/src/schema.gql index 13b47675..0112caed 100644 --- a/packages/graph-test-watcher/src/schema.gql +++ b/packages/graph-test-watcher/src/schema.gql @@ -74,6 +74,7 @@ type Query { _test(blockHash: String!, contractAddress: String!): ResultBigInt! relatedEntity(id: String!, blockHash: String!): RelatedEntity! exampleEntity(id: String!, blockHash: String!): ExampleEntity! + manyRelatedEntity(id: String!, blockHash: String!): ManyRelatedEntity! getStateByCID(cid: String!): ResultIPLDBlock getState(blockHash: String!, contractAddress: String!, kind: String): ResultIPLDBlock } @@ -86,10 +87,14 @@ enum EnumType { type RelatedEntity { id: ID! paramBigInt: BigInt! - examples: [ExampleEntity!]! bigIntArray: [BigInt!]! } +type ManyRelatedEntity { + id: ID! + count: BigInt! +} + type ExampleEntity { id: ID! count: BigInt! @@ -100,6 +105,7 @@ type ExampleEntity { paramEnum: EnumType! paramBigDecimal: BigDecimal! related: RelatedEntity! + manyRelated: [ManyRelatedEntity!]! } type Mutation { diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index d47cd736..664bdff8 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -420,7 +420,7 @@ export class Database { return selectQueryBuilder.getMany(); } - async getPrevEntityVersion (queryRunner: QueryRunner, repo: Repository, findOptions: { [key: string]: any }): Promise { + async getFrothyEntity (queryRunner: QueryRunner, repo: Repository, data: { blockHash: string, id: string }): Promise<{ blockHash: string, blockNumber: number, id: string }> { // Hierarchical query for getting the entity in the frothy region. const heirerchicalQuery = ` WITH RECURSIVE cte_query AS @@ -466,7 +466,13 @@ export class Database { `; // Fetching blockHash for previous entity in frothy region. - const [{ block_hash: blockHash, block_number: blockNumber, id }] = await queryRunner.query(heirerchicalQuery, [findOptions.where.blockHash, findOptions.where.id, MAX_REORG_DEPTH]); + const [{ block_hash: blockHash, block_number: blockNumber, id }] = await queryRunner.query(heirerchicalQuery, [data.blockHash, data.id, MAX_REORG_DEPTH]); + + return { blockHash, blockNumber, id }; + } + + async getPrevEntityVersion (queryRunner: QueryRunner, repo: Repository, findOptions: { [key: string]: any }): Promise { + const { blockHash, blockNumber, id } = await this.getFrothyEntity(queryRunner, repo, findOptions.where); if (id) { // Entity found in frothy region.