From 3e0c84b3336e4213331175a5383936bac702694b Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Tue, 7 Dec 2021 16:59:36 +0530 Subject: [PATCH] Use entity column type map to create entity in store get API (#72) * Use typeof to distinguish between BigInt and BigDecimal in store get API * Use entity column type map to create entity in store get API * Add entity column type map in eden-watcher --- packages/eden-watcher/src/indexer.ts | 221 +++++++++++++++++++++ packages/eden-watcher/src/schema.gql | 14 +- packages/graph-node/src/database.ts | 8 +- packages/graph-node/src/loader.ts | 8 +- packages/graph-node/src/utils.ts | 61 +++--- packages/graph-node/test/utils/indexer.ts | 4 + packages/graph-test-watcher/src/indexer.ts | 44 ++++ packages/util/src/types.ts | 1 + 8 files changed, 315 insertions(+), 46 deletions(-) diff --git a/packages/eden-watcher/src/indexer.ts b/packages/eden-watcher/src/indexer.ts index d98eac70..8eedd3db 100644 --- a/packages/eden-watcher/src/indexer.ts +++ b/packages/eden-watcher/src/indexer.ts @@ -133,6 +133,7 @@ export class Indexer implements IndexerInterface { _ipfsClient: IPFSClient + _entityTypesMap: Map _relationsMap: Map constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher: GraphWatcher) { @@ -179,6 +180,9 @@ export class Indexer implements IndexerInterface { this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); + this._entityTypesMap = new Map(); + this._populateEntityTypesMap(); + this._relationsMap = new Map(); this._populateRelationsMap(); } @@ -1139,6 +1143,223 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getAncestorAtDepth(blockHash, depth); } + getEntityTypesMap (): Map { + return this._entityTypesMap; + } + + _populateEntityTypesMap (): void { + this._entityTypesMap.set( + 'Producer', + { + id: 'ID', + active: 'Boolean', + rewardCollector: 'Bytes', + rewards: 'BigInt', + confirmedBlocks: 'BigInt', + pendingEpochBlocks: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'ProducerSet', + { + id: 'ID', + producers: 'Producer' + } + ); + + this._entityTypesMap.set( + 'ProducerSetChange', + { + id: 'ID', + blockNumber: 'BigInt', + producer: 'Bytes', + changeType: 'ProducerSetChangeType' + } + ); + + this._entityTypesMap.set( + 'ProducerRewardCollectorChange', + { + id: 'ID', + blockNumber: 'BigInt', + producer: 'Bytes', + rewardCollector: 'Bytes' + } + ); + + this._entityTypesMap.set( + 'RewardScheduleEntry', + { + id: 'ID', + startTime: 'BigInt', + epochDuration: 'BigInt', + rewardsPerEpoch: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'RewardSchedule', + { + id: 'ID', + rewardScheduleEntries: 'RewardScheduleEntry', + lastEpoch: 'Epoch', + pendingEpoch: 'Epoch', + activeRewardScheduleEntry: 'RewardScheduleEntry' + } + ); + + this._entityTypesMap.set( + 'Block', + { + id: 'ID', + fromActiveProducer: 'Boolean', + hash: 'Bytes', + parentHash: 'Bytes', + unclesHash: 'Bytes', + author: 'Bytes', + stateRoot: 'Bytes', + transactionsRoot: 'Bytes', + receiptsRoot: 'Bytes', + number: 'BigInt', + gasUsed: 'BigInt', + gasLimit: 'BigInt', + timestamp: 'BigInt', + difficulty: 'BigInt', + totalDifficulty: 'BigInt', + size: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'Epoch', + { + id: 'ID', + finalized: 'Boolean', + epochNumber: 'BigInt', + startBlock: 'Block', + endBlock: 'Block', + producerBlocks: 'BigInt', + allBlocks: 'BigInt', + producerBlocksRatio: 'BigDecimal' + } + ); + + this._entityTypesMap.set( + 'ProducerEpoch', + { + id: 'ID', + address: 'Bytes', + epoch: 'Epoch', + totalRewards: 'BigInt', + blocksProduced: 'BigInt', + blocksProducedRatio: 'BigDecimal' + } + ); + + this._entityTypesMap.set( + 'SlotClaim', + { + id: 'ID', + slot: 'Slot', + owner: 'Bytes', + winningBid: 'BigInt', + oldBid: 'BigInt', + startTime: 'BigInt', + expirationTime: 'BigInt', + taxRatePerDay: 'BigDecimal' + } + ); + + this._entityTypesMap.set( + 'Slot', + { + id: 'ID', + owner: 'Bytes', + delegate: 'Bytes', + winningBid: 'BigInt', + oldBid: 'BigInt', + startTime: 'BigInt', + expirationTime: 'BigInt', + taxRatePerDay: 'BigDecimal' + } + ); + + this._entityTypesMap.set( + 'Staker', + { + id: 'ID', + staked: 'BigInt', + rank: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'Network', + { + id: 'ID', + slot0: 'Slot', + slot1: 'Slot', + slot2: 'Slot', + stakers: 'Staker', + numStakers: 'BigInt', + totalStaked: 'BigInt', + stakedPercentiles: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'Distributor', + { + id: 'ID', + currentDistribution: 'Distribution' + } + ); + + this._entityTypesMap.set( + 'Distribution', + { + id: 'ID', + distributor: 'Distributor', + timestamp: 'BigInt', + distributionNumber: 'BigInt', + merkleRoot: 'Bytes', + metadataURI: 'String' + } + ); + + this._entityTypesMap.set( + 'Claim', + { + id: 'ID', + timestamp: 'BigInt', + index: 'BigInt', + account: 'Account', + totalEarned: 'BigInt', + claimed: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'Account', + { + id: 'ID', + totalClaimed: 'BigInt', + totalSlashed: 'BigInt' + } + ); + + this._entityTypesMap.set( + 'Slash', + { + id: 'ID', + timestamp: 'BigInt', + account: 'Account', + slashed: 'BigInt' + } + ); + } + _populateRelationsMap (): void { // Needs to be generated by codegen. this._relationsMap.set(ProducerSet, { diff --git a/packages/eden-watcher/src/schema.gql b/packages/eden-watcher/src/schema.gql index af70c65a..d2434e50 100644 --- a/packages/eden-watcher/src/schema.gql +++ b/packages/eden-watcher/src/schema.gql @@ -296,13 +296,13 @@ type RewardSchedule { type Block { id: ID! fromActiveProducer: Boolean! - hash: String! - parentHash: String! - unclesHash: String! - author: String! - stateRoot: String! - transactionsRoot: String! - receiptsRoot: String! + hash: Bytes! + parentHash: Bytes! + unclesHash: Bytes! + author: Bytes! + stateRoot: Bytes! + transactionsRoot: Bytes! + receiptsRoot: Bytes! number: BigInt! gasUsed: BigInt! gasLimit: BigInt! diff --git a/packages/graph-node/src/database.ts b/packages/graph-node/src/database.ts index 64951ebb..cf053e49 100644 --- a/packages/graph-node/src/database.ts +++ b/packages/graph-node/src/database.ts @@ -189,7 +189,7 @@ export class Database { await repo.save(dbEntity); } - async toGraphEntity (instanceExports: any, entity: string, data: any): Promise { + async toGraphEntity (instanceExports: any, entity: string, data: any, entityTypes: { [key: string]: string }): Promise { // TODO: Cache schema/columns. const repo = this._conn.getRepository(entity); const entityFields = repo.metadata.columns; @@ -210,11 +210,11 @@ export class Database { // Fill _blockNumber as blockNumber and _blockHash as blockHash in the entityInstance (wasm). if (['_blockNumber', '_blockHash'].includes(field.propertyName)) { field.propertyName = field.propertyName.slice(1); - - return toEntityValue(instanceExports, entityInstance, data, field); } - return toEntityValue(instanceExports, entityInstance, data, field); + const gqlType = entityTypes[field.propertyName]; + + return toEntityValue(instanceExports, entityInstance, data, field, gqlType); }, {}); await Promise.all(entityValuePromises); diff --git a/packages/graph-node/src/loader.ts b/packages/graph-node/src/loader.ts index fcb912ae..cb45021d 100644 --- a/packages/graph-node/src/loader.ts +++ b/packages/graph-node/src/loader.ts @@ -77,7 +77,13 @@ export const instantiate = async ( return null; } - return database.toGraphEntity(instanceExports, entityName, entityData); + assert(indexer.getEntityTypesMap); + const entityTypesMap = indexer.getEntityTypesMap(); + + const entityTypes = entityTypesMap.get(entityName); + assert(entityTypes); + + return database.toGraphEntity(instanceExports, entityName, entityData, entityTypes); }, 'store.set': async (entity: number, id: number, data: number) => { const entityName = __getString(entity); diff --git a/packages/graph-node/src/utils.ts b/packages/graph-node/src/utils.ts index 38d5c9ca..d7349a4d 100644 --- a/packages/graph-node/src/utils.ts +++ b/packages/graph-node/src/utils.ts @@ -3,7 +3,6 @@ import path from 'path'; import fs from 'fs-extra'; import debug from 'debug'; import yaml from 'js-yaml'; -import { ColumnType } from 'typeorm'; import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata'; import { GraphDecimal } from '@vulcanize/util'; @@ -382,9 +381,9 @@ export const getSubgraphConfig = async (subgraphPath: string): Promise => { return config; }; -export const toEntityValue = async (instanceExports: any, entityInstance: any, data: any, field: ColumnMetadata) => { +export const toEntityValue = async (instanceExports: any, entityInstance: any, data: any, field: ColumnMetadata, type: string) => { const { __newString, Value } = instanceExports; - const { type, isArray, propertyName } = field; + const { isArray, propertyName } = field; const entityKey = await __newString(propertyName); const entityValuePtr = await entityInstance.get(entityKey); @@ -475,11 +474,10 @@ const parseEntityValue = async (instanceExports: any, valuePtr: number) => { } }; -const formatEntityValue = async (instanceExports: any, subgraphValue: any, type: ColumnType, value: any, isArray: boolean): Promise => { +const formatEntityValue = async (instanceExports: any, subgraphValue: any, type: string, value: any, isArray: boolean): Promise => { const { __newString, __newArray, BigInt: ExportBigInt, Value, ByteArray, Bytes, BigDecimal, id_of_type: getIdOfType } = instanceExports; if (isArray) { - // TODO: Implement handling array of Bytes type field. const dataArrayPromises = value.map((el: any) => formatEntityValue(instanceExports, subgraphValue, type, el, false)); const dataArray = await Promise.all(dataArrayPromises); const arrayStoreValueId = await getIdOfType(TypeId.ArrayStoreValue); @@ -489,54 +487,49 @@ const formatEntityValue = async (instanceExports: any, subgraphValue: any, type: } switch (type) { - case 'varchar': { + case 'ID': + case 'String': { const entityValue = await __newString(value); - const kind = await subgraphValue.kind; - switch (kind) { - case ValueKind.BYTES: { - const byteArray = await ByteArray.fromHexString(entityValue); - const bytes = await Bytes.fromByteArray(byteArray); - - return Value.fromBytes(bytes); - } - - default: - return Value.fromString(entityValue); - } + return Value.fromString(entityValue); } - case 'integer': { + case 'Boolean': { + return Value.fromBoolean(value ? 1 : 0); + } + + case 'Int': { return Value.fromI32(value); } - case 'bigint': { + case 'BigInt': { const valueStringPtr = await __newString(value.toString()); const bigInt = await ExportBigInt.fromString(valueStringPtr); return Value.fromBigInt(bigInt); } - case 'boolean': { - return Value.fromBoolean(value ? 1 : 0); - } - - case 'enum': { - const entityValue = await __newString(value); - - return Value.fromString(entityValue); - } - - case 'numeric': { + case 'BigDecimal': { const valueStringPtr = await __newString(value.toString()); const bigDecimal = await BigDecimal.fromString(valueStringPtr); return Value.fromBigDecimal(bigDecimal); } - // TODO: Support more types. - default: - throw new Error(`Unsupported type: ${type}`); + case 'Bytes': { + const entityValue = await __newString(value); + const byteArray = await ByteArray.fromHexString(entityValue); + const bytes = await Bytes.fromByteArray(byteArray); + + return Value.fromBytes(bytes); + } + + // Return default as string for enum or custom type. + default: { + const entityValue = await __newString(value); + + return Value.fromString(entityValue); + } } }; diff --git a/packages/graph-node/test/utils/indexer.ts b/packages/graph-node/test/utils/indexer.ts index d38d62ab..2249d4e8 100644 --- a/packages/graph-node/test/utils/indexer.ts +++ b/packages/graph-node/test/utils/indexer.ts @@ -100,6 +100,10 @@ export class Indexer implements IndexerInterface { assert(blockHash); assert(data); } + + getEntityTypesMap (): Map { + return new Map(); + } } class SyncStatus implements SyncStatusInterface { diff --git a/packages/graph-test-watcher/src/indexer.ts b/packages/graph-test-watcher/src/indexer.ts index c447e440..8f742fa4 100644 --- a/packages/graph-test-watcher/src/indexer.ts +++ b/packages/graph-test-watcher/src/indexer.ts @@ -90,6 +90,7 @@ export class Indexer implements IndexerInterface { _ipfsClient: IPFSClient + _entityTypesMap: Map _relationsMap: Map constructor (serverConfig: ServerConfig, db: Database, ethClient: EthClient, postgraphileClient: EthClient, ethProvider: BaseProvider, jobQueue: JobQueue, graphWatcher: GraphWatcher) { @@ -117,6 +118,9 @@ export class Indexer implements IndexerInterface { this._ipfsClient = new IPFSClient(this._serverConfig.ipfsApiAddr); + this._entityTypesMap = new Map(); + this._populateEntityTypesMap(); + this._relationsMap = new Map(); this._populateRelationsMap(); } @@ -738,6 +742,46 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getAncestorAtDepth(blockHash, depth); } + getEntityTypesMap (): Map { + return this._entityTypesMap; + } + + _populateEntityTypesMap (): void { + this._entityTypesMap.set( + 'Author', + { + id: 'ID', + blogCount: 'BigInt', + name: 'String', + rating: 'BigDecimal', + paramInt: 'Int', + paramBigInt: 'BigInt', + paramBytes: 'Bytes' + } + ); + + this._entityTypesMap.set( + 'Blog', + { + id: 'ID', + kind: 'BlogKind', + isActive: 'Boolean', + reviews: 'BigInt', + author: 'Author', + categories: 'Category' + } + ); + + this._entityTypesMap.set( + 'Category', + { + id: 'ID', + name: 'String', + count: 'BigInt' + } + ); + } + _populateRelationsMap (): void { // Needs to be generated by codegen. this._relationsMap.set(Author, { diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index c49304f3..105cac05 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -74,6 +74,7 @@ export interface IndexerInterface { cacheContract?: (contract: ContractInterface) => void; createDiffStaged?: (contractAddress: string, blockHash: string, data: any) => Promise watchContract?: (address: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise + getEntityTypesMap?: () => Map } export interface EventWatcherInterface {