diff --git a/packages/azimuth-watcher/README.md b/packages/azimuth-watcher/README.md index 235f426..ba07311 100644 --- a/packages/azimuth-watcher/README.md +++ b/packages/azimuth-watcher/README.md @@ -81,7 +81,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3001/graphql * If the watcher is an `active` watcher: @@ -97,7 +97,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3001/graphql * To watch a contract: diff --git a/packages/censures-watcher/README.md b/packages/censures-watcher/README.md index 29bf664..b464749 100644 --- a/packages/censures-watcher/README.md +++ b/packages/censures-watcher/README.md @@ -81,7 +81,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3002/graphql * If the watcher is an `active` watcher: @@ -97,7 +97,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3002/graphql * To watch a contract: diff --git a/packages/claims-watcher/README.md b/packages/claims-watcher/README.md index a7b0061..855024a 100644 --- a/packages/claims-watcher/README.md +++ b/packages/claims-watcher/README.md @@ -81,7 +81,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3003/graphql * If the watcher is an `active` watcher: @@ -97,7 +97,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3003/graphql * To watch a contract: diff --git a/packages/conditional-star-release-watcher/README.md b/packages/conditional-star-release-watcher/README.md index 84e92be..d09c478 100644 --- a/packages/conditional-star-release-watcher/README.md +++ b/packages/conditional-star-release-watcher/README.md @@ -81,7 +81,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3004/graphql * If the watcher is an `active` watcher: @@ -97,7 +97,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3004/graphql * To watch a contract: diff --git a/packages/polls-watcher/README.md b/packages/polls-watcher/README.md index c724b53..8b06932 100644 --- a/packages/polls-watcher/README.md +++ b/packages/polls-watcher/README.md @@ -81,7 +81,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3008/graphql * If the watcher is an `active` watcher: @@ -97,7 +97,7 @@ To enable GQL requests caching: yarn server ``` - GQL console: http://localhost:3009/graphql + GQL console: http://localhost:3008/graphql * To watch a contract: diff --git a/packages/polls-watcher/environments/local.toml b/packages/polls-watcher/environments/local.toml index a828b8b..c87e5d2 100644 --- a/packages/polls-watcher/environments/local.toml +++ b/packages/polls-watcher/environments/local.toml @@ -1,7 +1,8 @@ [server] host = "127.0.0.1" port = 3008 - kind = "lazy" + kind = "active" + gqlPath = "/graphql" # Checkpointing state. checkpointing = true @@ -11,15 +12,15 @@ # Enable state creation # CAUTION: Disable only if state creation is not desired or can be filled subsequently - enableState = true - - # Boolean to filter logs by contract. - filterLogs = false + enableState = false # Max block range for which to return events in eventsInRange GQL query. # Use -1 for skipping check on block range. maxEventsBlockRange = 1000 + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + # GQL cache settings [server.gqlCache] enabled = true @@ -48,8 +49,19 @@ [upstream] [upstream.ethServer] - gqlApiEndpoint = "http://127.0.0.1:8083/graphql" - rpcProviderEndpoint = "http://127.0.0.1:8082" + gqlApiEndpoint = "http://127.0.0.1:8082/graphql" + rpcProviderEndpoint = "http://127.0.0.1:8081" + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = false + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = false [upstream.cache] name = "requests" @@ -61,6 +73,17 @@ maxCompletionLagInSecs = 300 jobDelayInMilliSecs = 100 eventsInBatch = 50 + subgraphEventsOrder = true blockDelayInMilliSecs = 2000 - prefetchBlocksInMem = true - prefetchBlockCount = 10 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 diff --git a/packages/polls-watcher/package.json b/packages/polls-watcher/package.json index a88ceac..181b7a1 100644 --- a/packages/polls-watcher/package.json +++ b/packages/polls-watcher/package.json @@ -38,28 +38,28 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", + "@cerc-io/cli": "^0.2.74", + "@cerc-io/ipld-eth-client": "^0.2.74", + "@cerc-io/solidity-mapper": "^0.2.74", + "@cerc-io/util": "^0.2.74", "@ethersproject/providers": "^5.4.4", - "@cerc-io/cli": "^0.2.40", - "@cerc-io/ipld-eth-client": "^0.2.40", - "@cerc-io/solidity-mapper": "^0.2.40", - "@cerc-io/util": "^0.2.40", - "apollo-type-bigint": "^0.1.3", "debug": "^4.3.1", + "decimal.js": "^10.3.1", "ethers": "^5.4.4", "graphql": "^15.5.0", "json-bigint": "^1.0.0", "reflect-metadata": "^0.1.13", - "typeorm": "^0.2.32", - "yargs": "^17.0.1", - "decimal.js": "^10.3.1" + "typeorm": "0.2.37", + "yargs": "^17.0.1" }, "devDependencies": { "@ethersproject/abi": "^5.3.0", - "@types/yargs": "^17.0.0", "@types/debug": "^4.1.5", "@types/json-bigint": "^1.0.0", + "@types/yargs": "^17.0.0", "@typescript-eslint/eslint-plugin": "^5.47.1", "@typescript-eslint/parser": "^5.47.1", + "copyfiles": "^2.4.1", "eslint": "^8.35.0", "eslint-config-semistandard": "^15.0.1", "eslint-config-standard": "^16.0.3", @@ -69,7 +69,6 @@ "eslint-plugin-standard": "^5.0.0", "husky": "^7.0.2", "ts-node": "^10.2.1", - "typescript": "^5.0.2", - "copyfiles": "^2.4.1" + "typescript": "^5.0.2" } } diff --git a/packages/polls-watcher/src/database.ts b/packages/polls-watcher/src/database.ts index 388517a..1924e4a 100644 --- a/packages/polls-watcher/src/database.ts +++ b/packages/polls-watcher/src/database.ts @@ -280,10 +280,10 @@ export class Database implements DatabaseInterface { return this._baseDatabase.saveBlockProgress(repo, block); } - async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { + async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise { const repo = queryRunner.manager.getRepository(Contract); - return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock); + return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context); } async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise { @@ -304,6 +304,18 @@ export class Database implements DatabaseInterface { return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force); } + async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise { + const repo = queryRunner.manager.getRepository(SyncStatus); + + return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force); + } + + async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise { + const repo = queryRunner.manager.getRepository(SyncStatus); + + return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError); + } + async getSyncStatus (queryRunner: QueryRunner): Promise { const repo = queryRunner.manager.getRepository(SyncStatus); diff --git a/packages/polls-watcher/src/entity/BlockProgress.ts b/packages/polls-watcher/src/entity/BlockProgress.ts index e744aae..ded4a86 100644 --- a/packages/polls-watcher/src/entity/BlockProgress.ts +++ b/packages/polls-watcher/src/entity/BlockProgress.ts @@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; - @Column('varchar') - cid!: string; + @Column('varchar', { nullable: true }) + cid!: string | null; @Column('varchar', { length: 66 }) blockHash!: string; diff --git a/packages/polls-watcher/src/entity/Contract.ts b/packages/polls-watcher/src/entity/Contract.ts index de66b38..e4defa8 100644 --- a/packages/polls-watcher/src/entity/Contract.ts +++ b/packages/polls-watcher/src/entity/Contract.ts @@ -21,4 +21,7 @@ export class Contract { @Column('integer') startingBlock!: number; + + @Column('jsonb', { nullable: true }) + context!: Record; } diff --git a/packages/polls-watcher/src/entity/StateSyncStatus.ts b/packages/polls-watcher/src/entity/StateSyncStatus.ts index c795212..1535eb4 100644 --- a/packages/polls-watcher/src/entity/StateSyncStatus.ts +++ b/packages/polls-watcher/src/entity/StateSyncStatus.ts @@ -12,6 +12,6 @@ export class StateSyncStatus { @Column('integer') latestIndexedBlockNumber!: number; - @Column('integer', { nullable: true }) + @Column('integer') latestCheckpointBlockNumber!: number; } diff --git a/packages/polls-watcher/src/entity/SyncStatus.ts b/packages/polls-watcher/src/entity/SyncStatus.ts index 19d0dfa..cc13c70 100644 --- a/packages/polls-watcher/src/entity/SyncStatus.ts +++ b/packages/polls-watcher/src/entity/SyncStatus.ts @@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface { @Column('integer') latestIndexedBlockNumber!: number; + @Column('varchar', { length: 66 }) + latestProcessedBlockHash!: string; + + @Column('integer') + latestProcessedBlockNumber!: number; + @Column('varchar', { length: 66 }) latestCanonicalBlockHash!: string; @@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface { @Column('integer') initialIndexedBlockNumber!: number; + + @Column('boolean', { default: false }) + hasIndexingError!: boolean; } diff --git a/packages/polls-watcher/src/gql/queries/getSyncStatus.gql b/packages/polls-watcher/src/gql/queries/getSyncStatus.gql index 2f80efd..48175b4 100644 --- a/packages/polls-watcher/src/gql/queries/getSyncStatus.gql +++ b/packages/polls-watcher/src/gql/queries/getSyncStatus.gql @@ -4,5 +4,9 @@ query getSyncStatus{ latestIndexedBlockNumber latestCanonicalBlockHash latestCanonicalBlockNumber + initialIndexedBlockHash + initialIndexedBlockNumber + latestProcessedBlockHash + latestProcessedBlockNumber } } \ No newline at end of file diff --git a/packages/polls-watcher/src/gql/queries/index.ts b/packages/polls-watcher/src/gql/queries/index.ts index 8708e36..1bfd022 100644 --- a/packages/polls-watcher/src/gql/queries/index.ts +++ b/packages/polls-watcher/src/gql/queries/index.ts @@ -10,6 +10,6 @@ export const getDocumentProposalCount = fs.readFileSync(path.join(__dirname, 'ge export const getDocumentMajorities = fs.readFileSync(path.join(__dirname, 'getDocumentMajorities.gql'), 'utf8'); export const hasVotedOnUpgradePoll = fs.readFileSync(path.join(__dirname, 'hasVotedOnUpgradePoll.gql'), 'utf8'); export const hasVotedOnDocumentPoll = fs.readFileSync(path.join(__dirname, 'hasVotedOnDocumentPoll.gql'), 'utf8'); -export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8'); export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8'); export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8'); +export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8'); diff --git a/packages/polls-watcher/src/indexer.ts b/packages/polls-watcher/src/indexer.ts index 550826d..36933b3 100644 --- a/packages/polls-watcher/src/indexer.ts +++ b/packages/polls-watcher/src/indexer.ts @@ -6,11 +6,10 @@ import assert from 'assert'; import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm'; import debug from 'debug'; import JSONbig from 'json-bigint'; -import { ethers } from 'ethers'; +import { ethers, constants } from 'ethers'; import { JsonFragment } from '@ethersproject/abi'; import { BaseProvider } from '@ethersproject/providers'; -import { EthClient } from '@cerc-io/ipld-eth-client'; import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper'; import { Indexer as BaseIndexer, @@ -25,7 +24,12 @@ import { ResultEvent, getResultEvent, DatabaseInterface, - Clients + Clients, + EthClient, + UpstreamConfig, + EthFullBlock, + EthFullTransaction, + ExtraEventData } from '@cerc-io/util'; import PollsArtifacts from './artifacts/Polls.json'; @@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface { _ethProvider: BaseProvider; _baseIndexer: BaseIndexer; _serverConfig: ServerConfig; + _upstreamConfig: UpstreamConfig; _abiMap: Map; _storageLayoutMap: Map; _contractMap: Map; + eventSignaturesMap: Map; - constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) { + constructor ( + config: { + server: ServerConfig; + upstream: UpstreamConfig; + }, + db: DatabaseInterface, + clients: Clients, + ethProvider: BaseProvider, + jobQueue: JobQueue + ) { assert(db); assert(clients.ethClient); this._db = db as Database; this._ethClient = clients.ethClient; this._ethProvider = ethProvider; - this._serverConfig = serverConfig; - this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue); + this._serverConfig = config.server; + this._upstreamConfig = config.upstream; + this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue); this._abiMap = new Map(); this._storageLayoutMap = new Map(); this._contractMap = new Map(); + this.eventSignaturesMap = new Map(); const { abi: PollsABI } = PollsArtifacts; assert(PollsABI); this._abiMap.set(KIND_POLLS, PollsABI); - this._contractMap.set(KIND_POLLS, new ethers.utils.Interface(PollsABI)); + + const PollsContractInterface = new ethers.utils.Interface(PollsABI); + this._contractMap.set(KIND_POLLS, PollsContractInterface); + + const PollsEventSignatures = Object.values(PollsContractInterface.events).map(value => { + return PollsContractInterface.getEventTopic(value); + }); + this.eventSignaturesMap.set(KIND_POLLS, PollsEventSignatures); } get serverConfig (): ServerConfig { return this._serverConfig; } + get upstreamConfig (): UpstreamConfig { + return this._upstreamConfig; + } + get storageLayoutMap (): Map { return this._storageLayoutMap; } @@ -416,16 +444,17 @@ export class Indexer implements IndexerInterface { await this._baseIndexer.removeStates(blockNumber, kind); } - async triggerIndexingOnEvent (event: Event): Promise { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise { const resultEvent = this.getResultEvent(event); // Call custom hook function for indexing on event. await handleEvent(this, resultEvent); } - async processEvent (event: Event): Promise { + async processEvent (event: Event, extraData: ExtraEventData): Promise { // Trigger indexing of data based on the event. - await this.triggerIndexingOnEvent(event); + await this.triggerIndexingOnEvent(event, extraData); } async processBlock (blockProgress: BlockProgress): Promise { @@ -456,7 +485,11 @@ export class Indexer implements IndexerInterface { return this._db.getStateSyncStatus(); } - async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { + async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise { + if (!this._serverConfig.enableState) { + return; + } + const dbTx = await this._db.createTransactionRunner(); let res; @@ -490,10 +523,14 @@ export class Indexer implements IndexerInterface { return res; } - async getLatestCanonicalBlock (): Promise { + async getLatestCanonicalBlock (): Promise { const syncStatus = await this.getSyncStatus(); assert(syncStatus); + if (syncStatus.latestCanonicalBlockHash === constants.HashZero) { + return; + } + const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash); assert(latestCanonicalBlock); @@ -504,8 +541,8 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getLatestStateIndexedBlock(); } - async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { - return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); + async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise { + return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context); } updateStateStatusMap (address: string, stateStatus: StateStatus): void { @@ -520,6 +557,10 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.saveEventEntity(dbEvent); } + async saveEvents (dbEvents: Event[]): Promise { + return this._baseIndexer.saveEvents(dbEvents); + } + async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise> { return this._baseIndexer.getEventsByFilter(blockHash, contract, name); } @@ -528,6 +569,10 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.isWatchedContract(address); } + getWatchedContracts (): Contract[] { + return this._baseIndexer.getWatchedContracts(); + } + getContractsByKind (kind: string): Contract[] { return this._baseIndexer.getContractsByKind(kind); } @@ -562,6 +607,14 @@ export class Indexer implements IndexerInterface { return syncStatus; } + async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise { + return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force); + } + + async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise { + return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError); + } + async getEvent (id: string): Promise { return this._baseIndexer.getEvent(id); } @@ -578,7 +631,24 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getBlocksAtHeight(height, isPruned); } - async saveBlockAndFetchEvents (block: DeepPartial): Promise<[BlockProgress, DeepPartial[]]> { + async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{ + blockProgress: BlockProgress, + events: DeepPartial[], + ethFullBlock: EthFullBlock; + ethFullTransactions: EthFullTransaction[]; + }[]> { + return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this)); + } + + async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise[]> { + return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this)); + } + + async saveBlockAndFetchEvents (block: DeepPartial): Promise<[ + BlockProgress, + DeepPartial[], + EthFullTransaction[] + ]> { return this._saveBlockAndFetchEvents(block); } @@ -607,17 +677,26 @@ export class Indexer implements IndexerInterface { await this._baseIndexer.resetWatcherToBlock(blockNumber, entities); } + async clearProcessedBlockData (block: BlockProgress): Promise { + const entities = [...ENTITIES]; + await this._baseIndexer.clearProcessedBlockData(block, entities); + } + async _saveBlockAndFetchEvents ({ cid: blockCid, blockHash, blockNumber, blockTimestamp, parentHash - }: DeepPartial): Promise<[BlockProgress, DeepPartial[]]> { + }: DeepPartial): Promise<[ + BlockProgress, + DeepPartial[], + EthFullTransaction[] + ]> { assert(blockHash); assert(blockNumber); - const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this)); + const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this)); const dbTx = await this._db.createTransactionRunner(); try { @@ -634,7 +713,7 @@ export class Indexer implements IndexerInterface { await dbTx.commitTransaction(); console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`); - return [blockProgress, []]; + return [blockProgress, [], transactions]; } catch (error) { await dbTx.rollbackTransaction(); throw error; diff --git a/packages/polls-watcher/src/job-runner.ts b/packages/polls-watcher/src/job-runner.ts index 27cfa69..b5aa6df 100644 --- a/packages/polls-watcher/src/job-runner.ts +++ b/packages/polls-watcher/src/job-runner.ts @@ -20,6 +20,7 @@ export const main = async (): Promise => { await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise => { await jobRunner.subscribeBlockProcessingQueue(); + await jobRunner.subscribeHistoricalProcessingQueue(); await jobRunner.subscribeEventProcessingQueue(); await jobRunner.subscribeBlockCheckpointQueue(); await jobRunner.subscribeHooksQueue(); diff --git a/packages/polls-watcher/src/resolvers.ts b/packages/polls-watcher/src/resolvers.ts index 4214d79..62884b7 100644 --- a/packages/polls-watcher/src/resolvers.ts +++ b/packages/polls-watcher/src/resolvers.ts @@ -3,13 +3,8 @@ // import assert from 'assert'; -import BigInt from 'apollo-type-bigint'; import debug from 'debug'; -import Decimal from 'decimal.js'; -import { - GraphQLScalarType, - GraphQLResolveInfo -} from 'graphql'; +import { GraphQLResolveInfo } from 'graphql'; import { ValueResult, @@ -17,6 +12,8 @@ import { gqlQueryCount, getResultState, IndexerInterface, + GraphQLBigInt, + GraphQLBigDecimal, EventWatcher, // eslint-disable-next-line @typescript-eslint/no-unused-vars setGQLCacheHints @@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher const gqlCacheConfig = indexer.serverConfig.gqlCache; return { - BigInt: new BigInt('bigInt'), + BigInt: GraphQLBigInt, - BigDecimal: new GraphQLScalarType({ - name: 'BigDecimal', - description: 'BigDecimal custom scalar type', - parseValue (value) { - // value from the client - return new Decimal(value); - }, - serialize (value: Decimal) { - // value sent to the client - return value.toFixed(); - } - }), + BigDecimal: GraphQLBigDecimal, Event: { __resolveType: (obj: any) => { @@ -217,9 +203,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher gqlTotalQueryCount.inc(1); gqlQueryCount.labels('eventsInRange').inc(1); - const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber); - if (expected !== actual) { - throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`); + const syncStatus = await indexer.getSyncStatus(); + + if (!syncStatus) { + throw new Error('No blocks processed yet'); + } + + if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) { + throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`); } const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber); diff --git a/packages/polls-watcher/src/schema.gql b/packages/polls-watcher/src/schema.gql index ea6c4e3..c1ef0ba 100644 --- a/packages/polls-watcher/src/schema.gql +++ b/packages/polls-watcher/src/schema.gql @@ -16,7 +16,7 @@ type Proof { } type _Block_ { - cid: String! + cid: String hash: String! number: Int! timestamp: Int! @@ -81,13 +81,6 @@ type ResultBoolean { proof: Proof } -type SyncStatus { - latestIndexedBlockHash: String! - latestIndexedBlockNumber: Int! - latestCanonicalBlockHash: String! - latestCanonicalBlockNumber: Int! -} - type ResultState { block: _Block_! contractAddress: String! @@ -96,6 +89,17 @@ type ResultState { data: String! } +type SyncStatus { + latestIndexedBlockHash: String! + latestIndexedBlockNumber: Int! + latestCanonicalBlockHash: String! + latestCanonicalBlockNumber: Int! + initialIndexedBlockHash: String! + initialIndexedBlockNumber: Int! + latestProcessedBlockHash: String! + latestProcessedBlockNumber: Int! +} + type Query { events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!] eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!] @@ -106,9 +110,9 @@ type Query { getDocumentMajorities(blockHash: String!, contractAddress: String!): ResultStringArray! hasVotedOnUpgradePoll(blockHash: String!, contractAddress: String!, _galaxy: Int!, _proposal: String!): ResultBoolean! hasVotedOnDocumentPoll(blockHash: String!, contractAddress: String!, _galaxy: Int!, _proposal: String!): ResultBoolean! - getSyncStatus: SyncStatus getStateByCID(cid: String!): ResultState getState(blockHash: String!, contractAddress: String!, kind: String): ResultState + getSyncStatus: SyncStatus } type Mutation { diff --git a/yarn.lock b/yarn.lock index 54ced74..45d14db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -183,17 +183,6 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@cerc-io/cache@^0.2.40": - version "0.2.40" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.40/cache-0.2.40.tgz#b44cc8323d1e1a7f6761efdca61b00d4f337823e" - integrity sha512-tQ5kYLTpwX/40hpNg1eEnojHtq2dl0nfkW78+lFUXiqtjc+sVFvO6nIqKLI8NBkeCMWFg3KyNDmv86xn/0mPgw== - dependencies: - canonical-json "^0.0.4" - debug "^4.3.1" - ethers "^5.4.4" - fs-extra "^10.0.0" - level "^7.0.0" - "@cerc-io/cache@^0.2.74": version "0.2.74" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.74/cache-0.2.74.tgz#035805fc289aa02cbc34e5ecf14e3728fa288763" @@ -205,25 +194,6 @@ fs-extra "^10.0.0" level "^7.0.0" -"@cerc-io/cli@^0.2.40": - version "0.2.40" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.40/cli-0.2.40.tgz#9116f1e190910df4aa1480e9ae28380ff7d59e4b" - integrity sha512-Sxxy86SjSb0QgCbEHdr9xFwhbfGEqv+lG3HBRizCobaIYGZveIWCFU34tzpQJCoEYorRJybkC1uqsJDDCUMCAg== - dependencies: - "@cerc-io/peer" "^0.2.40" - "@cerc-io/util" "^0.2.40" - "@ethersproject/providers" "^5.4.4" - "@graphql-tools/utils" "^9.1.1" - "@ipld/dag-cbor" "^8.0.0" - "@libp2p/interface-peer-id" "^2.0.0" - apollo-server-express "^3.11.1" - debug "^4.3.1" - express "^4.18.2" - graphql-subscriptions "^2.0.0" - reflect-metadata "^0.1.13" - typeorm "^0.2.32" - yargs "^17.0.1" - "@cerc-io/cli@^0.2.74": version "0.2.74" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.74/cli-0.2.74.tgz#db13cf6a2cacd144e5dfba80929a6511be27afe9" @@ -252,22 +222,6 @@ typeorm "0.2.37" yargs "^17.0.1" -"@cerc-io/ipld-eth-client@^0.2.40": - version "0.2.40" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.40/ipld-eth-client-0.2.40.tgz#aa3a01af6dcb6591388616d9c7e55ce721a44621" - integrity sha512-Oc+WkdeHkLG9fk7Uwwa4YHqBreYAEWu9CnRmqWm9sMwi48bnSEyDDzpuzUvg+R8RenSLlqaH/d9cBqhN9bInKw== - dependencies: - "@apollo/client" "^3.7.1" - "@cerc-io/cache" "^0.2.40" - cross-fetch "^3.1.4" - debug "^4.3.1" - ethers "^5.4.4" - graphql "^15.5.0" - graphql-ws "^5.11.2" - left-pad "^1.3.0" - ws "^8.11.0" - zen-observable-ts "^1.1.0" - "@cerc-io/ipld-eth-client@^0.2.74": version "0.2.74" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.74/ipld-eth-client-0.2.74.tgz#f96acb32937a035de2a54e32153420581c9b5375" @@ -285,78 +239,6 @@ ws "^8.11.0" zen-observable-ts "^1.1.0" -"@cerc-io/libp2p@0.42.2-laconic-0.1.3": - version "0.42.2-laconic-0.1.3" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Flibp2p/-/0.42.2-laconic-0.1.3/libp2p-0.42.2-laconic-0.1.3.tgz#482c88e135141788fdd201fc27fb4f1bdc954f53" - integrity sha512-D8nBtFVB2xUN/QRCSHQVSoJBAk2nH4caqROpWUipRxiDgeoBVpTqYEM9fBzOln6/Yl3BtBxR/fgyaZPTSJQdrg== - dependencies: - "@achingbrain/nat-port-mapper" "^1.0.3" - "@libp2p/crypto" "^1.0.4" - "@libp2p/interface-address-manager" "^2.0.0" - "@libp2p/interface-connection" "^3.0.2" - "@libp2p/interface-connection-encrypter" "^3.0.1" - "@libp2p/interface-connection-manager" "^1.1.1" - "@libp2p/interface-content-routing" "^2.0.0" - "@libp2p/interface-dht" "^2.0.0" - "@libp2p/interface-keychain" "^2.0.4" - "@libp2p/interface-libp2p" "^1.0.0" - "@libp2p/interface-metrics" "^4.0.0" - "@libp2p/interface-peer-discovery" "^1.0.1" - "@libp2p/interface-peer-id" "^2.0.0" - "@libp2p/interface-peer-info" "^1.0.3" - "@libp2p/interface-peer-routing" "^1.0.1" - "@libp2p/interface-peer-store" "^1.2.2" - "@libp2p/interface-pubsub" "^3.0.0" - "@libp2p/interface-registrar" "^2.0.3" - "@libp2p/interface-stream-muxer" "^3.0.0" - "@libp2p/interface-transport" "^2.1.0" - "@libp2p/interfaces" "^3.0.3" - "@libp2p/keychain" "^1.0.0" - "@libp2p/logger" "^2.0.1" - "@libp2p/multistream-select" "^3.0.0" - "@libp2p/peer-collections" "^3.0.0" - "@libp2p/peer-id" "^2.0.0" - "@libp2p/peer-id-factory" "^2.0.0" - "@libp2p/peer-record" "^5.0.0" - "@libp2p/peer-store" "^6.0.0" - "@libp2p/tracked-map" "^3.0.0" - "@libp2p/utils" "^3.0.2" - "@libp2p/webrtc-peer" "^2.0.2" - "@multiformats/mafmt" "^11.0.2" - "@multiformats/multiaddr" "^11.0.0" - abortable-iterator "^4.0.2" - any-signal "^3.0.0" - datastore-core "^8.0.1" - err-code "^3.0.1" - interface-datastore "^7.0.0" - it-all "^2.0.0" - it-drain "^2.0.0" - it-filter "^2.0.0" - it-first "^2.0.0" - it-handshake "^4.1.2" - it-length-prefixed "^8.0.2" - it-map "^2.0.0" - it-merge "^2.0.0" - it-pair "^2.0.2" - it-pipe "^2.0.3" - it-pushable "^3.1.2" - it-sort "^2.0.1" - it-stream-types "^1.0.4" - merge-options "^3.0.4" - multiformats "^11.0.0" - p-fifo "^1.0.0" - p-settle "^5.0.0" - private-ip "^3.0.0" - protons-runtime "^4.0.1" - rate-limiter-flexible "^2.3.11" - retimer "^3.0.0" - set-delayed-interval "^1.0.0" - timeout-abort-controller "^3.0.0" - uint8arraylist "^2.3.2" - uint8arrays "^4.0.2" - wherearewe "^2.0.0" - xsalsa20 "^1.1.0" - "@cerc-io/libp2p@0.42.2-laconic-0.1.4", "@cerc-io/libp2p@^0.42.2-laconic-0.1.4": version "0.42.2-laconic-0.1.4" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Flibp2p/-/0.42.2-laconic-0.1.4/libp2p-0.42.2-laconic-0.1.4.tgz#ac9347e70d6d3cee040ad02074cae3070b91a8fb" @@ -470,35 +352,6 @@ lodash "^4.17.21" uint8arrays "^4.0.3" -"@cerc-io/peer@^0.2.40": - version "0.2.40" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.40/peer-0.2.40.tgz#ba965079c2006195c92f5457e5976a8af54391c3" - integrity sha512-PFYNDqabyAZYdpOh5ZGLXR/v3jqSELLMHkU1lVFCUOGON8awOeXU/RJlMEt+v6BJjPo5Y7y60et4sUb7fo1MfA== - dependencies: - "@cerc-io/libp2p" "0.42.2-laconic-0.1.3" - "@cerc-io/prometheus-metrics" "1.1.4" - "@chainsafe/libp2p-noise" "^11.0.0" - "@libp2p/floodsub" "^6.0.0" - "@libp2p/mplex" "^7.1.1" - "@libp2p/peer-id-factory" "^2.0.0" - "@libp2p/pubsub-peer-discovery" "^8.0.0" - "@libp2p/websockets" "^5.0.5" - "@multiformats/multiaddr" "^11.1.4" - assert "^2.0.0" - buffer "^6.0.3" - chai "^4.3.4" - debug "^4.3.1" - dotenv "^16.0.3" - it-length-prefixed "^8.0.4" - it-map "^2.0.0" - it-pipe "^2.0.5" - it-pushable "^3.1.2" - mocha "^8.4.0" - p-event "^5.0.1" - uint8arrays "^4.0.3" - unique-names-generator "^4.7.1" - yargs "^17.0.1" - "@cerc-io/peer@^0.2.65", "@cerc-io/peer@^0.2.74": version "0.2.74" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fpeer/-/0.2.74/peer-0.2.74.tgz#cc54e513e1857b04630f6b11e9a65dcdcc532790" @@ -554,13 +407,6 @@ left-pad "^1.3.0" mocha "^8.4.0" -"@cerc-io/solidity-mapper@^0.2.40": - version "0.2.40" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.40/solidity-mapper-0.2.40.tgz#c4263f86cb3f1f2950118d348e3a9df39fcc861c" - integrity sha512-1gnBxlqaBSnHYlKzindV/lLc+UdJOjf2ytyYqxlT70w1444U39IQ18AWHlykMDCjgScczDTSNobFodQIdwwqNQ== - dependencies: - dotenv "^10.0.0" - "@cerc-io/solidity-mapper@^0.2.74": version "0.2.74" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.74/solidity-mapper-0.2.74.tgz#770115a57481b94a3d21eb00b17cb4b47eb441ad" @@ -573,43 +419,6 @@ resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fts-channel/-/1.0.3-ts-nitro-0.1.1/ts-channel-1.0.3-ts-nitro-0.1.1.tgz#0768781313a167295c0bf21307f47e02dc17e936" integrity sha512-2jFICUSyffuZ+8+qRhXuLSJq4GJ6Y02wxiXoubH0Kzv2lIKkJtWICY1ZQQhtXAvP0ncAQB85WJHqtqwH8l7J3Q== -"@cerc-io/util@^0.2.40": - version "0.2.40" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.40/util-0.2.40.tgz#5ad35b3c57dc3c0997141d3bb82ff2f86046fb9e" - integrity sha512-PeGYiEd7XEy3tj4YMAku0Jm8bNEeaTcntF0U8Cx1Vo9XntJls34lrwKiz+IU+Rt/sEwk0U0sQrKtgqZzjQ6Opg== - dependencies: - "@apollo/utils.keyvaluecache" "^1.0.1" - "@cerc-io/solidity-mapper" "^0.2.40" - "@ethersproject/providers" "^5.4.4" - "@graphql-tools/schema" "^9.0.10" - "@graphql-tools/utils" "^9.1.1" - "@ipld/dag-cbor" "^6.0.12" - apollo-server-core "^3.11.1" - apollo-server-express "^3.11.1" - apollo-server-plugin-response-cache "^3.8.1" - debug "^4.3.1" - decimal.js "^10.3.1" - ethers "^5.4.4" - express "^4.18.2" - express-queue "^0.0.13" - fs-extra "^10.0.0" - graphql "^15.5.0" - graphql-subscriptions "^2.0.0" - graphql-ws "^5.11.2" - ipfs-http-client "^56.0.3" - js-yaml "^4.1.0" - json-bigint "^1.0.0" - lodash "^4.17.21" - multiformats "^9.4.8" - pg "^8.5.1" - pg-boss "^6.1.0" - prom-client "^14.0.1" - toml "^3.0.0" - typeorm "^0.2.32" - typeorm-naming-strategies "^2.0.0" - ws "^8.11.0" - yargs "^17.0.1" - "@cerc-io/util@^0.2.74": version "0.2.74" resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.74/util-0.2.74.tgz#3f07fa529df9c2633e0b6b42e3d44779c8f05ae6" @@ -4820,11 +4629,6 @@ dotenv@^10.0.0, dotenv@~10.0.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== -dotenv@^16.0.3: - version "16.0.3" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.3.tgz#115aec42bac5053db3c456db30cc243a5a836a07" - integrity sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ== - dotenv@^8.2.0: version "8.6.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b" @@ -10642,29 +10446,6 @@ typeorm@0.2.37: yargs "^17.0.1" zen-observable-ts "^1.0.0" -typeorm@^0.2.32: - version "0.2.45" - resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.2.45.tgz#e5bbb3af822dc4646bad96cfa48cd22fa4687cea" - integrity sha512-c0rCO8VMJ3ER7JQ73xfk0zDnVv0WDjpsP6Q1m6CVKul7DB9iVdWLRjPzc8v2eaeBuomsbZ2+gTaYr8k1gm3bYA== - dependencies: - "@sqltools/formatter" "^1.2.2" - app-root-path "^3.0.0" - buffer "^6.0.3" - chalk "^4.1.0" - cli-highlight "^2.1.11" - debug "^4.3.1" - dotenv "^8.2.0" - glob "^7.1.6" - js-yaml "^4.0.0" - mkdirp "^1.0.4" - reflect-metadata "^0.1.13" - sha.js "^2.4.11" - tslib "^2.1.0" - uuid "^8.3.2" - xml2js "^0.4.23" - yargs "^17.0.1" - zen-observable-ts "^1.0.0" - "typescript@^3 || ^4", typescript@^4.9.4: version "4.9.5" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a"