Update watchers with fix for historical processing sync status update (#40)

* Update azimuth watcher with fix for historical processing sync status

* Upgrade packages in all watchers

* Upgrade package versions to 0.2.76

* Update watcher version to 0.1.2

lerna.json

@@ -3,6 +3,6 @@
     "packages/*"
   ],
   "useWorkspaces": true,
-  "version": "0.0.0",
+  "version": "0.1.2",
   "npmClient": "yarn"
 }

package.json

@@ -2,9 +2,15 @@
   "name": "azimuth-watcher-ts",
   "private": true,
   "license": "AGPL-3.0",
-  "workspaces": [
-    "packages/*"
-  ],
+  "workspaces": {
+    "packages": [
+      "packages/*"
+    ],
+    "nohoist": [
+      "**/@cerc-io/rpc-eth-client",
+      "**/typeorm-naming-strategies"
+    ]
+  },
   "scripts": {
     "build": "lerna run build --stream",
     "lint": "lerna run lint --stream -- --max-warnings=0",

packages/azimuth-watcher/README.md

@@ -81,7 +81,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3001/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -97,7 +97,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3001/graphql
 
   * To watch a contract:
 

packages/azimuth-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3001
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/azimuth-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/azimuth-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "azimuth-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/azimuth-watcher/src/database.ts

@@ -888,10 +888,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -912,6 +912,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/azimuth-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/azimuth-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/azimuth-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/azimuth-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/azimuth-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/azimuth-watcher/src/gql/queries/index.ts

@@ -47,6 +47,6 @@ export const canTransfer = fs.readFileSync(path.join(__dirname, 'canTransfer.gql
 export const getTransferringForCount = fs.readFileSync(path.join(__dirname, 'getTransferringForCount.gql'), 'utf8');
 export const getTransferringFor = fs.readFileSync(path.join(__dirname, 'getTransferringFor.gql'), 'utf8');
 export const isOperator = fs.readFileSync(path.join(__dirname, 'isOperator.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');

packages/azimuth-watcher/src/indexer.ts

@@ -6,11 +6,10 @@ import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import debug from 'debug';
 import JSONbig from 'json-bigint';
-import { ethers } from 'ethers';
+import { ethers, constants } from 'ethers';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -25,7 +24,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  EthClient,
+  UpstreamConfig,
+  EthFullBlock,
+  EthFullTransaction,
+  ExtraEventData
 } from '@cerc-io/util';
 
 import AzimuthArtifacts from './artifacts/Azimuth.json';
@@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface {
   _ethProvider: BaseProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
+  _upstreamConfig: UpstreamConfig;
 
   _abiMap: Map<string, JsonFragment[]>;
   _storageLayoutMap: Map<string, StorageLayout>;
   _contractMap: Map<string, ethers.utils.Interface>;
+  eventSignaturesMap: Map<string, string[]>;
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) {
+  constructor (
+    config: {
+      server: ServerConfig;
+      upstream: UpstreamConfig;
+    },
+    db: DatabaseInterface,
+    clients: Clients,
+    ethProvider: BaseProvider,
+    jobQueue: JobQueue
+  ) {
     assert(db);
     assert(clients.ethClient);
 
     this._db = db as Database;
     this._ethClient = clients.ethClient;
     this._ethProvider = ethProvider;
-    this._serverConfig = serverConfig;
-    this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
+    this._serverConfig = config.server;
+    this._upstreamConfig = config.upstream;
+    this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
+    this.eventSignaturesMap = new Map();
 
     const { abi: AzimuthABI } = AzimuthArtifacts;
 
     assert(AzimuthABI);
     this._abiMap.set(KIND_AZIMUTH, AzimuthABI);
-    this._contractMap.set(KIND_AZIMUTH, new ethers.utils.Interface(AzimuthABI));
+
+    const AzimuthContractInterface = new ethers.utils.Interface(AzimuthABI);
+    this._contractMap.set(KIND_AZIMUTH, AzimuthContractInterface);
+
+    const AzimuthEventSignatures = Object.values(AzimuthContractInterface.events).map(value => {
+      return AzimuthContractInterface.getEventTopic(value);
+    });
+    this.eventSignaturesMap.set(KIND_AZIMUTH, AzimuthEventSignatures);
   }
 
   get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
+  get upstreamConfig (): UpstreamConfig {
+    return this._upstreamConfig;
+  }
+
   get storageLayoutMap (): Map<string, StorageLayout> {
     return this._storageLayoutMap;
   }
@@ -1548,16 +1576,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.removeStates(blockNumber, kind);
   }
 
-  async triggerIndexingOnEvent (event: Event): Promise<void> {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     // Call custom hook function for indexing on event.
     await handleEvent(this, resultEvent);
   }
 
-  async processEvent (event: Event): Promise<void> {
+  async processEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     // Trigger indexing of data based on the event.
-    await this.triggerIndexingOnEvent(event);
+    await this.triggerIndexingOnEvent(event, extraData);
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
@@ -1588,7 +1617,11 @@ export class Indexer implements IndexerInterface {
     return this._db.getStateSyncStatus();
   }
 
-  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus | undefined> {
+    if (!this._serverConfig.enableState) {
+      return;
+    }
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -1622,10 +1655,14 @@ export class Indexer implements IndexerInterface {
     return res;
   }
 
-  async getLatestCanonicalBlock (): Promise<BlockProgress> {
+  async getLatestCanonicalBlock (): Promise<BlockProgress | undefined> {
     const syncStatus = await this.getSyncStatus();
     assert(syncStatus);
 
+    if (syncStatus.latestCanonicalBlockHash === constants.HashZero) {
+      return;
+    }
+
     const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
     assert(latestCanonicalBlock);
 
@@ -1636,8 +1673,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getLatestStateIndexedBlock();
   }
 
-  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
@@ -1652,6 +1689,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.saveEventEntity(dbEvent);
   }
 
+  async saveEvents (dbEvents: Event[]): Promise<void> {
+    return this._baseIndexer.saveEvents(dbEvents);
+  }
+
   async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise<Array<Event>> {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
@@ -1660,6 +1701,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
+  getWatchedContracts (): Contract[] {
+    return this._baseIndexer.getWatchedContracts();
+  }
+
   getContractsByKind (kind: string): Contract[] {
     return this._baseIndexer.getContractsByKind(kind);
   }
@@ -1694,6 +1739,18 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
+  }
+
+  async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatus(syncStatus);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -1710,7 +1767,24 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{
+    blockProgress: BlockProgress,
+    events: DeepPartial<Event>[],
+    ethFullBlock: EthFullBlock;
+    ethFullTransactions: EthFullTransaction[];
+  }[]> {
+    return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise<DeepPartial<Event>[]> {
+    return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     return this._saveBlockAndFetchEvents(block);
   }
 
@@ -1739,17 +1813,26 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 
+  async clearProcessedBlockData (block: BlockProgress): Promise<void> {
+    const entities = [...ENTITIES];
+    await this._baseIndexer.clearProcessedBlockData(block, entities);
+  }
+
   async _saveBlockAndFetchEvents ({
     cid: blockCid,
     blockHash,
     blockNumber,
     blockTimestamp,
     parentHash
-  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  }: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     assert(blockHash);
     assert(blockNumber);
 
-    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this));
+    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
     try {
@@ -1766,7 +1849,7 @@ export class Indexer implements IndexerInterface {
       await dbTx.commitTransaction();
       console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return [blockProgress, []];
+      return [blockProgress, [], transactions];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/azimuth-watcher/src/job-runner.ts

@@ -20,6 +20,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();

packages/azimuth-watcher/src/resolvers.ts

@@ -3,13 +3,8 @@
 //
 
 import assert from 'assert';
-import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
-import Decimal from 'decimal.js';
-import {
-  GraphQLScalarType,
-  GraphQLResolveInfo
-} from 'graphql';
+import { GraphQLResolveInfo } from 'graphql';
 
 import {
   ValueResult,
@@ -17,6 +12,8 @@ import {
   gqlQueryCount,
   getResultState,
   IndexerInterface,
+  GraphQLBigInt,
+  GraphQLBigDecimal,
   EventWatcher,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   setGQLCacheHints
@@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
   const gqlCacheConfig = indexer.serverConfig.gqlCache;
 
   return {
-    BigInt: new BigInt('bigInt'),
+    BigInt: GraphQLBigInt,
 
-    BigDecimal: new GraphQLScalarType({
-      name: 'BigDecimal',
-      description: 'BigDecimal custom scalar type',
-      parseValue (value) {
-        // value from the client
-        return new Decimal(value);
-      },
-      serialize (value: Decimal) {
-        // value sent to the client
-        return value.toFixed();
-      }
-    }),
+    BigDecimal: GraphQLBigDecimal,
 
     Event: {
       __resolveType: (obj: any) => {
@@ -883,9 +869,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('eventsInRange').inc(1);
 
-        const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
-        if (expected !== actual) {
-          throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`);
+        const syncStatus = await indexer.getSyncStatus();
+
+        if (!syncStatus) {
+          throw new Error('No blocks processed yet');
+        }
+
+        if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) {
+          throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`);
         }
 
         const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);

packages/azimuth-watcher/src/schema.gql

@@ -16,7 +16,7 @@ type Proof {
 }
 
 type _Block_ {
-  cid: String!
+  cid: String
   hash: String!
   number: Int!
   timestamp: Int!
@@ -155,13 +155,6 @@ type ResultString {
   proof: Proof
 }
 
-type SyncStatus {
-  latestIndexedBlockHash: String!
-  latestIndexedBlockNumber: Int!
-  latestCanonicalBlockHash: String!
-  latestCanonicalBlockNumber: Int!
-}
-
 type ResultState {
   block: _Block_!
   contractAddress: String!
@@ -170,6 +163,17 @@ type ResultState {
   data: String!
 }
 
+type SyncStatus {
+  latestIndexedBlockHash: String!
+  latestIndexedBlockNumber: Int!
+  latestCanonicalBlockHash: String!
+  latestCanonicalBlockNumber: Int!
+  initialIndexedBlockHash: String!
+  initialIndexedBlockNumber: Int!
+  latestProcessedBlockHash: String!
+  latestProcessedBlockNumber: Int!
+}
+
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
@@ -217,9 +221,9 @@ type Query {
   getTransferringForCount(blockHash: String!, contractAddress: String!, _proxy: String!): ResultBigInt!
   getTransferringFor(blockHash: String!, contractAddress: String!, _proxy: String!): ResultBigIntArray!
   isOperator(blockHash: String!, contractAddress: String!, _owner: String!, _operator: String!): ResultBoolean!
-  getSyncStatus: SyncStatus
   getStateByCID(cid: String!): ResultState
   getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
+  getSyncStatus: SyncStatus
 }
 
 type Mutation {

packages/censures-watcher/README.md

@@ -81,7 +81,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3002/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -97,7 +97,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3002/graphql
 
   * To watch a contract:
 

packages/censures-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3002
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/censures-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/censures-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "censures-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/censures-watcher/src/database.ts

@@ -235,10 +235,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -259,6 +259,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/censures-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/censures-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/censures-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/censures-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/censures-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/censures-watcher/src/gql/queries/index.ts

@@ -7,6 +7,6 @@ export const getCensuringCount = fs.readFileSync(path.join(__dirname, 'getCensur
 export const getCensuring = fs.readFileSync(path.join(__dirname, 'getCensuring.gql'), 'utf8');
 export const getCensuredByCount = fs.readFileSync(path.join(__dirname, 'getCensuredByCount.gql'), 'utf8');
 export const getCensuredBy = fs.readFileSync(path.join(__dirname, 'getCensuredBy.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');

packages/censures-watcher/src/indexer.ts

@@ -6,11 +6,10 @@ import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import debug from 'debug';
 import JSONbig from 'json-bigint';
-import { ethers } from 'ethers';
+import { ethers, constants } from 'ethers';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -25,7 +24,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  EthClient,
+  UpstreamConfig,
+  EthFullBlock,
+  EthFullTransaction,
+  ExtraEventData
 } from '@cerc-io/util';
 
 import CensuresArtifacts from './artifacts/Censures.json';
@@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface {
   _ethProvider: BaseProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
+  _upstreamConfig: UpstreamConfig;
 
   _abiMap: Map<string, JsonFragment[]>;
   _storageLayoutMap: Map<string, StorageLayout>;
   _contractMap: Map<string, ethers.utils.Interface>;
+  eventSignaturesMap: Map<string, string[]>;
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) {
+  constructor (
+    config: {
+      server: ServerConfig;
+      upstream: UpstreamConfig;
+    },
+    db: DatabaseInterface,
+    clients: Clients,
+    ethProvider: BaseProvider,
+    jobQueue: JobQueue
+  ) {
     assert(db);
     assert(clients.ethClient);
 
     this._db = db as Database;
     this._ethClient = clients.ethClient;
     this._ethProvider = ethProvider;
-    this._serverConfig = serverConfig;
-    this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
+    this._serverConfig = config.server;
+    this._upstreamConfig = config.upstream;
+    this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
+    this.eventSignaturesMap = new Map();
 
     const { abi: CensuresABI } = CensuresArtifacts;
 
     assert(CensuresABI);
     this._abiMap.set(KIND_CENSURES, CensuresABI);
-    this._contractMap.set(KIND_CENSURES, new ethers.utils.Interface(CensuresABI));
+
+    const CensuresContractInterface = new ethers.utils.Interface(CensuresABI);
+    this._contractMap.set(KIND_CENSURES, CensuresContractInterface);
+
+    const CensuresEventSignatures = Object.values(CensuresContractInterface.events).map(value => {
+      return CensuresContractInterface.getEventTopic(value);
+    });
+    this.eventSignaturesMap.set(KIND_CENSURES, CensuresEventSignatures);
   }
 
   get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
+  get upstreamConfig (): UpstreamConfig {
+    return this._upstreamConfig;
+  }
+
   get storageLayoutMap (): Map<string, StorageLayout> {
     return this._storageLayoutMap;
   }
@@ -326,16 +354,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.removeStates(blockNumber, kind);
   }
 
-  async triggerIndexingOnEvent (event: Event): Promise<void> {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     // Call custom hook function for indexing on event.
     await handleEvent(this, resultEvent);
   }
 
-  async processEvent (event: Event): Promise<void> {
+  async processEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     // Trigger indexing of data based on the event.
-    await this.triggerIndexingOnEvent(event);
+    await this.triggerIndexingOnEvent(event, extraData);
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
@@ -366,7 +395,11 @@ export class Indexer implements IndexerInterface {
     return this._db.getStateSyncStatus();
   }
 
-  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus | undefined> {
+    if (!this._serverConfig.enableState) {
+      return;
+    }
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -400,10 +433,14 @@ export class Indexer implements IndexerInterface {
     return res;
   }
 
-  async getLatestCanonicalBlock (): Promise<BlockProgress> {
+  async getLatestCanonicalBlock (): Promise<BlockProgress | undefined> {
     const syncStatus = await this.getSyncStatus();
     assert(syncStatus);
 
+    if (syncStatus.latestCanonicalBlockHash === constants.HashZero) {
+      return;
+    }
+
     const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
     assert(latestCanonicalBlock);
 
@@ -414,8 +451,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getLatestStateIndexedBlock();
   }
 
-  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
@@ -430,6 +467,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.saveEventEntity(dbEvent);
   }
 
+  async saveEvents (dbEvents: Event[]): Promise<void> {
+    return this._baseIndexer.saveEvents(dbEvents);
+  }
+
   async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise<Array<Event>> {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
@@ -438,6 +479,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
+  getWatchedContracts (): Contract[] {
+    return this._baseIndexer.getWatchedContracts();
+  }
+
   getContractsByKind (kind: string): Contract[] {
     return this._baseIndexer.getContractsByKind(kind);
   }
@@ -472,6 +517,18 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
+  }
+
+  async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatus(syncStatus);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -488,7 +545,24 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{
+    blockProgress: BlockProgress,
+    events: DeepPartial<Event>[],
+    ethFullBlock: EthFullBlock;
+    ethFullTransactions: EthFullTransaction[];
+  }[]> {
+    return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise<DeepPartial<Event>[]> {
+    return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     return this._saveBlockAndFetchEvents(block);
   }
 
@@ -517,17 +591,26 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 
+  async clearProcessedBlockData (block: BlockProgress): Promise<void> {
+    const entities = [...ENTITIES];
+    await this._baseIndexer.clearProcessedBlockData(block, entities);
+  }
+
   async _saveBlockAndFetchEvents ({
     cid: blockCid,
     blockHash,
     blockNumber,
     blockTimestamp,
     parentHash
-  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  }: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     assert(blockHash);
     assert(blockNumber);
 
-    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this));
+    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
     try {
@@ -544,7 +627,7 @@ export class Indexer implements IndexerInterface {
       await dbTx.commitTransaction();
       console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return [blockProgress, []];
+      return [blockProgress, [], transactions];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/censures-watcher/src/job-runner.ts

@@ -20,6 +20,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();

packages/censures-watcher/src/resolvers.ts

@@ -3,13 +3,8 @@
 //
 
 import assert from 'assert';
-import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
-import Decimal from 'decimal.js';
-import {
-  GraphQLScalarType,
-  GraphQLResolveInfo
-} from 'graphql';
+import { GraphQLResolveInfo } from 'graphql';
 
 import {
   ValueResult,
@@ -17,6 +12,8 @@ import {
   gqlQueryCount,
   getResultState,
   IndexerInterface,
+  GraphQLBigInt,
+  GraphQLBigDecimal,
   EventWatcher,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   setGQLCacheHints
@@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
   const gqlCacheConfig = indexer.serverConfig.gqlCache;
 
   return {
-    BigInt: new BigInt('bigInt'),
+    BigInt: GraphQLBigInt,
 
-    BigDecimal: new GraphQLScalarType({
-      name: 'BigDecimal',
-      description: 'BigDecimal custom scalar type',
-      parseValue (value) {
-        // value from the client
-        return new Decimal(value);
-      },
-      serialize (value: Decimal) {
-        // value sent to the client
-        return value.toFixed();
-      }
-    }),
+    BigDecimal: GraphQLBigDecimal,
 
     Event: {
       __resolveType: (obj: any) => {
@@ -163,9 +149,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('eventsInRange').inc(1);
 
-        const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
-        if (expected !== actual) {
-          throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`);
+        const syncStatus = await indexer.getSyncStatus();
+
+        if (!syncStatus) {
+          throw new Error('No blocks processed yet');
+        }
+
+        if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) {
+          throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`);
         }
 
         const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);

packages/censures-watcher/src/schema.gql

@@ -16,7 +16,7 @@ type Proof {
 }
 
 type _Block_ {
-  cid: String!
+  cid: String
   hash: String!
   number: Int!
   timestamp: Int!
@@ -66,13 +66,6 @@ type ResultIntArray {
   proof: Proof
 }
 
-type SyncStatus {
-  latestIndexedBlockHash: String!
-  latestIndexedBlockNumber: Int!
-  latestCanonicalBlockHash: String!
-  latestCanonicalBlockNumber: Int!
-}
-
 type ResultState {
   block: _Block_!
   contractAddress: String!
@@ -81,6 +74,17 @@ type ResultState {
   data: String!
 }
 
+type SyncStatus {
+  latestIndexedBlockHash: String!
+  latestIndexedBlockNumber: Int!
+  latestCanonicalBlockHash: String!
+  latestCanonicalBlockNumber: Int!
+  initialIndexedBlockHash: String!
+  initialIndexedBlockNumber: Int!
+  latestProcessedBlockHash: String!
+  latestProcessedBlockNumber: Int!
+}
+
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
@@ -88,9 +92,9 @@ type Query {
   getCensuring(blockHash: String!, contractAddress: String!, _whose: Int!): ResultBigIntArray!
   getCensuredByCount(blockHash: String!, contractAddress: String!, _who: Int!): ResultBigInt!
   getCensuredBy(blockHash: String!, contractAddress: String!, _who: Int!): ResultIntArray!
-  getSyncStatus: SyncStatus
   getStateByCID(cid: String!): ResultState
   getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
+  getSyncStatus: SyncStatus
 }
 
 type Mutation {

packages/claims-watcher/README.md

@@ -81,7 +81,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3003/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -97,7 +97,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3003/graphql
 
   * To watch a contract:
 

packages/claims-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3003
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/claims-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/claims-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "claims-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/claims-watcher/src/database.ts

@@ -189,10 +189,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -213,6 +213,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/claims-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/claims-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/claims-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/claims-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/claims-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/claims-watcher/src/gql/queries/index.ts

@@ -4,6 +4,6 @@ import path from 'path';
 export const events = fs.readFileSync(path.join(__dirname, 'events.gql'), 'utf8');
 export const eventsInRange = fs.readFileSync(path.join(__dirname, 'eventsInRange.gql'), 'utf8');
 export const findClaim = fs.readFileSync(path.join(__dirname, 'findClaim.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');

packages/claims-watcher/src/indexer.ts

@@ -6,11 +6,10 @@ import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import debug from 'debug';
 import JSONbig from 'json-bigint';
-import { ethers } from 'ethers';
+import { ethers, constants } from 'ethers';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -25,7 +24,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  EthClient,
+  UpstreamConfig,
+  EthFullBlock,
+  EthFullTransaction,
+  ExtraEventData
 } from '@cerc-io/util';
 
 import ClaimsArtifacts from './artifacts/Claims.json';
@@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface {
   _ethProvider: BaseProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
+  _upstreamConfig: UpstreamConfig;
 
   _abiMap: Map<string, JsonFragment[]>;
   _storageLayoutMap: Map<string, StorageLayout>;
   _contractMap: Map<string, ethers.utils.Interface>;
+  eventSignaturesMap: Map<string, string[]>;
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) {
+  constructor (
+    config: {
+      server: ServerConfig;
+      upstream: UpstreamConfig;
+    },
+    db: DatabaseInterface,
+    clients: Clients,
+    ethProvider: BaseProvider,
+    jobQueue: JobQueue
+  ) {
     assert(db);
     assert(clients.ethClient);
 
     this._db = db as Database;
     this._ethClient = clients.ethClient;
     this._ethProvider = ethProvider;
-    this._serverConfig = serverConfig;
-    this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
+    this._serverConfig = config.server;
+    this._upstreamConfig = config.upstream;
+    this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
+    this.eventSignaturesMap = new Map();
 
     const { abi: ClaimsABI } = ClaimsArtifacts;
 
     assert(ClaimsABI);
     this._abiMap.set(KIND_CLAIMS, ClaimsABI);
-    this._contractMap.set(KIND_CLAIMS, new ethers.utils.Interface(ClaimsABI));
+
+    const ClaimsContractInterface = new ethers.utils.Interface(ClaimsABI);
+    this._contractMap.set(KIND_CLAIMS, ClaimsContractInterface);
+
+    const ClaimsEventSignatures = Object.values(ClaimsContractInterface.events).map(value => {
+      return ClaimsContractInterface.getEventTopic(value);
+    });
+    this.eventSignaturesMap.set(KIND_CLAIMS, ClaimsEventSignatures);
   }
 
   get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
+  get upstreamConfig (): UpstreamConfig {
+    return this._upstreamConfig;
+  }
+
   get storageLayoutMap (): Map<string, StorageLayout> {
     return this._storageLayoutMap;
   }
@@ -234,16 +262,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.removeStates(blockNumber, kind);
   }
 
-  async triggerIndexingOnEvent (event: Event): Promise<void> {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     // Call custom hook function for indexing on event.
     await handleEvent(this, resultEvent);
   }
 
-  async processEvent (event: Event): Promise<void> {
+  async processEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     // Trigger indexing of data based on the event.
-    await this.triggerIndexingOnEvent(event);
+    await this.triggerIndexingOnEvent(event, extraData);
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
@@ -274,7 +303,11 @@ export class Indexer implements IndexerInterface {
     return this._db.getStateSyncStatus();
   }
 
-  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus | undefined> {
+    if (!this._serverConfig.enableState) {
+      return;
+    }
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -308,10 +341,14 @@ export class Indexer implements IndexerInterface {
     return res;
   }
 
-  async getLatestCanonicalBlock (): Promise<BlockProgress> {
+  async getLatestCanonicalBlock (): Promise<BlockProgress | undefined> {
     const syncStatus = await this.getSyncStatus();
     assert(syncStatus);
 
+    if (syncStatus.latestCanonicalBlockHash === constants.HashZero) {
+      return;
+    }
+
     const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
     assert(latestCanonicalBlock);
 
@@ -322,8 +359,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getLatestStateIndexedBlock();
   }
 
-  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
@@ -338,6 +375,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.saveEventEntity(dbEvent);
   }
 
+  async saveEvents (dbEvents: Event[]): Promise<void> {
+    return this._baseIndexer.saveEvents(dbEvents);
+  }
+
   async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise<Array<Event>> {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
@@ -346,6 +387,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
+  getWatchedContracts (): Contract[] {
+    return this._baseIndexer.getWatchedContracts();
+  }
+
   getContractsByKind (kind: string): Contract[] {
     return this._baseIndexer.getContractsByKind(kind);
   }
@@ -380,6 +425,18 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
+  }
+
+  async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatus(syncStatus);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -396,7 +453,24 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{
+    blockProgress: BlockProgress,
+    events: DeepPartial<Event>[],
+    ethFullBlock: EthFullBlock;
+    ethFullTransactions: EthFullTransaction[];
+  }[]> {
+    return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise<DeepPartial<Event>[]> {
+    return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     return this._saveBlockAndFetchEvents(block);
   }
 
@@ -425,17 +499,26 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 
+  async clearProcessedBlockData (block: BlockProgress): Promise<void> {
+    const entities = [...ENTITIES];
+    await this._baseIndexer.clearProcessedBlockData(block, entities);
+  }
+
   async _saveBlockAndFetchEvents ({
     cid: blockCid,
     blockHash,
     blockNumber,
     blockTimestamp,
     parentHash
-  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  }: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     assert(blockHash);
     assert(blockNumber);
 
-    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this));
+    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
     try {
@@ -452,7 +535,7 @@ export class Indexer implements IndexerInterface {
       await dbTx.commitTransaction();
       console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return [blockProgress, []];
+      return [blockProgress, [], transactions];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/claims-watcher/src/job-runner.ts

@@ -20,6 +20,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();

packages/claims-watcher/src/resolvers.ts

@@ -3,13 +3,8 @@
 //
 
 import assert from 'assert';
-import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
-import Decimal from 'decimal.js';
-import {
-  GraphQLScalarType,
-  GraphQLResolveInfo
-} from 'graphql';
+import { GraphQLResolveInfo } from 'graphql';
 
 import {
   ValueResult,
@@ -17,6 +12,8 @@ import {
   gqlQueryCount,
   getResultState,
   IndexerInterface,
+  GraphQLBigInt,
+  GraphQLBigDecimal,
   EventWatcher,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   setGQLCacheHints
@@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
   const gqlCacheConfig = indexer.serverConfig.gqlCache;
 
   return {
-    BigInt: new BigInt('bigInt'),
+    BigInt: GraphQLBigInt,
 
-    BigDecimal: new GraphQLScalarType({
-      name: 'BigDecimal',
-      description: 'BigDecimal custom scalar type',
-      parseValue (value) {
-        // value from the client
-        return new Decimal(value);
-      },
-      serialize (value: Decimal) {
-        // value sent to the client
-        return value.toFixed();
-      }
-    }),
+    BigDecimal: GraphQLBigDecimal,
 
     Event: {
       __resolveType: (obj: any) => {
@@ -109,9 +95,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('eventsInRange').inc(1);
 
-        const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
-        if (expected !== actual) {
-          throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`);
+        const syncStatus = await indexer.getSyncStatus();
+
+        if (!syncStatus) {
+          throw new Error('No blocks processed yet');
+        }
+
+        if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) {
+          throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`);
         }
 
         const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);

packages/claims-watcher/src/schema.gql

@@ -16,7 +16,7 @@ type Proof {
 }
 
 type _Block_ {
-  cid: String!
+  cid: String
   hash: String!
   number: Int!
   timestamp: Int!
@@ -59,13 +59,6 @@ type ResultInt {
   proof: Proof
 }
 
-type SyncStatus {
-  latestIndexedBlockHash: String!
-  latestIndexedBlockNumber: Int!
-  latestCanonicalBlockHash: String!
-  latestCanonicalBlockNumber: Int!
-}
-
 type ResultState {
   block: _Block_!
   contractAddress: String!
@@ -74,13 +67,24 @@ type ResultState {
   data: String!
 }
 
+type SyncStatus {
+  latestIndexedBlockHash: String!
+  latestIndexedBlockNumber: Int!
+  latestCanonicalBlockHash: String!
+  latestCanonicalBlockNumber: Int!
+  initialIndexedBlockHash: String!
+  initialIndexedBlockNumber: Int!
+  latestProcessedBlockHash: String!
+  latestProcessedBlockNumber: Int!
+}
+
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
   findClaim(blockHash: String!, contractAddress: String!, _whose: BigInt!, _protocol: String!, _claim: String!): ResultInt!
-  getSyncStatus: SyncStatus
   getStateByCID(cid: String!): ResultState
   getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
+  getSyncStatus: SyncStatus
 }
 
 type Mutation {

packages/conditional-star-release-watcher/README.md

@@ -81,7 +81,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3004/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -97,7 +97,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3004/graphql
 
   * To watch a contract:
 

packages/conditional-star-release-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3004
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/conditional-star-release-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/conditional-star-release-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "conditional-star-release-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/conditional-star-release-watcher/src/database.ts

@@ -334,10 +334,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -358,6 +358,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/conditional-star-release-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/conditional-star-release-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/conditional-star-release-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/conditional-star-release-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/conditional-star-release-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/conditional-star-release-watcher/src/gql/queries/index.ts

@@ -13,6 +13,6 @@ export const getForfeited = fs.readFileSync(path.join(__dirname, 'getForfeited.g
 export const hasForfeitedBatch = fs.readFileSync(path.join(__dirname, 'hasForfeitedBatch.gql'), 'utf8');
 export const getRemainingStars = fs.readFileSync(path.join(__dirname, 'getRemainingStars.gql'), 'utf8');
 export const getConditionsState = fs.readFileSync(path.join(__dirname, 'getConditionsState.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');

packages/conditional-star-release-watcher/src/indexer.ts

@@ -6,11 +6,10 @@ import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import debug from 'debug';
 import JSONbig from 'json-bigint';
-import { ethers } from 'ethers';
+import { ethers, constants } from 'ethers';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -25,7 +24,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  EthClient,
+  UpstreamConfig,
+  EthFullBlock,
+  EthFullTransaction,
+  ExtraEventData
 } from '@cerc-io/util';
 
 import ConditionalStarReleaseArtifacts from './artifacts/ConditionalStarRelease.json';
@@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface {
   _ethProvider: BaseProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
+  _upstreamConfig: UpstreamConfig;
 
   _abiMap: Map<string, JsonFragment[]>;
   _storageLayoutMap: Map<string, StorageLayout>;
   _contractMap: Map<string, ethers.utils.Interface>;
+  eventSignaturesMap: Map<string, string[]>;
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) {
+  constructor (
+    config: {
+      server: ServerConfig;
+      upstream: UpstreamConfig;
+    },
+    db: DatabaseInterface,
+    clients: Clients,
+    ethProvider: BaseProvider,
+    jobQueue: JobQueue
+  ) {
     assert(db);
     assert(clients.ethClient);
 
     this._db = db as Database;
     this._ethClient = clients.ethClient;
     this._ethProvider = ethProvider;
-    this._serverConfig = serverConfig;
-    this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
+    this._serverConfig = config.server;
+    this._upstreamConfig = config.upstream;
+    this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
+    this.eventSignaturesMap = new Map();
 
     const { abi: ConditionalStarReleaseABI } = ConditionalStarReleaseArtifacts;
 
     assert(ConditionalStarReleaseABI);
     this._abiMap.set(KIND_CONDITIONALSTARRELEASE, ConditionalStarReleaseABI);
-    this._contractMap.set(KIND_CONDITIONALSTARRELEASE, new ethers.utils.Interface(ConditionalStarReleaseABI));
+
+    const ConditionalStarReleaseContractInterface = new ethers.utils.Interface(ConditionalStarReleaseABI);
+    this._contractMap.set(KIND_CONDITIONALSTARRELEASE, ConditionalStarReleaseContractInterface);
+
+    const ConditionalStarReleaseEventSignatures = Object.values(ConditionalStarReleaseContractInterface.events).map(value => {
+      return ConditionalStarReleaseContractInterface.getEventTopic(value);
+    });
+    this.eventSignaturesMap.set(KIND_CONDITIONALSTARRELEASE, ConditionalStarReleaseEventSignatures);
   }
 
   get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
+  get upstreamConfig (): UpstreamConfig {
+    return this._upstreamConfig;
+  }
+
   get storageLayoutMap (): Map<string, StorageLayout> {
     return this._storageLayoutMap;
   }
@@ -515,16 +543,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.removeStates(blockNumber, kind);
   }
 
-  async triggerIndexingOnEvent (event: Event): Promise<void> {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     // Call custom hook function for indexing on event.
     await handleEvent(this, resultEvent);
   }
 
-  async processEvent (event: Event): Promise<void> {
+  async processEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     // Trigger indexing of data based on the event.
-    await this.triggerIndexingOnEvent(event);
+    await this.triggerIndexingOnEvent(event, extraData);
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
@@ -555,7 +584,11 @@ export class Indexer implements IndexerInterface {
     return this._db.getStateSyncStatus();
   }
 
-  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus | undefined> {
+    if (!this._serverConfig.enableState) {
+      return;
+    }
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -589,10 +622,14 @@ export class Indexer implements IndexerInterface {
     return res;
   }
 
-  async getLatestCanonicalBlock (): Promise<BlockProgress> {
+  async getLatestCanonicalBlock (): Promise<BlockProgress | undefined> {
     const syncStatus = await this.getSyncStatus();
     assert(syncStatus);
 
+    if (syncStatus.latestCanonicalBlockHash === constants.HashZero) {
+      return;
+    }
+
     const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
     assert(latestCanonicalBlock);
 
@@ -603,8 +640,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getLatestStateIndexedBlock();
   }
 
-  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
@@ -619,6 +656,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.saveEventEntity(dbEvent);
   }
 
+  async saveEvents (dbEvents: Event[]): Promise<void> {
+    return this._baseIndexer.saveEvents(dbEvents);
+  }
+
   async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise<Array<Event>> {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
@@ -627,6 +668,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
+  getWatchedContracts (): Contract[] {
+    return this._baseIndexer.getWatchedContracts();
+  }
+
   getContractsByKind (kind: string): Contract[] {
     return this._baseIndexer.getContractsByKind(kind);
   }
@@ -661,6 +706,18 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
+  }
+
+  async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatus(syncStatus);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -677,7 +734,24 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{
+    blockProgress: BlockProgress,
+    events: DeepPartial<Event>[],
+    ethFullBlock: EthFullBlock;
+    ethFullTransactions: EthFullTransaction[];
+  }[]> {
+    return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise<DeepPartial<Event>[]> {
+    return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     return this._saveBlockAndFetchEvents(block);
   }
 
@@ -706,17 +780,26 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 
+  async clearProcessedBlockData (block: BlockProgress): Promise<void> {
+    const entities = [...ENTITIES];
+    await this._baseIndexer.clearProcessedBlockData(block, entities);
+  }
+
   async _saveBlockAndFetchEvents ({
     cid: blockCid,
     blockHash,
     blockNumber,
     blockTimestamp,
     parentHash
-  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  }: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     assert(blockHash);
     assert(blockNumber);
 
-    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this));
+    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
     try {
@@ -733,7 +816,7 @@ export class Indexer implements IndexerInterface {
       await dbTx.commitTransaction();
       console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return [blockProgress, []];
+      return [blockProgress, [], transactions];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/conditional-star-release-watcher/src/job-runner.ts

@@ -20,6 +20,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();

packages/conditional-star-release-watcher/src/resolvers.ts

@@ -3,13 +3,8 @@
 //
 
 import assert from 'assert';
-import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
-import Decimal from 'decimal.js';
-import {
-  GraphQLScalarType,
-  GraphQLResolveInfo
-} from 'graphql';
+import { GraphQLResolveInfo } from 'graphql';
 
 import {
   ValueResult,
@@ -17,6 +12,8 @@ import {
   gqlQueryCount,
   getResultState,
   IndexerInterface,
+  GraphQLBigInt,
+  GraphQLBigDecimal,
   EventWatcher,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   setGQLCacheHints
@@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
   const gqlCacheConfig = indexer.serverConfig.gqlCache;
 
   return {
-    BigInt: new BigInt('bigInt'),
+    BigInt: GraphQLBigInt,
 
-    BigDecimal: new GraphQLScalarType({
-      name: 'BigDecimal',
-      description: 'BigDecimal custom scalar type',
-      parseValue (value) {
-        // value from the client
-        return new Decimal(value);
-      },
-      serialize (value: Decimal) {
-        // value sent to the client
-        return value.toFixed();
-      }
-    }),
+    BigDecimal: GraphQLBigDecimal,
 
     Event: {
       __resolveType: (obj: any) => {
@@ -271,9 +257,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('eventsInRange').inc(1);
 
-        const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
-        if (expected !== actual) {
-          throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`);
+        const syncStatus = await indexer.getSyncStatus();
+
+        if (!syncStatus) {
+          throw new Error('No blocks processed yet');
+        }
+
+        if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) {
+          throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`);
         }
 
         const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);

packages/conditional-star-release-watcher/src/schema.gql

@@ -16,7 +16,7 @@ type Proof {
 }
 
 type _Block_ {
-  cid: String!
+  cid: String
   hash: String!
   number: Int!
   timestamp: Int!
@@ -93,13 +93,6 @@ type GetConditionsStateType {
   value3: [BigInt!]!
 }
 
-type SyncStatus {
-  latestIndexedBlockHash: String!
-  latestIndexedBlockNumber: Int!
-  latestCanonicalBlockHash: String!
-  latestCanonicalBlockNumber: Int!
-}
-
 type ResultState {
   block: _Block_!
   contractAddress: String!
@@ -108,6 +101,17 @@ type ResultState {
   data: String!
 }
 
+type SyncStatus {
+  latestIndexedBlockHash: String!
+  latestIndexedBlockNumber: Int!
+  latestCanonicalBlockHash: String!
+  latestCanonicalBlockNumber: Int!
+  initialIndexedBlockHash: String!
+  initialIndexedBlockNumber: Int!
+  latestProcessedBlockHash: String!
+  latestProcessedBlockNumber: Int!
+}
+
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
@@ -121,9 +125,9 @@ type Query {
   hasForfeitedBatch(blockHash: String!, contractAddress: String!, _participant: String!, _batch: Int!): ResultBoolean!
   getRemainingStars(blockHash: String!, contractAddress: String!, _participant: String!): ResultIntArray!
   getConditionsState(blockHash: String!, contractAddress: String!): ResultGetConditionsStateType!
-  getSyncStatus: SyncStatus
   getStateByCID(cid: String!): ResultState
   getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
+  getSyncStatus: SyncStatus
 }
 
 type Mutation {

packages/delegated-sending-watcher/README.md

@@ -83,7 +83,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3005/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -99,7 +99,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3005/graphql
 
   * To watch a contract:
 

packages/delegated-sending-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3005
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/delegated-sending-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/delegated-sending-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "delegated-sending-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/delegated-sending-watcher/src/database.ts

@@ -204,10 +204,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -228,6 +228,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/delegated-sending-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/delegated-sending-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/delegated-sending-watcher/src/entity/GetInvited.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+import { bigintTransformer, bigintArrayTransformer } from '@cerc-io/util';
+
+@Entity()
+@Index(['blockHash', 'contractAddress', '_who'], { unique: true })
+export class GetInvited {
+  @PrimaryGeneratedColumn()
+    id!: number;
+
+  @Column('varchar', { length: 66 })
+    blockHash!: string;
+
+  @Column('integer')
+    blockNumber!: number;
+
+  @Column('varchar', { length: 42 })
+    contractAddress!: string;
+
+  @Column('numeric', { transformer: bigintTransformer })
+    _who!: bigint;
+
+  @Column('numeric', { array: true, transformer: bigintArrayTransformer })
+    value!: bigint[];
+
+  @Column('text', { nullable: true })
+    proof!: string;
+}

packages/delegated-sending-watcher/src/entity/GetInviters.ts

@@ -0,0 +1,28 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+import { bigintArrayTransformer } from '@cerc-io/util';
+
+@Entity()
+@Index(['blockHash', 'contractAddress'], { unique: true })
+export class GetInviters {
+  @PrimaryGeneratedColumn()
+    id!: number;
+
+  @Column('varchar', { length: 66 })
+    blockHash!: string;
+
+  @Column('integer')
+    blockNumber!: number;
+
+  @Column('varchar', { length: 42 })
+    contractAddress!: string;
+
+  @Column('numeric', { array: true, transformer: bigintArrayTransformer })
+    value!: bigint[];
+
+  @Column('text', { nullable: true })
+    proof!: string;
+}

packages/delegated-sending-watcher/src/entity/GetPoolStars.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
+import { bigintTransformer } from '@cerc-io/util';
+
+@Entity()
+@Index(['blockHash', 'contractAddress', '_who'], { unique: true })
+export class GetPoolStars {
+  @PrimaryGeneratedColumn()
+    id!: number;
+
+  @Column('varchar', { length: 66 })
+    blockHash!: string;
+
+  @Column('integer')
+    blockNumber!: number;
+
+  @Column('varchar', { length: 42 })
+    contractAddress!: string;
+
+  @Column('numeric', { transformer: bigintTransformer })
+    _who!: bigint;
+
+  @Column('integer', { array: true })
+    value!: number[];
+
+  @Column('text', { nullable: true })
+    proof!: string;
+}

packages/delegated-sending-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/delegated-sending-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/delegated-sending-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/delegated-sending-watcher/src/gql/queries/index.ts

@@ -5,6 +5,6 @@ export const events = fs.readFileSync(path.join(__dirname, 'events.gql'), 'utf8'
 export const eventsInRange = fs.readFileSync(path.join(__dirname, 'eventsInRange.gql'), 'utf8');
 export const canSend = fs.readFileSync(path.join(__dirname, 'canSend.gql'), 'utf8');
 export const canReceive = fs.readFileSync(path.join(__dirname, 'canReceive.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');

packages/delegated-sending-watcher/src/indexer.ts

@@ -6,11 +6,10 @@ import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import debug from 'debug';
 import JSONbig from 'json-bigint';
-import { ethers } from 'ethers';
+import { ethers, constants } from 'ethers';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -25,7 +24,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  EthClient,
+  UpstreamConfig,
+  EthFullBlock,
+  EthFullTransaction,
+  ExtraEventData
 } from '@cerc-io/util';
 
 import DelegatedSendingArtifacts from './artifacts/DelegatedSending.json';
@@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface {
   _ethProvider: BaseProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
+  _upstreamConfig: UpstreamConfig;
 
   _abiMap: Map<string, JsonFragment[]>;
   _storageLayoutMap: Map<string, StorageLayout>;
   _contractMap: Map<string, ethers.utils.Interface>;
+  eventSignaturesMap: Map<string, string[]>;
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) {
+  constructor (
+    config: {
+      server: ServerConfig;
+      upstream: UpstreamConfig;
+    },
+    db: DatabaseInterface,
+    clients: Clients,
+    ethProvider: BaseProvider,
+    jobQueue: JobQueue
+  ) {
     assert(db);
     assert(clients.ethClient);
 
     this._db = db as Database;
     this._ethClient = clients.ethClient;
     this._ethProvider = ethProvider;
-    this._serverConfig = serverConfig;
-    this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
+    this._serverConfig = config.server;
+    this._upstreamConfig = config.upstream;
+    this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
+    this.eventSignaturesMap = new Map();
 
     const { abi: DelegatedSendingABI } = DelegatedSendingArtifacts;
 
     assert(DelegatedSendingABI);
     this._abiMap.set(KIND_DELEGATEDSENDING, DelegatedSendingABI);
-    this._contractMap.set(KIND_DELEGATEDSENDING, new ethers.utils.Interface(DelegatedSendingABI));
+
+    const DelegatedSendingContractInterface = new ethers.utils.Interface(DelegatedSendingABI);
+    this._contractMap.set(KIND_DELEGATEDSENDING, DelegatedSendingContractInterface);
+
+    const DelegatedSendingEventSignatures = Object.values(DelegatedSendingContractInterface.events).map(value => {
+      return DelegatedSendingContractInterface.getEventTopic(value);
+    });
+    this.eventSignaturesMap.set(KIND_DELEGATEDSENDING, DelegatedSendingEventSignatures);
   }
 
   get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
+  get upstreamConfig (): UpstreamConfig {
+    return this._upstreamConfig;
+  }
+
   get storageLayoutMap (): Map<string, StorageLayout> {
     return this._storageLayoutMap;
   }
@@ -264,16 +292,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.removeStates(blockNumber, kind);
   }
 
-  async triggerIndexingOnEvent (event: Event): Promise<void> {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     // Call custom hook function for indexing on event.
     await handleEvent(this, resultEvent);
   }
 
-  async processEvent (event: Event): Promise<void> {
+  async processEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     // Trigger indexing of data based on the event.
-    await this.triggerIndexingOnEvent(event);
+    await this.triggerIndexingOnEvent(event, extraData);
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
@@ -304,7 +333,11 @@ export class Indexer implements IndexerInterface {
     return this._db.getStateSyncStatus();
   }
 
-  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus | undefined> {
+    if (!this._serverConfig.enableState) {
+      return;
+    }
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -338,10 +371,14 @@ export class Indexer implements IndexerInterface {
     return res;
   }
 
-  async getLatestCanonicalBlock (): Promise<BlockProgress> {
+  async getLatestCanonicalBlock (): Promise<BlockProgress | undefined> {
     const syncStatus = await this.getSyncStatus();
     assert(syncStatus);
 
+    if (syncStatus.latestCanonicalBlockHash === constants.HashZero) {
+      return;
+    }
+
     const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
     assert(latestCanonicalBlock);
 
@@ -352,8 +389,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getLatestStateIndexedBlock();
   }
 
-  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
@@ -368,6 +405,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.saveEventEntity(dbEvent);
   }
 
+  async saveEvents (dbEvents: Event[]): Promise<void> {
+    return this._baseIndexer.saveEvents(dbEvents);
+  }
+
   async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise<Array<Event>> {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
@@ -376,6 +417,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
+  getWatchedContracts (): Contract[] {
+    return this._baseIndexer.getWatchedContracts();
+  }
+
   getContractsByKind (kind: string): Contract[] {
     return this._baseIndexer.getContractsByKind(kind);
   }
@@ -410,6 +455,18 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
+  }
+
+  async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatus(syncStatus);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -426,7 +483,24 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{
+    blockProgress: BlockProgress,
+    events: DeepPartial<Event>[],
+    ethFullBlock: EthFullBlock;
+    ethFullTransactions: EthFullTransaction[];
+  }[]> {
+    return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise<DeepPartial<Event>[]> {
+    return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     return this._saveBlockAndFetchEvents(block);
   }
 
@@ -455,17 +529,26 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 
+  async clearProcessedBlockData (block: BlockProgress): Promise<void> {
+    const entities = [...ENTITIES];
+    await this._baseIndexer.clearProcessedBlockData(block, entities);
+  }
+
   async _saveBlockAndFetchEvents ({
     cid: blockCid,
     blockHash,
     blockNumber,
     blockTimestamp,
     parentHash
-  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  }: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     assert(blockHash);
     assert(blockNumber);
 
-    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this));
+    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
     try {
@@ -482,7 +565,7 @@ export class Indexer implements IndexerInterface {
       await dbTx.commitTransaction();
       console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return [blockProgress, []];
+      return [blockProgress, [], transactions];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/delegated-sending-watcher/src/job-runner.ts

@@ -20,6 +20,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();

packages/delegated-sending-watcher/src/resolvers.ts

@@ -3,13 +3,8 @@
 //
 
 import assert from 'assert';
-import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
-import Decimal from 'decimal.js';
-import {
-  GraphQLScalarType,
-  GraphQLResolveInfo
-} from 'graphql';
+import { GraphQLResolveInfo } from 'graphql';
 
 import {
   ValueResult,
@@ -17,6 +12,8 @@ import {
   gqlQueryCount,
   getResultState,
   IndexerInterface,
+  GraphQLBigInt,
+  GraphQLBigDecimal,
   EventWatcher,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   setGQLCacheHints
@@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
   const gqlCacheConfig = indexer.serverConfig.gqlCache;
 
   return {
-    BigInt: new BigInt('bigInt'),
+    BigInt: GraphQLBigInt,
 
-    BigDecimal: new GraphQLScalarType({
-      name: 'BigDecimal',
-      description: 'BigDecimal custom scalar type',
-      parseValue (value) {
-        // value from the client
-        return new Decimal(value);
-      },
-      serialize (value: Decimal) {
-        // value sent to the client
-        return value.toFixed();
-      }
-    }),
+    BigDecimal: GraphQLBigDecimal,
 
     Event: {
       __resolveType: (obj: any) => {
@@ -127,9 +113,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('eventsInRange').inc(1);
 
-        const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
-        if (expected !== actual) {
-          throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`);
+        const syncStatus = await indexer.getSyncStatus();
+
+        if (!syncStatus) {
+          throw new Error('No blocks processed yet');
+        }
+
+        if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) {
+          throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`);
         }
 
         const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);

packages/delegated-sending-watcher/src/schema.gql

@@ -16,7 +16,7 @@ type Proof {
 }
 
 type _Block_ {
-  cid: String!
+  cid: String
   hash: String!
   number: Int!
   timestamp: Int!
@@ -54,13 +54,6 @@ type ResultBoolean {
   proof: Proof
 }
 
-type SyncStatus {
-  latestIndexedBlockHash: String!
-  latestIndexedBlockNumber: Int!
-  latestCanonicalBlockHash: String!
-  latestCanonicalBlockNumber: Int!
-}
-
 type ResultState {
   block: _Block_!
   contractAddress: String!
@@ -69,14 +62,25 @@ type ResultState {
   data: String!
 }
 
+type SyncStatus {
+  latestIndexedBlockHash: String!
+  latestIndexedBlockNumber: Int!
+  latestCanonicalBlockHash: String!
+  latestCanonicalBlockNumber: Int!
+  initialIndexedBlockHash: String!
+  initialIndexedBlockNumber: Int!
+  latestProcessedBlockHash: String!
+  latestProcessedBlockNumber: Int!
+}
+
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
   canSend(blockHash: String!, contractAddress: String!, _as: BigInt!, _point: BigInt!): ResultBoolean!
   canReceive(blockHash: String!, contractAddress: String!, _recipient: String!): ResultBoolean!
-  getSyncStatus: SyncStatus
   getStateByCID(cid: String!): ResultState
   getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
+  getSyncStatus: SyncStatus
 }
 
 type Mutation {

packages/ecliptic-watcher/README.md

@@ -81,7 +81,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3006/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -97,7 +97,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3006/graphql
 
   * To watch a contract:
 

packages/ecliptic-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3006
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/ecliptic-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/ecliptic-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "ecliptic-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/ecliptic-watcher/src/database.ts

@@ -350,10 +350,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -374,6 +374,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/ecliptic-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/ecliptic-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/ecliptic-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/ecliptic-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/ecliptic-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/ecliptic-watcher/src/gql/queries/index.ts

@@ -14,6 +14,6 @@ export const getApproved = fs.readFileSync(path.join(__dirname, 'getApproved.gql
 export const isApprovedForAll = fs.readFileSync(path.join(__dirname, 'isApprovedForAll.gql'), 'utf8');
 export const getSpawnLimit = fs.readFileSync(path.join(__dirname, 'getSpawnLimit.gql'), 'utf8');
 export const canEscapeTo = fs.readFileSync(path.join(__dirname, 'canEscapeTo.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');

packages/ecliptic-watcher/src/indexer.ts

@@ -6,11 +6,10 @@ import assert from 'assert';
 import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import debug from 'debug';
 import JSONbig from 'json-bigint';
-import { ethers } from 'ethers';
+import { ethers, constants } from 'ethers';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
-import { EthClient } from '@cerc-io/ipld-eth-client';
 import { MappingKey, StorageLayout } from '@cerc-io/solidity-mapper';
 import {
   Indexer as BaseIndexer,
@@ -25,7 +24,12 @@ import {
   ResultEvent,
   getResultEvent,
   DatabaseInterface,
-  Clients
+  Clients,
+  EthClient,
+  UpstreamConfig,
+  EthFullBlock,
+  EthFullTransaction,
+  ExtraEventData
 } from '@cerc-io/util';
 
 import EclipticArtifacts from './artifacts/Ecliptic.json';
@@ -50,36 +54,60 @@ export class Indexer implements IndexerInterface {
   _ethProvider: BaseProvider;
   _baseIndexer: BaseIndexer;
   _serverConfig: ServerConfig;
+  _upstreamConfig: UpstreamConfig;
 
   _abiMap: Map<string, JsonFragment[]>;
   _storageLayoutMap: Map<string, StorageLayout>;
   _contractMap: Map<string, ethers.utils.Interface>;
+  eventSignaturesMap: Map<string, string[]>;
 
-  constructor (serverConfig: ServerConfig, db: DatabaseInterface, clients: Clients, ethProvider: BaseProvider, jobQueue: JobQueue) {
+  constructor (
+    config: {
+      server: ServerConfig;
+      upstream: UpstreamConfig;
+    },
+    db: DatabaseInterface,
+    clients: Clients,
+    ethProvider: BaseProvider,
+    jobQueue: JobQueue
+  ) {
     assert(db);
     assert(clients.ethClient);
 
     this._db = db as Database;
     this._ethClient = clients.ethClient;
     this._ethProvider = ethProvider;
-    this._serverConfig = serverConfig;
-    this._baseIndexer = new BaseIndexer(this._serverConfig, this._db, this._ethClient, this._ethProvider, jobQueue);
+    this._serverConfig = config.server;
+    this._upstreamConfig = config.upstream;
+    this._baseIndexer = new BaseIndexer(config, this._db, this._ethClient, this._ethProvider, jobQueue);
 
     this._abiMap = new Map();
     this._storageLayoutMap = new Map();
     this._contractMap = new Map();
+    this.eventSignaturesMap = new Map();
 
     const { abi: EclipticABI } = EclipticArtifacts;
 
     assert(EclipticABI);
     this._abiMap.set(KIND_ECLIPTIC, EclipticABI);
-    this._contractMap.set(KIND_ECLIPTIC, new ethers.utils.Interface(EclipticABI));
+
+    const EclipticContractInterface = new ethers.utils.Interface(EclipticABI);
+    this._contractMap.set(KIND_ECLIPTIC, EclipticContractInterface);
+
+    const EclipticEventSignatures = Object.values(EclipticContractInterface.events).map(value => {
+      return EclipticContractInterface.getEventTopic(value);
+    });
+    this.eventSignaturesMap.set(KIND_ECLIPTIC, EclipticEventSignatures);
   }
 
   get serverConfig (): ServerConfig {
     return this._serverConfig;
   }
 
+  get upstreamConfig (): UpstreamConfig {
+    return this._upstreamConfig;
+  }
+
   get storageLayoutMap (): Map<string, StorageLayout> {
     return this._storageLayoutMap;
   }
@@ -536,16 +564,17 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.removeStates(blockNumber, kind);
   }
 
-  async triggerIndexingOnEvent (event: Event): Promise<void> {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async triggerIndexingOnEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     const resultEvent = this.getResultEvent(event);
 
     // Call custom hook function for indexing on event.
     await handleEvent(this, resultEvent);
   }
 
-  async processEvent (event: Event): Promise<void> {
+  async processEvent (event: Event, extraData: ExtraEventData): Promise<void> {
     // Trigger indexing of data based on the event.
-    await this.triggerIndexingOnEvent(event);
+    await this.triggerIndexingOnEvent(event, extraData);
   }
 
   async processBlock (blockProgress: BlockProgress): Promise<void> {
@@ -576,7 +605,11 @@ export class Indexer implements IndexerInterface {
     return this._db.getStateSyncStatus();
   }
 
-  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus> {
+  async updateStateSyncStatusIndexedBlock (blockNumber: number, force?: boolean): Promise<StateSyncStatus | undefined> {
+    if (!this._serverConfig.enableState) {
+      return;
+    }
+
     const dbTx = await this._db.createTransactionRunner();
     let res;
 
@@ -610,10 +643,14 @@ export class Indexer implements IndexerInterface {
     return res;
   }
 
-  async getLatestCanonicalBlock (): Promise<BlockProgress> {
+  async getLatestCanonicalBlock (): Promise<BlockProgress | undefined> {
     const syncStatus = await this.getSyncStatus();
     assert(syncStatus);
 
+    if (syncStatus.latestCanonicalBlockHash === constants.HashZero) {
+      return;
+    }
+
     const latestCanonicalBlock = await this.getBlockProgress(syncStatus.latestCanonicalBlockHash);
     assert(latestCanonicalBlock);
 
@@ -624,8 +661,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getLatestStateIndexedBlock();
   }
 
-  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<void> {
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock, context);
   }
 
   updateStateStatusMap (address: string, stateStatus: StateStatus): void {
@@ -640,6 +677,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.saveEventEntity(dbEvent);
   }
 
+  async saveEvents (dbEvents: Event[]): Promise<void> {
+    return this._baseIndexer.saveEvents(dbEvents);
+  }
+
   async getEventsByFilter (blockHash: string, contract?: string, name?: string): Promise<Array<Event>> {
     return this._baseIndexer.getEventsByFilter(blockHash, contract, name);
   }
@@ -648,6 +689,10 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
+  getWatchedContracts (): Contract[] {
+    return this._baseIndexer.getWatchedContracts();
+  }
+
   getContractsByKind (kind: string): Contract[] {
     return this._baseIndexer.getContractsByKind(kind);
   }
@@ -682,6 +727,18 @@ export class Indexer implements IndexerInterface {
     return syncStatus;
   }
 
+  async updateSyncStatusProcessedBlock (blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatusProcessedBlock(blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    return this._baseIndexer.updateSyncStatusIndexingError(hasIndexingError);
+  }
+
+  async updateSyncStatus (syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    return this._baseIndexer.updateSyncStatus(syncStatus);
+  }
+
   async getEvent (id: string): Promise<Event | undefined> {
     return this._baseIndexer.getEvent(id);
   }
@@ -698,7 +755,24 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.getBlocksAtHeight(height, isPruned);
   }
 
-  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  async fetchAndSaveFilteredEventsAndBlocks (startBlock: number, endBlock: number): Promise<{
+    blockProgress: BlockProgress,
+    events: DeepPartial<Event>[],
+    ethFullBlock: EthFullBlock;
+    ethFullTransactions: EthFullTransaction[];
+  }[]> {
+    return this._baseIndexer.fetchAndSaveFilteredEventsAndBlocks(startBlock, endBlock, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async fetchEventsForContracts (blockHash: string, blockNumber: number, addresses: string[]): Promise<DeepPartial<Event>[]> {
+    return this._baseIndexer.fetchEventsForContracts(blockHash, blockNumber, addresses, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
+  }
+
+  async saveBlockAndFetchEvents (block: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     return this._saveBlockAndFetchEvents(block);
   }
 
@@ -727,17 +801,26 @@ export class Indexer implements IndexerInterface {
     await this._baseIndexer.resetWatcherToBlock(blockNumber, entities);
   }
 
+  async clearProcessedBlockData (block: BlockProgress): Promise<void> {
+    const entities = [...ENTITIES];
+    await this._baseIndexer.clearProcessedBlockData(block, entities);
+  }
+
   async _saveBlockAndFetchEvents ({
     cid: blockCid,
     blockHash,
     blockNumber,
     blockTimestamp,
     parentHash
-  }: DeepPartial<BlockProgress>): Promise<[BlockProgress, DeepPartial<Event>[]]> {
+  }: DeepPartial<BlockProgress>): Promise<[
+    BlockProgress,
+    DeepPartial<Event>[],
+    EthFullTransaction[]
+  ]> {
     assert(blockHash);
     assert(blockNumber);
 
-    const dbEvents = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.parseEventNameAndArgs.bind(this));
+    const { events: dbEvents, transactions } = await this._baseIndexer.fetchEvents(blockHash, blockNumber, this.eventSignaturesMap, this.parseEventNameAndArgs.bind(this));
 
     const dbTx = await this._db.createTransactionRunner();
     try {
@@ -754,7 +837,7 @@ export class Indexer implements IndexerInterface {
       await dbTx.commitTransaction();
       console.timeEnd(`time:indexer#_saveBlockAndFetchEvents-db-save-${blockNumber}`);
 
-      return [blockProgress, []];
+      return [blockProgress, [], transactions];
     } catch (error) {
       await dbTx.rollbackTransaction();
       throw error;

packages/ecliptic-watcher/src/job-runner.ts

@@ -20,6 +20,7 @@ export const main = async (): Promise<any> => {
 
   await jobRunnerCmd.exec(async (jobRunner: JobRunner): Promise<void> => {
     await jobRunner.subscribeBlockProcessingQueue();
+    await jobRunner.subscribeHistoricalProcessingQueue();
     await jobRunner.subscribeEventProcessingQueue();
     await jobRunner.subscribeBlockCheckpointQueue();
     await jobRunner.subscribeHooksQueue();

packages/ecliptic-watcher/src/resolvers.ts

@@ -3,13 +3,8 @@
 //
 
 import assert from 'assert';
-import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
-import Decimal from 'decimal.js';
-import {
-  GraphQLScalarType,
-  GraphQLResolveInfo
-} from 'graphql';
+import { GraphQLResolveInfo } from 'graphql';
 
 import {
   ValueResult,
@@ -17,6 +12,8 @@ import {
   gqlQueryCount,
   getResultState,
   IndexerInterface,
+  GraphQLBigInt,
+  GraphQLBigDecimal,
   EventWatcher,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   setGQLCacheHints
@@ -33,20 +30,9 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
   const gqlCacheConfig = indexer.serverConfig.gqlCache;
 
   return {
-    BigInt: new BigInt('bigInt'),
+    BigInt: GraphQLBigInt,
 
-    BigDecimal: new GraphQLScalarType({
-      name: 'BigDecimal',
-      description: 'BigDecimal custom scalar type',
-      parseValue (value) {
-        // value from the client
-        return new Decimal(value);
-      },
-      serialize (value: Decimal) {
-        // value sent to the client
-        return value.toFixed();
-      }
-    }),
+    BigDecimal: GraphQLBigDecimal,
 
     Event: {
       __resolveType: (obj: any) => {
@@ -289,9 +275,14 @@ export const createResolvers = async (indexerArg: IndexerInterface, eventWatcher
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('eventsInRange').inc(1);
 
-        const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
-        if (expected !== actual) {
-          throw new Error(`Range not available, expected ${expected}, got ${actual} blocks in range`);
+        const syncStatus = await indexer.getSyncStatus();
+
+        if (!syncStatus) {
+          throw new Error('No blocks processed yet');
+        }
+
+        if ((fromBlockNumber < syncStatus.initialIndexedBlockNumber) || (toBlockNumber > syncStatus.latestProcessedBlockNumber)) {
+          throw new Error(`Block range should be between ${syncStatus.initialIndexedBlockNumber} and ${syncStatus.latestProcessedBlockNumber}`);
         }
 
         const events = await indexer.getEventsInRange(fromBlockNumber, toBlockNumber);

packages/ecliptic-watcher/src/schema.gql

@@ -16,7 +16,7 @@ type Proof {
 }
 
 type _Block_ {
-  cid: String!
+  cid: String
   hash: String!
   number: Int!
   timestamp: Int!
@@ -87,13 +87,6 @@ type ResultBigInt {
   proof: Proof
 }
 
-type SyncStatus {
-  latestIndexedBlockHash: String!
-  latestIndexedBlockNumber: Int!
-  latestCanonicalBlockHash: String!
-  latestCanonicalBlockNumber: Int!
-}
-
 type ResultState {
   block: _Block_!
   contractAddress: String!
@@ -102,6 +95,17 @@ type ResultState {
   data: String!
 }
 
+type SyncStatus {
+  latestIndexedBlockHash: String!
+  latestIndexedBlockNumber: Int!
+  latestCanonicalBlockHash: String!
+  latestCanonicalBlockNumber: Int!
+  initialIndexedBlockHash: String!
+  initialIndexedBlockNumber: Int!
+  latestProcessedBlockHash: String!
+  latestProcessedBlockNumber: Int!
+}
+
 type Query {
   events(blockHash: String!, contractAddress: String!, name: String): [ResultEvent!]
   eventsInRange(fromBlockNumber: Int!, toBlockNumber: Int!): [ResultEvent!]
@@ -116,9 +120,9 @@ type Query {
   isApprovedForAll(blockHash: String!, contractAddress: String!, _owner: String!, _operator: String!): ResultBoolean!
   getSpawnLimit(blockHash: String!, contractAddress: String!, _point: BigInt!, _time: BigInt!): ResultBigInt!
   canEscapeTo(blockHash: String!, contractAddress: String!, _point: BigInt!, _sponsor: BigInt!): ResultBoolean!
-  getSyncStatus: SyncStatus
   getStateByCID(cid: String!): ResultState
   getState(blockHash: String!, contractAddress: String!, kind: String): ResultState
+  getSyncStatus: SyncStatus
 }
 
 type Mutation {

packages/gateway-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/gateway-server",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "main": "index.js",
   "license": "AGPL-3.0",
   "private": true,
@@ -21,6 +21,7 @@
     "is-reachable": "^5.2.1"
   },
   "devDependencies": {
+    "@types/debug": "^4.1.5",
     "@types/node": "^18.11.18",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",

packages/linear-star-release-watcher/README.md

@@ -81,7 +81,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3007/graphql
 
 * If the watcher is an `active` watcher:
 
@@ -97,7 +97,7 @@ To enable GQL requests caching:
     yarn server
     ```
 
-    GQL console: http://localhost:3009/graphql
+    GQL console: http://localhost:3007/graphql
 
   * To watch a contract:
 

packages/linear-star-release-watcher/environments/local.toml

@@ -1,7 +1,8 @@
 [server]
   host = "127.0.0.1"
   port = 3007
-  kind = "lazy"
+  kind = "active"
+  gqlPath = "/graphql"
 
   # Checkpointing state.
   checkpointing = true
@@ -11,15 +12,15 @@
 
   # Enable state creation
   # CAUTION: Disable only if state creation is not desired or can be filled subsequently
-  enableState = true
-
-  # Boolean to filter logs by contract.
-  filterLogs = false
+  enableState = false
 
   # Max block range for which to return events in eventsInRange GQL query.
   # Use -1 for skipping check on block range.
   maxEventsBlockRange = 1000
 
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
   # GQL cache settings
   [server.gqlCache]
     enabled = true
@@ -48,8 +49,19 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "http://127.0.0.1:8083/graphql"
-    rpcProviderEndpoint = "http://127.0.0.1:8082"
+    gqlApiEndpoint = "http://127.0.0.1:8082/graphql"
+    rpcProviderEndpoint = "http://127.0.0.1:8081"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = false
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
 
   [upstream.cache]
     name = "requests"
@@ -61,6 +73,17 @@
   maxCompletionLagInSecs = 300
   jobDelayInMilliSecs = 100
   eventsInBatch = 50
+  subgraphEventsOrder = true
   blockDelayInMilliSecs = 2000
-  prefetchBlocksInMem = true
-  prefetchBlockCount = 10
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

packages/linear-star-release-watcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerc-io/linear-star-release-watcher",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "linear-star-release-watcher",
   "private": true,
   "main": "dist/index.js",
@@ -38,28 +38,28 @@
   "homepage": "https://github.com/cerc-io/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
+    "@cerc-io/cli": "^0.2.76",
+    "@cerc-io/ipld-eth-client": "^0.2.76",
+    "@cerc-io/solidity-mapper": "^0.2.76",
+    "@cerc-io/util": "^0.2.76",
     "@ethersproject/providers": "^5.4.4",
-    "@cerc-io/cli": "^0.2.40",
-    "@cerc-io/ipld-eth-client": "^0.2.40",
-    "@cerc-io/solidity-mapper": "^0.2.40",
-    "@cerc-io/util": "^0.2.40",
-    "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "graphql": "^15.5.0",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
-    "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "typeorm": "0.2.37",
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
-    "@types/yargs": "^17.0.0",
     "@types/debug": "^4.1.5",
     "@types/json-bigint": "^1.0.0",
+    "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
+    "copyfiles": "^2.4.1",
     "eslint": "^8.35.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",
@@ -69,7 +69,6 @@
     "eslint-plugin-standard": "^5.0.0",
     "husky": "^7.0.2",
     "ts-node": "^10.2.1",
-    "typescript": "^5.0.2",
-    "copyfiles": "^2.4.1"
+    "typescript": "^5.0.2"
   }
 }

packages/linear-star-release-watcher/src/database.ts

@@ -219,10 +219,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.saveBlockProgress(repo, block);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number, context?: any): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock, context);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
@@ -243,6 +243,24 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.updateSyncStatusChainHead(repo, blockHash, blockNumber, force);
   }
 
+  async updateSyncStatusProcessedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusProcessedBlock(repo, blockHash, blockNumber, force);
+  }
+
+  async updateSyncStatusIndexingError (queryRunner: QueryRunner, hasIndexingError: boolean): Promise<SyncStatus | undefined> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatusIndexingError(repo, hasIndexingError);
+  }
+
+  async updateSyncStatus (queryRunner: QueryRunner, syncStatus: DeepPartial<SyncStatus>): Promise<SyncStatus> {
+    const repo = queryRunner.manager.getRepository(SyncStatus);
+
+    return this._baseDatabase.updateSyncStatus(repo, syncStatus);
+  }
+
   async getSyncStatus (queryRunner: QueryRunner): Promise<SyncStatus | undefined> {
     const repo = queryRunner.manager.getRepository(SyncStatus);
 

packages/linear-star-release-watcher/src/entity/BlockProgress.ts

@@ -13,8 +13,8 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
     id!: number;
 
-  @Column('varchar')
-    cid!: string;
+  @Column('varchar', { nullable: true })
+    cid!: string | null;
 
   @Column('varchar', { length: 66 })
     blockHash!: string;

packages/linear-star-release-watcher/src/entity/Contract.ts

@@ -21,4 +21,7 @@ export class Contract {
 
   @Column('integer')
     startingBlock!: number;
+
+  @Column('jsonb', { nullable: true })
+    context!: Record<string, { data: any, type: number }>;
 }

packages/linear-star-release-watcher/src/entity/StateSyncStatus.ts

@@ -12,6 +12,6 @@ export class StateSyncStatus {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
-  @Column('integer', { nullable: true })
+  @Column('integer')
     latestCheckpointBlockNumber!: number;
 }

packages/linear-star-release-watcher/src/entity/SyncStatus.ts

@@ -22,6 +22,12 @@ export class SyncStatus implements SyncStatusInterface {
   @Column('integer')
     latestIndexedBlockNumber!: number;
 
+  @Column('varchar', { length: 66 })
+    latestProcessedBlockHash!: string;
+
+  @Column('integer')
+    latestProcessedBlockNumber!: number;
+
   @Column('varchar', { length: 66 })
     latestCanonicalBlockHash!: string;
 
@@ -33,4 +39,7 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
     initialIndexedBlockNumber!: number;
+
+  @Column('boolean', { default: false })
+    hasIndexingError!: boolean;
 }

packages/linear-star-release-watcher/src/gql/queries/getSyncStatus.gql

@@ -4,5 +4,9 @@ query getSyncStatus{
         latestIndexedBlockNumber
         latestCanonicalBlockHash
         latestCanonicalBlockNumber
+        initialIndexedBlockHash
+        initialIndexedBlockNumber
+        latestProcessedBlockHash
+        latestProcessedBlockNumber
     }
 }

packages/linear-star-release-watcher/src/gql/queries/index.ts

@@ -6,6 +6,6 @@ export const eventsInRange = fs.readFileSync(path.join(__dirname, 'eventsInRange
 export const withdrawLimit = fs.readFileSync(path.join(__dirname, 'withdrawLimit.gql'), 'utf8');
 export const verifyBalance = fs.readFileSync(path.join(__dirname, 'verifyBalance.gql'), 'utf8');
 export const getRemainingStars = fs.readFileSync(path.join(__dirname, 'getRemainingStars.gql'), 'utf8');
-export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');
 export const getStateByCID = fs.readFileSync(path.join(__dirname, 'getStateByCID.gql'), 'utf8');
 export const getState = fs.readFileSync(path.join(__dirname, 'getState.gql'), 'utf8');
+export const getSyncStatus = fs.readFileSync(path.join(__dirname, 'getSyncStatus.gql'), 'utf8');