Auto generating state from indexer methods (#277)

* Generate default derived state for Mapping type variables

* Update IPLDBlock in storage methods instead of using a private class variable

* Default state flag for indexer storage methods

* Helper functions to update state objects

* Add checkpoint flag in Contract table and corresponding changes in existing watchers

* Update codegen docs

* Add examples to generated docs

* Turn default state off by default

* Make state parameter to indexer storage methods default to none

* Add method to get prev. state in indexer

packages/codegen/README.md

@@ -29,16 +29,16 @@
     * `output-folder`(alias: `o`): Output folder path. (logs output using `stdout` if not provided).
     * `mode`(alias: `m`): Code generation mode (default: `all`).
     * `flatten`(alias: `f`): Flatten the input contract file (default: `true`).
-    * `kind` (alias: `k`): Kind of watcher (default; `active`).
+    * `kind` (alias: `k`): Kind of watcher (default: `active`).
 
   **Note**: When passed an *URL* as `input-file`, it is assumed that it points to an already flattened contract file.
 
   Examples:
 
-  Generate code in both `eth_call` and `storage` mode, `active` kind.
+  Generate code in `storage` mode, `lazy` kind.
 
   ```bash
-  yarn codegen --input-file ./test/examples/contracts/ERC20.sol --contract-name ERC20 --output-folder ../my-erc20-watcher --mode all --kind active
+  yarn codegen --input-file ./test/examples/contracts/ERC721.sol --contract-name ERC721 --output-folder ../my-erc721-watcher --mode storage --kind lazy
   ```
 
   Generate code in `eth_call` mode using a contract provided by an URL.
@@ -47,10 +47,10 @@
   yarn codegen --input-file https://git.io/Jupci --contract-name ERC721 --output-folder ../my-erc721-watcher --mode eth_call
   ```
 
-  Generate code in `storage` mode, `lazy` kind.
+  Generate code for `ERC721` in both `eth_call` and `storage` mode, `active` kind.
 
   ```bash
-  yarn codegen --input-file ./test/examples/contracts/ERC721.sol --contract-name ERC721 --output-folder ../my-erc721-watcher --mode storage --kind lazy
+  yarn codegen --input-file ../../node_modules/@openzeppelin/contracts/token/ERC721/ERC721.sol --contract-name ERC721 --output-folder ../demo-erc721-watcher --mode all --kind active
   ```
 
   Generate code for `ERC20` contract in both `eth_call` and `storage` mode, `active` kind:
@@ -73,7 +73,7 @@
 
 * Create the databases configured in `environments/local.toml`.
 
-* Update the derived state checkpoint settings in `environments/local.toml`.
+* Update the state checkpoint settings in `environments/local.toml`.
 
 ### Customize
 
@@ -81,11 +81,15 @@
 
   * Edit the custom hook function `handleEvent` (triggered on an event) in `src/hooks.ts` to perform corresponding indexing using the `Indexer` object.
 
-  * Edit the custom hook function `postBlockHook` (triggered on a block) in `src/hooks.ts` to save `IPLDBlock`s using the `Indexer` object.
+  * While using the indexer storage methods for indexing, pass the optional arg. `state` as `diff` or `checkpoint` if default state is desired to be generated using the state variables being indexed else pass `none`.
 
-  * Edit the custom hook function `initialCheckpointHook` (triggered on watch-contract) in `src/hooks.ts` to save an initial checkpoint `IPLDBlock` using the `Indexer` object.
+* Generating state:
 
-  * The existing example hooks in `src/hooks.ts` are for an `ERC20` contract.
+  * Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in `src/hooks.ts` to save an initial checkpoint `IPLDBlock` using the `Indexer` object.
+
+  * Edit the custom hook function `createStateDiff` (triggered on a block) in `src/hooks.ts` to save the state in an `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
+
+* The existing example hooks in `src/hooks.ts` are for an `ERC20` contract.
 
 ### Run
 
@@ -112,7 +116,7 @@
   * To watch a contract:
 
     ```bash
-    yarn watch:contract --address <contract-address> --kind <contract-kind> --starting-block [block-number]
+    yarn watch:contract --address <contract-address> --kind <contract-kind> --checkpoint <true | false> --starting-block [block-number]
     ```
 
   * To fill a block range:

packages/codegen/src/data/entities/Contract.yaml

@@ -15,9 +15,10 @@ columns:
     pgType: varchar
     tsType: string
     columnType: Column
-    columnOptions:
+  - name: checkpoint
-      - option: length
+    pgType: boolean
-        value: 8
+    tsType: boolean
+    columnType: Column
   - name: startingBlock
     pgType: integer
     tsType: number

packages/codegen/src/indexer.ts

@@ -32,8 +32,9 @@ export class Indexer {
    * @param name Name of the query.
    * @param params Parameters to the query.
    * @param returnType Return type for the query.
+   * @param stateVariableTypeName Type of the state variable in case of state variable query.
    */
-  addQuery (mode: string, name: string, params: Array<Param>, returnType: string): void {
+  addQuery (mode: string, name: string, params: Array<Param>, returnType: string, stateVariableType?: string): void {
     // Check if the query is already added.
     if (this._queries.some(query => query.name === name)) {
       return;
@@ -45,7 +46,8 @@ export class Indexer {
       saveQueryName: '',
       params: _.cloneDeep(params),
       returnType,
-      mode
+      mode,
+      stateVariableType
     };
 
     if (name.charAt(0) === '_') {
@@ -69,6 +71,10 @@ export class Indexer {
     assert(tsReturnType);
     queryObject.returnType = tsReturnType;
 
+    if (stateVariableType) {
+      queryObject.stateVariableType = stateVariableType;
+    }
+
     this._queries.push(queryObject);
   }
 

packages/codegen/src/schema.ts

@@ -292,8 +292,9 @@ export class Schema {
       watchContract: {
         type: 'Boolean!',
         args: {
-          contractAddress: 'String!',
+          address: 'String!',
           kind: 'String!',
+          checkpoint: 'Boolean!',
           startingBlock: 'Int'
         }
       }

packages/codegen/src/templates/config-template.handlebars

@@ -3,7 +3,7 @@
   port = 3008
   kind = "{{watcherKind}}"
 
-  # Checkpointing derived state.
+  # Checkpointing state.
   checkpointing = true
 
   # Checkpoint interval in number of blocks.

packages/codegen/src/templates/database-template.handlebars

@@ -309,11 +309,11 @@ export class Database {
     return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events);
   }
 
-  async saveContract (address: string, kind: string, startingBlock: number): Promise<void> {
+  async saveContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
     await this._conn.transaction(async (tx) => {
       const repo = tx.getRepository(Contract);
 
-      return this._baseDatabase.saveContract(repo, address, startingBlock, kind);
+      return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
     });
   }
 

packages/codegen/src/templates/hooks-template.handlebars

@@ -3,9 +3,8 @@
 //
 
 import assert from 'assert';
-import _ from 'lodash';
 
-import { UNKNOWN_EVENT_NAME } from '@vulcanize/util';
+import { UNKNOWN_EVENT_NAME, updateStateForMappingType, updateStateForElementaryType } from '@vulcanize/util';
 
 import { Indexer, ResultEvent } from './indexer';
 import { BlockProgress } from './entity/BlockProgress';
@@ -15,19 +14,23 @@ const ACCOUNTS = [
 ];
 
 /**
- * Initial checkpoint hook function.
+ * Hook function to create an initial checkpoint.
  * @param indexer Indexer instance.
  * @param block Concerned block.
  * @param contractAddress Address of the concerned contract.
  */
-export async function initialCheckpointHook (indexer: Indexer, block: BlockProgress, contractAddress: string): Promise<void> {
+export async function createInitialCheckpoint (indexer: Indexer, block: BlockProgress, contractAddress: string): Promise<void> {
+  assert(indexer);
+  assert(block);
+  assert(contractAddress);
+
   // Store the initial state values in an IPLDBlock.
-  const ipldBlockData: any = {};
+  let ipldBlockData: any = {};
 
   // Setting the initial balances of accounts.
   for (const account of ACCOUNTS) {
     const balance = await indexer._balances(block.blockHash, contractAddress, account);
-    _.set(ipldBlockData, `state._balances[${account}]`, balance.value.toString());
+    ipldBlockData = updateStateForMappingType(ipldBlockData, '_balances', [account], balance.value.toString());
   }
 
   const ipldBlock = await indexer.prepareIPLDBlock(block, contractAddress, ipldBlockData, 'checkpoint');
@@ -35,11 +38,14 @@ export async function initialCheckpointHook (indexer: Indexer, block: BlockProgr
 }
 
 /**
- * Post-block hook function.
+ * Hook function to create and store state diffs.
  * @param indexer Indexer instance that contains methods to fetch the contract varaiable values.
  * @param blockHash Block hash of the concerned block.
  */
-export async function postBlockHook (indexer: Indexer, blockHash: string): Promise<void> {
+export async function createStateDiff (indexer: Indexer, blockHash: string): Promise<void> {
+  assert(indexer);
+  assert(blockHash);
+
   // Get events for current block and make an entry of updated values in IPLDBlock.
   const events = await indexer.getEventsByFilter(blockHash);
 
@@ -58,7 +64,7 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi
 
     const eventData = indexer.getResultEvent(event);
 
-    const ipldBlockData: any = {};
+    let ipldBlockData: any = {};
 
     switch (event.eventName) {
       case 'Transfer': {
@@ -73,8 +79,8 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi
         //      "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": "999999999999999999900"
         //   }
         // }
-        _.set(ipldBlockData, `state._balances[${from}]`, fromBalance.value.toString());
+        ipldBlockData = updateStateForMappingType(ipldBlockData, '_balances', [from], fromBalance.value.toString());
-        _.set(ipldBlockData, `state._balances[${to}]`, toBalance.value.toString());
+        ipldBlockData = updateStateForMappingType(ipldBlockData, '_balances', [to], toBalance.value.toString());
 
         break;
       }
@@ -90,7 +96,7 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi
         //      }
         //   }
         // }
-        _.set(ipldBlockData, `state._allowances[${owner}][${spender}]`, allowance.value.toString());
+        ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString());
 
         break;
       }
@@ -123,10 +129,10 @@ export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Pr
       const { from, to } = eventData.event;
 
       // Update balance entry for sender in the database.
-      await indexer.balanceOf(eventData.block.hash, eventData.contract, from);
+      await indexer._balances(eventData.block.hash, eventData.contract, from);
 
       // Update balance entry for receiver in the database.
-      await indexer.balanceOf(eventData.block.hash, eventData.contract, to);
+      await indexer._balances(eventData.block.hash, eventData.contract, to);
 
       break;
     }
@@ -138,7 +144,7 @@ export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Pr
       const { owner, spender } = eventData.event;
 
       // Update allowance entry for (owner, spender) combination in the database.
-      await indexer.allowance(eventData.block.hash, eventData.contract, owner, spender);
+      await indexer._allowances(eventData.block.hash, eventData.contract, owner, spender);
 
       break;
     }

packages/codegen/src/templates/indexer-template.handlebars

@@ -16,7 +16,7 @@ import { BaseProvider } from '@ethersproject/providers';
 import * as codec from '@ipld/dag-json';
 import { EthClient } from '@vulcanize/ipld-eth-client';
 import { StorageLayout } from '@vulcanize/solidity-mapper';
-import { Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, ServerConfig, Where, QueryOptions } from '@vulcanize/util';
+import { Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, ServerConfig, Where, QueryOptions, updateStateForElementaryType, updateStateForMappingType } from '@vulcanize/util';
 
 import { Database } from './database';
 import { Contract } from './entity/Contract';
@@ -26,7 +26,7 @@ import { HookStatus } from './entity/HookStatus';
 import { BlockProgress } from './entity/BlockProgress';
 import { IPLDBlock } from './entity/IPLDBlock';
 import artifacts from './artifacts/{{inputFileName}}.json';
-import { initialCheckpointHook, handleEvent, postBlockHook } from './hooks';
+import { createInitialCheckpoint, handleEvent, createStateDiff } from './hooks';
 
 const log = debug('vulcanize:indexer');
 
@@ -75,7 +75,7 @@ export class Indexer {
   _db: Database
   _ethClient: EthClient
   _ethProvider: BaseProvider
-  _postgraphileClient: EthClient;
+  _postgraphileClient: EthClient
   _baseIndexer: BaseIndexer
   _serverConfig: ServerConfig
 
@@ -158,7 +158,14 @@ export class Indexer {
 
   {{#each queries as | query |}}
   async {{query.name}} (blockHash: string, contractAddress: string
-    {{~#each query.params}}, {{this.name~}}: {{this.type~}} {{/each}}): Promise<ValueResult> {
+    {{~#each query.params}}, {{this.name~}}: {{this.type~}} {{/each}}
+    {{~#if query.stateVariableType~}}
+    , state = 'none'): Promise<ValueResult> {
+    assert(_.includes(['diff', 'checkpoint', 'none'], state));
+
+    {{else~}}
+    ): Promise<ValueResult> {
+    {{/if}}
     const entity = await this._db.{{query.getQueryName}}({ blockHash, contractAddress
     {{~#each query.params}}, {{this.name~}} {{~/each}} });
     if (entity) {
@@ -203,6 +210,28 @@ export class Indexer {
     await this._db.{{query.saveQueryName}}({ blockHash, contractAddress
     {{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbig.stringify(result.proof) });
 
+    {{#if query.stateVariableType}}
+    {{#if (compare query.stateVariableType 'Mapping')}}
+    if (state !== 'none') {
+      const stateUpdate = updateStateForMappingType({}, '{{query.name}}', [
+      {{~#each query.params}}
+      {{~this.name}}.toString() {{~#unless @last}}, {{/unless~}}
+      {{/each~}}
+      ], result.value.toString());
+      await this.storeIPLDData(blockHash, contractAddress, stateUpdate, state);
+    }
+
+    {{else if (compare query.stateVariableType 'ElementaryTypeName')}}
+    if (state !== 'none') {
+      const stateUpdate = updateStateForElementaryType({}, '{{query.name}}', result.value.toString());
+      await this.storeIPLDData(blockHash, contractAddress, stateUpdate, state);
+    }
+
+    {{else}}
+    assert(state === 'none', 'Type not supported for default state.');
+
+    {{/if}}
+    {{/if}}
     return result;
   }
 
@@ -210,8 +239,8 @@ export class Indexer {
   async processBlock (job: any): Promise<void> {
     const { data: { blockHash } } = job;
 
-    // Call custom post-block hook.
+    // Call custom stateDiff hook.
-    await postBlockHook(this, blockHash);
+    await createStateDiff(this, blockHash);
   }
 
   async processCheckpoint (job: any): Promise<void> {
@@ -225,11 +254,12 @@ export class Indexer {
 
     // Get all the contracts.
     const contracts = await this._db.getContracts({});
-    const contractAddresses = contracts.map(contract => contract.address);
 
-    // For each contractAddress, merge the diff till now to create a checkpoint.
+    // For each contract, merge the diff till now to create a checkpoint.
-    for (const contractAddress of contractAddresses) {
+    for (const contract of contracts) {
-      await this.createCheckpoint(contractAddress, currentBlockHash, checkpointInterval);
+      if (contract.checkpoint) {
+        await this.createCheckpoint(contract.address, currentBlockHash, checkpointInterval);
+      }
     }
   }
 
@@ -303,6 +333,15 @@ export class Indexer {
     return ipldBlocks[0];
   }
 
+  async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise<any> {
+    const ipldBlock = await this.getPrevIPLDBlock(blockHash, contractAddress, kind);
+
+    if (ipldBlock) {
+      const data = codec.decode(Buffer.from(ipldBlock.data)) as any;
+      return data.state;
+    }
+  }
+
   async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise<IPLDBlock | undefined> {
     const dbTx = await this._db.createTransactionRunner();
     let res;
@@ -336,11 +375,21 @@ export class Indexer {
     return res;
   }
 
+  async storeIPLDData (blockHash: string, contractAddress: string, data: any, kind: string): Promise<void> {
+    const block = await this.getBlockProgress(blockHash);
+    assert(block);
+
+    const ipldBlock = await this.prepareIPLDBlock(block, contractAddress, data, kind);
+    await this.saveOrUpdateIPLDBlock(ipldBlock);
+  }
+
   async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise<IPLDBlock> {
     return this._db.saveOrUpdateIPLDBlock(ipldBlock);
   }
 
   async prepareIPLDBlock (block: BlockProgress, contractAddress: string, data: any, kind: string):Promise<any> {
+    assert(_.includes(['diff', 'checkpoint'], kind));
+
     // Get an existing IPLDBlock for current block and contractAddress.
     const currentIPLDBlocks = await this.getIPLDBlocks(block, contractAddress, 'diff');
     // There can be only one IPLDBlock for a (block, contractAddress, 'diff') combination.
@@ -364,7 +413,7 @@ export class Indexer {
       // Setting the meta-data for an IPLDBlock (done only once per block).
       data.meta = {
         id: contractAddress,
-        kind: kind || 'diff',
+        kind,
         parent: {
           '/': parentIPLDBlock ? parentIPLDBlock.cid : null
         },
@@ -441,16 +490,21 @@ export class Indexer {
     return { eventName, eventInfo };
   }
 
-  async watchContract (address: string, kind: string, startingBlock: number): Promise<boolean> {
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<boolean> {
-    // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress).
+    // Use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress) if input to address is a contract address.
-    await this._db.saveContract(ethers.utils.getAddress(address), kind, startingBlock);
+    // If a contract identifier is passed as address instead, no need to convert to checksum address.
+    // Customize: use the kind input to filter out non-contract-address input to address.
+    const formattedAddress = (kind === '__protocol__') ? address : ethers.utils.getAddress(address);
+    await this._db.saveContract(formattedAddress, kind, checkpoint, startingBlock);
 
+    if (checkpoint) {
       // Getting the current block.
       const currentBlock = await this._db.getLatestBlockProgress();
       assert(currentBlock);
 
       // Call custom initial checkpoint hook.
-    await initialCheckpointHook(this, currentBlock, address);
+      await createInitialCheckpoint(this, currentBlock, address);
+    }
 
     return true;
   }

packages/codegen/src/templates/readme-template.handlebars

@@ -39,7 +39,7 @@
 
 * Update the `upstream` config in the [config file](./environments/local.toml) and provide the `ipld-eth-server` GQL API and the `indexer-db` postgraphile endpoints.
 
-* Update the [config](./environments/local.toml) with derived state checkpoint settings.
+* Update the [config](./environments/local.toml) with state checkpoint settings.
 
 ## Customize
 
@@ -47,11 +47,15 @@
 
   * Edit the custom hook function `handleEvent` (triggered on an event) in [hooks.ts](./src/hooks.ts) to perform corresponding indexing using the `Indexer` object.
 
-  * Edit the custom hook function `postBlockHook` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save `IPLDBlock`s using the `Indexer` object.
+  * While using the indexer storage methods for indexing, pass the optional arg. `state` as `diff` or `checkpoint` if default state is desired to be generated using the state variables being indexed else pass `none`.
 
-  * Edit the custom hook function `initialCheckpointHook` (triggered on watch-contract) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `IPLDBlock` using the `Indexer` object.
+* Generating state:
 
-  * The existing example hooks in [hooks.ts](./src/hooks.ts) are for an `ERC20` contract.
+  * Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `IPLDBlock` using the `Indexer` object.
+
+  * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in an `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated.
+
+* The existing example hooks in [hooks.ts](./src/hooks.ts) are for an `ERC20` contract.
 
 ## Run
 
@@ -74,7 +78,26 @@ GQL console: http://localhost:3008/graphql
   * To watch a contract:
 
     ```bash
-    yarn watch:contract --address <contract-address> --kind {{contractName}} --starting-block [block-number]
+    yarn watch:contract --address <contract-address> --kind <contract-kind> --checkpoint <true | false> --starting-block [block-number]
+    ```
+
+    * `address`: Address or identifier of the contract to be watched.
+    * `kind`: Kind of the contract.
+    * `checkpoint`: Turn checkpointing on (`true` | `false`).
+    * `starting-block`: Starting block for the contract (default: `1`).
+
+    Examples:
+    
+    Watch a contract with its address and checkpointing on:
+
+    ```bash
+    yarn watch:contract --address 0x1F78641644feB8b64642e833cE4AFE93DD6e7833 --kind ERC20 --checkpoint true
+    ```
+
+    Watch a contract with its identifier and checkpointing on:
+
+    ```bash
+    yarn watch:contract --address MyProtocol --kind protocol --checkpoint true
     ```
 
   * To fill a block range:
@@ -83,8 +106,14 @@ GQL console: http://localhost:3008/graphql
     yarn fill --start-block <from-block> --end-block <to-block>
     ```
 
+    * `start-block`: Block number to start filling from.
+    * `end-block`: Block number till which to fill. 
+
   * To create a checkpoint for a contract:
 
     ```bash
     yarn checkpoint --address <contract-address> --block-hash [block-hash]
     ```
+
+    * `address`: Address or identifier of the contract for which to create a checkpoint.
+    * `block-hash`: Hash of the block at which to create the checkpoint (default: current block hash).

packages/codegen/src/templates/resolvers-template.handlebars

@@ -34,9 +34,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
     },
 
     Mutation: {
-      watchContract: (_: any, { contractAddress, kind, startingBlock = 1 }: { contractAddress: string, kind: string, startingBlock: number }): Promise<boolean> => {
+      watchContract: (_: any, { address, kind, checkpoint, startingBlock = 1 }: { address: string, kind: string, checkpoint: boolean, startingBlock: number }): Promise<boolean> => {
-        log('watchContract', contractAddress, kind, startingBlock);
+        log('watchContract', address, kind, checkpoint, startingBlock);
-        return indexer.watchContract(contractAddress, kind, startingBlock);
+        return indexer.watchContract(address, kind, checkpoint, startingBlock);
       }
     },
 

packages/codegen/src/templates/watch-contract-template.handlebars

@@ -41,6 +41,12 @@ const main = async (): Promise<void> => {
       demandOption: true,
       describe: 'Kind of contract'
     },
+    checkpoint: {
+      type: 'boolean',
+      require: true,
+      demandOption: true,
+      describe: 'Turn checkpointing on'
+    },
     startingBlock: {
       type: 'number',
       default: 1,
@@ -79,7 +85,7 @@ const main = async (): Promise<void> => {
   const ethProvider = getDefaultProvider(rpcProviderEndpoint);
 
   const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider);
-  await indexer.watchContract(argv.address, argv.kind, argv.startingBlock);
+  await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock);
 
   await db.close();
 };

packages/codegen/src/visitor.ts

@@ -60,11 +60,13 @@ export class Visitor {
   stateVariableDeclarationVisitor (node: any): void {
     // TODO Handle multiples variables in a single line.
     // TODO Handle array types.
-    const name: string = node.variables[0].name;
+    const variable = node.variables[0];
+    const name: string = variable.name;
+    const stateVariableType: string = variable.typeName.type;
 
     const params: Param[] = [];
 
-    let typeName = node.variables[0].typeName;
+    let typeName = variable.typeName;
     let numParams = 0;
 
     // If the variable type is mapping, extract key as a param:
@@ -79,7 +81,7 @@ export class Visitor {
 
     this._schema.addQuery(name, params, returnType);
     this._resolvers.addQuery(name, params, returnType);
-    this._indexer.addQuery(MODE_STORAGE, name, params, returnType);
+    this._indexer.addQuery(MODE_STORAGE, name, params, returnType, stateVariableType);
     this._entity.addQuery(name, params, returnType);
     this._database.addQuery(name, params, returnType);
     this._client.addQuery(name, params, returnType);

packages/erc20-watcher/src/cli/watch-contract.ts

@@ -29,6 +29,11 @@ import { Indexer } from '../indexer';
       demandOption: true,
       describe: 'Address of the deployed contract'
     },
+    checkpoint: {
+      type: 'boolean',
+      default: false,
+      describe: 'Turn checkpointing on'
+    },
     startingBlock: {
       type: 'number',
       default: 1,
@@ -55,7 +60,7 @@ import { Indexer } from '../indexer';
 
   const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, jobQueue, mode);
 
-  await indexer.watchContract(argv.address, argv.startingBlock);
+  await indexer.watchContract(argv.address, argv.checkpoint, argv.startingBlock);
 
   await db.close();
   await jobQueue.stop();

packages/erc20-watcher/src/database.ts

@@ -114,10 +114,10 @@ export class Database {
     return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, startingBlock, kind);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {

packages/erc20-watcher/src/entity/Contract.ts

@@ -16,6 +16,9 @@ export class Contract {
   @Column('varchar', { length: 8 })
   kind!: string;
 
+  @Column('boolean')
+  checkpoint!: boolean;
+
   @Column('integer')
   startingBlock!: number;
 }

packages/erc20-watcher/src/indexer.ts

@@ -305,8 +305,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
-  async watchContract (address: string, startingBlock: number): Promise<void> {
+  async watchContract (address: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, CONTRACT_KIND, startingBlock);
+    return this._baseIndexer.watchContract(address, CONTRACT_KIND, checkpoint, startingBlock);
   }
 
   async saveEventEntity (dbEvent: Event): Promise<Event> {

packages/erc20-watcher/src/resolvers.ts

@@ -34,9 +34,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
     },
 
     Mutation: {
-      watchToken: async (_: any, { token, startingBlock = 1 }: { token: string, startingBlock: number }): Promise<boolean> => {
+      watchToken: async (_: any, { token, checkpoint = false, startingBlock = 1 }: { token: string, checkpoint: boolean, startingBlock: number }): Promise<boolean> => {
-        log('watchToken', token, startingBlock);
+        log('watchToken', token, checkpoint, startingBlock);
-        await indexer.watchContract(token, startingBlock);
+        await indexer.watchContract(token, checkpoint, startingBlock);
 
         return true;
       }

packages/erc20-watcher/src/schema.ts

@@ -158,6 +158,7 @@ type Mutation {
   # Actively watch and index data for the token.
   watchToken(
     token: String!
+    checkpoint: Boolean
     startingBlock: Int
   ): Boolean!
 }

packages/uni-watcher/src/cli/watch-contract.ts

@@ -35,6 +35,11 @@ import { Indexer } from '../indexer';
       demandOption: true,
       describe: 'Kind of contract (factory|pool|nfpm)'
     },
+    checkpoint: {
+      type: 'boolean',
+      default: false,
+      describe: 'Turn checkpointing on'
+    },
     startingBlock: {
       type: 'number',
       default: 1,
@@ -62,7 +67,7 @@ import { Indexer } from '../indexer';
   const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, jobQueue);
   await indexer.init();
 
-  await indexer.watchContract(argv.address, argv.kind, argv.startingBlock);
+  await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock);
 
   await db.close();
   await jobQueue.stop();

packages/uni-watcher/src/database.ts

@@ -51,10 +51,10 @@ export class Database implements DatabaseInterface {
     return this._baseDatabase.getContracts(repo);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, startingBlock, kind);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
   }
 
   async createTransactionRunner (): Promise<QueryRunner> {

packages/uni-watcher/src/entity/Contract.ts

@@ -20,6 +20,9 @@ export class Contract {
   @Column('varchar', { length: 8 })
   kind!: string;
 
+  @Column('boolean')
+  checkpoint!: boolean;
+
   @Column('integer')
   startingBlock!: number;
 }

packages/uni-watcher/src/indexer.ts

@@ -102,7 +102,7 @@ export class Indexer implements IndexerInterface {
     switch (re.event.__typename) {
       case 'PoolCreatedEvent': {
         const poolContract = ethers.utils.getAddress(re.event.pool);
-        await this.watchContract(poolContract, KIND_POOL, dbEvent.block.blockNumber);
+        await this.watchContract(poolContract, KIND_POOL, false, dbEvent.block.blockNumber);
       }
     }
   }
@@ -353,8 +353,8 @@ export class Indexer implements IndexerInterface {
     return this._baseIndexer.isWatchedContract(address);
   }
 
-  async watchContract (address: string, kind: string, startingBlock: number): Promise<void> {
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
-    return this._baseIndexer.watchContract(address, kind, startingBlock);
+    return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock);
   }
 
   cacheContract (contract: Contract): void {

packages/uni-watcher/test/init.ts

@@ -30,7 +30,7 @@ const deployFactoryContract = async (indexer: Indexer, signer: Signer): Promise<
   assert(factory.address, 'Factory contract not deployed.');
 
   // Watch factory contract.
-  await indexer.watchContract(factory.address, 'factory', 100);
+  await indexer.watchContract(factory.address, 'factory', false, 100);
 
   return factory;
 };
@@ -45,7 +45,7 @@ const deployNFPMContract = async (indexer: Indexer, signer: Signer, factory: Con
   assert(nfpm.address, 'NFPM contract not deployed.');
 
   // Watch NFPM contract.
-  await indexer.watchContract(nfpm.address, 'nfpm', 100);
+  await indexer.watchContract(nfpm.address, 'nfpm', false, 100);
 };
 
 const main = async () => {

packages/util/index.ts

@@ -13,3 +13,4 @@ export * from './src/types';
 export * from './src/indexer';
 export * from './src/job-runner';
 export * from './src/graph-decimal';
+export * from './src/ipldHelper';

packages/util/src/database.ts

@@ -538,13 +538,13 @@ export class Database {
       .getMany();
   }
 
-  async saveContract (repo: Repository<ContractInterface>, address: string, startingBlock: number, kind?: string): Promise<ContractInterface> {
+  async saveContract (repo: Repository<ContractInterface>, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<ContractInterface> {
     const contract = await repo
       .createQueryBuilder()
       .where('address = :address', { address })
       .getOne();
 
-    const entity = repo.create({ address, kind, startingBlock });
+    const entity = repo.create({ address, kind, checkpoint, startingBlock });
 
     // If contract already present, overwrite fields.
     if (contract) {

packages/util/src/indexer.ts

@@ -308,7 +308,7 @@ export class Indexer {
     return this._watchedContracts[address];
   }
 
-  async watchContract (address: string, kind: string, startingBlock: number): Promise<void> {
+  async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<void> {
     assert(this._db.saveContract);
     const dbTx = await this._db.createTransactionRunner();
 
@@ -316,7 +316,7 @@ export class Indexer {
     const contractAddress = ethers.utils.getAddress(address);
 
     try {
-      const contract = await this._db.saveContract(dbTx, contractAddress, kind, startingBlock);
+      const contract = await this._db.saveContract(dbTx, contractAddress, kind, checkpoint, startingBlock);
       this.cacheContract(contract);
       await dbTx.commitTransaction();
 

packages/util/src/ipldHelper.ts

@@ -0,0 +1,16 @@
+import _ from 'lodash';
+
+export const updateStateForElementaryType = (initialObject: any, stateVariable: string, value: string): any => {
+  const object = _.cloneDeep(initialObject);
+  const path = ['state', stateVariable];
+
+  return _.set(object, path, value);
+};
+
+export const updateStateForMappingType = (initialObject: any, stateVariable: string, keys: string[], value: string): any => {
+  const object = _.cloneDeep(initialObject);
+  keys.unshift('state', stateVariable);
+
+  // Use _.setWith() with Object as customizer as _.set() treats numeric value in path as an index to an array.
+  return _.setWith(object, keys, value, Object);
+};

packages/util/src/types.ts

@@ -48,6 +48,7 @@ export interface ContractInterface {
   address: string;
   startingBlock: number;
   kind: string;
+  checkpoint: boolean;
 }
 
 export interface IndexerInterface {
@@ -101,5 +102,5 @@ export interface DatabaseInterface {
   saveEventEntity (queryRunner: QueryRunner, entity: EventInterface): Promise<EventInterface>;
   removeEntities<Entity> (queryRunner: QueryRunner, entity: new () => Entity, findConditions?: FindManyOptions<Entity> | FindConditions<Entity>): Promise<void>;
   getContracts?: () => Promise<ContractInterface[]>
-  saveContract?: (queryRunner: QueryRunner, contractAddress: string, kind: string, startingBlock: number) => Promise<ContractInterface>
+  saveContract?: (queryRunner: QueryRunner, contractAddress: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise<ContractInterface>
 }