diff --git a/packages/codegen/README.md b/packages/codegen/README.md index a5f1cc57..7b9c3791 100644 --- a/packages/codegen/README.md +++ b/packages/codegen/README.md @@ -29,16 +29,16 @@ * `output-folder`(alias: `o`): Output folder path. (logs output using `stdout` if not provided). * `mode`(alias: `m`): Code generation mode (default: `all`). * `flatten`(alias: `f`): Flatten the input contract file (default: `true`). - * `kind` (alias: `k`): Kind of watcher (default; `active`). + * `kind` (alias: `k`): Kind of watcher (default: `active`). **Note**: When passed an *URL* as `input-file`, it is assumed that it points to an already flattened contract file. Examples: - Generate code in both `eth_call` and `storage` mode, `active` kind. + Generate code in `storage` mode, `lazy` kind. ```bash - yarn codegen --input-file ./test/examples/contracts/ERC20.sol --contract-name ERC20 --output-folder ../my-erc20-watcher --mode all --kind active + yarn codegen --input-file ./test/examples/contracts/ERC721.sol --contract-name ERC721 --output-folder ../my-erc721-watcher --mode storage --kind lazy ``` Generate code in `eth_call` mode using a contract provided by an URL. @@ -47,10 +47,10 @@ yarn codegen --input-file https://git.io/Jupci --contract-name ERC721 --output-folder ../my-erc721-watcher --mode eth_call ``` - Generate code in `storage` mode, `lazy` kind. + Generate code for `ERC721` in both `eth_call` and `storage` mode, `active` kind. ```bash - yarn codegen --input-file ./test/examples/contracts/ERC721.sol --contract-name ERC721 --output-folder ../my-erc721-watcher --mode storage --kind lazy + yarn codegen --input-file ../../node_modules/@openzeppelin/contracts/token/ERC721/ERC721.sol --contract-name ERC721 --output-folder ../demo-erc721-watcher --mode all --kind active ``` Generate code for `ERC20` contract in both `eth_call` and `storage` mode, `active` kind: @@ -73,7 +73,7 @@ * Create the databases configured in `environments/local.toml`. -* Update the derived state checkpoint settings in `environments/local.toml`. +* Update the state checkpoint settings in `environments/local.toml`. ### Customize @@ -81,11 +81,15 @@ * Edit the custom hook function `handleEvent` (triggered on an event) in `src/hooks.ts` to perform corresponding indexing using the `Indexer` object. - * Edit the custom hook function `postBlockHook` (triggered on a block) in `src/hooks.ts` to save `IPLDBlock`s using the `Indexer` object. + * While using the indexer storage methods for indexing, pass the optional arg. `state` as `diff` or `checkpoint` if default state is desired to be generated using the state variables being indexed else pass `none`. - * Edit the custom hook function `initialCheckpointHook` (triggered on watch-contract) in `src/hooks.ts` to save an initial checkpoint `IPLDBlock` using the `Indexer` object. +* Generating state: - * The existing example hooks in `src/hooks.ts` are for an `ERC20` contract. + * Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in `src/hooks.ts` to save an initial checkpoint `IPLDBlock` using the `Indexer` object. + + * Edit the custom hook function `createStateDiff` (triggered on a block) in `src/hooks.ts` to save the state in an `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + +* The existing example hooks in `src/hooks.ts` are for an `ERC20` contract. ### Run @@ -112,7 +116,7 @@ * To watch a contract: ```bash - yarn watch:contract --address --kind --starting-block [block-number] + yarn watch:contract --address --kind --checkpoint --starting-block [block-number] ``` * To fill a block range: diff --git a/packages/codegen/src/data/entities/Contract.yaml b/packages/codegen/src/data/entities/Contract.yaml index e7019fbf..f01e768c 100644 --- a/packages/codegen/src/data/entities/Contract.yaml +++ b/packages/codegen/src/data/entities/Contract.yaml @@ -15,9 +15,10 @@ columns: pgType: varchar tsType: string columnType: Column - columnOptions: - - option: length - value: 8 + - name: checkpoint + pgType: boolean + tsType: boolean + columnType: Column - name: startingBlock pgType: integer tsType: number diff --git a/packages/codegen/src/indexer.ts b/packages/codegen/src/indexer.ts index 329ad66a..ef0d9769 100644 --- a/packages/codegen/src/indexer.ts +++ b/packages/codegen/src/indexer.ts @@ -32,8 +32,9 @@ export class Indexer { * @param name Name of the query. * @param params Parameters to the query. * @param returnType Return type for the query. + * @param stateVariableTypeName Type of the state variable in case of state variable query. */ - addQuery (mode: string, name: string, params: Array, returnType: string): void { + addQuery (mode: string, name: string, params: Array, returnType: string, stateVariableType?: string): void { // Check if the query is already added. if (this._queries.some(query => query.name === name)) { return; @@ -45,7 +46,8 @@ export class Indexer { saveQueryName: '', params: _.cloneDeep(params), returnType, - mode + mode, + stateVariableType }; if (name.charAt(0) === '_') { @@ -69,6 +71,10 @@ export class Indexer { assert(tsReturnType); queryObject.returnType = tsReturnType; + if (stateVariableType) { + queryObject.stateVariableType = stateVariableType; + } + this._queries.push(queryObject); } diff --git a/packages/codegen/src/schema.ts b/packages/codegen/src/schema.ts index 76c36e99..615b45e6 100644 --- a/packages/codegen/src/schema.ts +++ b/packages/codegen/src/schema.ts @@ -292,8 +292,9 @@ export class Schema { watchContract: { type: 'Boolean!', args: { - contractAddress: 'String!', + address: 'String!', kind: 'String!', + checkpoint: 'Boolean!', startingBlock: 'Int' } } diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index ad20defc..26efb854 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -3,7 +3,7 @@ port = 3008 kind = "{{watcherKind}}" - # Checkpointing derived state. + # Checkpointing state. checkpointing = true # Checkpoint interval in number of blocks. diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index e0e6d6b2..8c94fb6d 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -309,11 +309,11 @@ export class Database { return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events); } - async saveContract (address: string, kind: string, startingBlock: number): Promise { + async saveContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { await this._conn.transaction(async (tx) => { const repo = tx.getRepository(Contract); - return this._baseDatabase.saveContract(repo, address, startingBlock, kind); + return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock); }); } diff --git a/packages/codegen/src/templates/hooks-template.handlebars b/packages/codegen/src/templates/hooks-template.handlebars index 132f8e34..9e600051 100644 --- a/packages/codegen/src/templates/hooks-template.handlebars +++ b/packages/codegen/src/templates/hooks-template.handlebars @@ -3,9 +3,8 @@ // import assert from 'assert'; -import _ from 'lodash'; -import { UNKNOWN_EVENT_NAME } from '@vulcanize/util'; +import { UNKNOWN_EVENT_NAME, updateStateForMappingType, updateStateForElementaryType } from '@vulcanize/util'; import { Indexer, ResultEvent } from './indexer'; import { BlockProgress } from './entity/BlockProgress'; @@ -15,19 +14,23 @@ const ACCOUNTS = [ ]; /** - * Initial checkpoint hook function. + * Hook function to create an initial checkpoint. * @param indexer Indexer instance. * @param block Concerned block. * @param contractAddress Address of the concerned contract. */ -export async function initialCheckpointHook (indexer: Indexer, block: BlockProgress, contractAddress: string): Promise { +export async function createInitialCheckpoint (indexer: Indexer, block: BlockProgress, contractAddress: string): Promise { + assert(indexer); + assert(block); + assert(contractAddress); + // Store the initial state values in an IPLDBlock. - const ipldBlockData: any = {}; + let ipldBlockData: any = {}; // Setting the initial balances of accounts. for (const account of ACCOUNTS) { const balance = await indexer._balances(block.blockHash, contractAddress, account); - _.set(ipldBlockData, `state._balances[${account}]`, balance.value.toString()); + ipldBlockData = updateStateForMappingType(ipldBlockData, '_balances', [account], balance.value.toString()); } const ipldBlock = await indexer.prepareIPLDBlock(block, contractAddress, ipldBlockData, 'checkpoint'); @@ -35,11 +38,14 @@ export async function initialCheckpointHook (indexer: Indexer, block: BlockProgr } /** - * Post-block hook function. + * Hook function to create and store state diffs. * @param indexer Indexer instance that contains methods to fetch the contract varaiable values. * @param blockHash Block hash of the concerned block. */ -export async function postBlockHook (indexer: Indexer, blockHash: string): Promise { +export async function createStateDiff (indexer: Indexer, blockHash: string): Promise { + assert(indexer); + assert(blockHash); + // Get events for current block and make an entry of updated values in IPLDBlock. const events = await indexer.getEventsByFilter(blockHash); @@ -58,7 +64,7 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi const eventData = indexer.getResultEvent(event); - const ipldBlockData: any = {}; + let ipldBlockData: any = {}; switch (event.eventName) { case 'Transfer': { @@ -73,8 +79,8 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi // "0xDC7d7A8920C8Eecc098da5B7522a5F31509b5Bfc": "999999999999999999900" // } // } - _.set(ipldBlockData, `state._balances[${from}]`, fromBalance.value.toString()); - _.set(ipldBlockData, `state._balances[${to}]`, toBalance.value.toString()); + ipldBlockData = updateStateForMappingType(ipldBlockData, '_balances', [from], fromBalance.value.toString()); + ipldBlockData = updateStateForMappingType(ipldBlockData, '_balances', [to], toBalance.value.toString()); break; } @@ -90,7 +96,7 @@ export async function postBlockHook (indexer: Indexer, blockHash: string): Promi // } // } // } - _.set(ipldBlockData, `state._allowances[${owner}][${spender}]`, allowance.value.toString()); + ipldBlockData = updateStateForMappingType(ipldBlockData, '_allowances', [owner, spender], allowance.value.toString()); break; } @@ -123,10 +129,10 @@ export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Pr const { from, to } = eventData.event; // Update balance entry for sender in the database. - await indexer.balanceOf(eventData.block.hash, eventData.contract, from); + await indexer._balances(eventData.block.hash, eventData.contract, from); // Update balance entry for receiver in the database. - await indexer.balanceOf(eventData.block.hash, eventData.contract, to); + await indexer._balances(eventData.block.hash, eventData.contract, to); break; } @@ -138,7 +144,7 @@ export async function handleEvent (indexer: Indexer, eventData: ResultEvent): Pr const { owner, spender } = eventData.event; // Update allowance entry for (owner, spender) combination in the database. - await indexer.allowance(eventData.block.hash, eventData.contract, owner, spender); + await indexer._allowances(eventData.block.hash, eventData.contract, owner, spender); break; } diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index c8d89a1b..55cfaabf 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -16,7 +16,7 @@ import { BaseProvider } from '@ethersproject/providers'; import * as codec from '@ipld/dag-json'; import { EthClient } from '@vulcanize/ipld-eth-client'; import { StorageLayout } from '@vulcanize/solidity-mapper'; -import { Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, ServerConfig, Where, QueryOptions } from '@vulcanize/util'; +import { Indexer as BaseIndexer, ValueResult, UNKNOWN_EVENT_NAME, ServerConfig, Where, QueryOptions, updateStateForElementaryType, updateStateForMappingType } from '@vulcanize/util'; import { Database } from './database'; import { Contract } from './entity/Contract'; @@ -26,7 +26,7 @@ import { HookStatus } from './entity/HookStatus'; import { BlockProgress } from './entity/BlockProgress'; import { IPLDBlock } from './entity/IPLDBlock'; import artifacts from './artifacts/{{inputFileName}}.json'; -import { initialCheckpointHook, handleEvent, postBlockHook } from './hooks'; +import { createInitialCheckpoint, handleEvent, createStateDiff } from './hooks'; const log = debug('vulcanize:indexer'); @@ -75,7 +75,7 @@ export class Indexer { _db: Database _ethClient: EthClient _ethProvider: BaseProvider - _postgraphileClient: EthClient; + _postgraphileClient: EthClient _baseIndexer: BaseIndexer _serverConfig: ServerConfig @@ -158,7 +158,14 @@ export class Indexer { {{#each queries as | query |}} async {{query.name}} (blockHash: string, contractAddress: string - {{~#each query.params}}, {{this.name~}}: {{this.type~}} {{/each}}): Promise { + {{~#each query.params}}, {{this.name~}}: {{this.type~}} {{/each}} + {{~#if query.stateVariableType~}} + , state = 'none'): Promise { + assert(_.includes(['diff', 'checkpoint', 'none'], state)); + + {{else~}} + ): Promise { + {{/if}} const entity = await this._db.{{query.getQueryName}}({ blockHash, contractAddress {{~#each query.params}}, {{this.name~}} {{~/each}} }); if (entity) { @@ -203,6 +210,28 @@ export class Indexer { await this._db.{{query.saveQueryName}}({ blockHash, contractAddress {{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbig.stringify(result.proof) }); + {{#if query.stateVariableType}} + {{#if (compare query.stateVariableType 'Mapping')}} + if (state !== 'none') { + const stateUpdate = updateStateForMappingType({}, '{{query.name}}', [ + {{~#each query.params}} + {{~this.name}}.toString() {{~#unless @last}}, {{/unless~}} + {{/each~}} + ], result.value.toString()); + await this.storeIPLDData(blockHash, contractAddress, stateUpdate, state); + } + + {{else if (compare query.stateVariableType 'ElementaryTypeName')}} + if (state !== 'none') { + const stateUpdate = updateStateForElementaryType({}, '{{query.name}}', result.value.toString()); + await this.storeIPLDData(blockHash, contractAddress, stateUpdate, state); + } + + {{else}} + assert(state === 'none', 'Type not supported for default state.'); + + {{/if}} + {{/if}} return result; } @@ -210,8 +239,8 @@ export class Indexer { async processBlock (job: any): Promise { const { data: { blockHash } } = job; - // Call custom post-block hook. - await postBlockHook(this, blockHash); + // Call custom stateDiff hook. + await createStateDiff(this, blockHash); } async processCheckpoint (job: any): Promise { @@ -225,11 +254,12 @@ export class Indexer { // Get all the contracts. const contracts = await this._db.getContracts({}); - const contractAddresses = contracts.map(contract => contract.address); - // For each contractAddress, merge the diff till now to create a checkpoint. - for (const contractAddress of contractAddresses) { - await this.createCheckpoint(contractAddress, currentBlockHash, checkpointInterval); + // For each contract, merge the diff till now to create a checkpoint. + for (const contract of contracts) { + if (contract.checkpoint) { + await this.createCheckpoint(contract.address, currentBlockHash, checkpointInterval); + } } } @@ -303,6 +333,15 @@ export class Indexer { return ipldBlocks[0]; } + async getPrevState (blockHash: string, contractAddress: string, kind?: string): Promise { + const ipldBlock = await this.getPrevIPLDBlock(blockHash, contractAddress, kind); + + if (ipldBlock) { + const data = codec.decode(Buffer.from(ipldBlock.data)) as any; + return data.state; + } + } + async getPrevIPLDBlock (blockHash: string, contractAddress: string, kind?: string): Promise { const dbTx = await this._db.createTransactionRunner(); let res; @@ -336,11 +375,21 @@ export class Indexer { return res; } + async storeIPLDData (blockHash: string, contractAddress: string, data: any, kind: string): Promise { + const block = await this.getBlockProgress(blockHash); + assert(block); + + const ipldBlock = await this.prepareIPLDBlock(block, contractAddress, data, kind); + await this.saveOrUpdateIPLDBlock(ipldBlock); + } + async saveOrUpdateIPLDBlock (ipldBlock: IPLDBlock): Promise { return this._db.saveOrUpdateIPLDBlock(ipldBlock); } async prepareIPLDBlock (block: BlockProgress, contractAddress: string, data: any, kind: string):Promise { + assert(_.includes(['diff', 'checkpoint'], kind)); + // Get an existing IPLDBlock for current block and contractAddress. const currentIPLDBlocks = await this.getIPLDBlocks(block, contractAddress, 'diff'); // There can be only one IPLDBlock for a (block, contractAddress, 'diff') combination. @@ -364,7 +413,7 @@ export class Indexer { // Setting the meta-data for an IPLDBlock (done only once per block). data.meta = { id: contractAddress, - kind: kind || 'diff', + kind, parent: { '/': parentIPLDBlock ? parentIPLDBlock.cid : null }, @@ -441,16 +490,21 @@ export class Indexer { return { eventName, eventInfo }; } - async watchContract (address: string, kind: string, startingBlock: number): Promise { - // Always use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress). - await this._db.saveContract(ethers.utils.getAddress(address), kind, startingBlock); + async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { + // Use the checksum address (https://docs.ethers.io/v5/api/utils/address/#utils-getAddress) if input to address is a contract address. + // If a contract identifier is passed as address instead, no need to convert to checksum address. + // Customize: use the kind input to filter out non-contract-address input to address. + const formattedAddress = (kind === '__protocol__') ? address : ethers.utils.getAddress(address); + await this._db.saveContract(formattedAddress, kind, checkpoint, startingBlock); - // Getting the current block. - const currentBlock = await this._db.getLatestBlockProgress(); - assert(currentBlock); + if (checkpoint) { + // Getting the current block. + const currentBlock = await this._db.getLatestBlockProgress(); + assert(currentBlock); - // Call custom initial checkpoint hook. - await initialCheckpointHook(this, currentBlock, address); + // Call custom initial checkpoint hook. + await createInitialCheckpoint(this, currentBlock, address); + } return true; } diff --git a/packages/codegen/src/templates/readme-template.handlebars b/packages/codegen/src/templates/readme-template.handlebars index 6b214eb6..79f4359c 100644 --- a/packages/codegen/src/templates/readme-template.handlebars +++ b/packages/codegen/src/templates/readme-template.handlebars @@ -39,7 +39,7 @@ * Update the `upstream` config in the [config file](./environments/local.toml) and provide the `ipld-eth-server` GQL API and the `indexer-db` postgraphile endpoints. -* Update the [config](./environments/local.toml) with derived state checkpoint settings. +* Update the [config](./environments/local.toml) with state checkpoint settings. ## Customize @@ -47,11 +47,15 @@ * Edit the custom hook function `handleEvent` (triggered on an event) in [hooks.ts](./src/hooks.ts) to perform corresponding indexing using the `Indexer` object. - * Edit the custom hook function `postBlockHook` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save `IPLDBlock`s using the `Indexer` object. + * While using the indexer storage methods for indexing, pass the optional arg. `state` as `diff` or `checkpoint` if default state is desired to be generated using the state variables being indexed else pass `none`. - * Edit the custom hook function `initialCheckpointHook` (triggered on watch-contract) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `IPLDBlock` using the `Indexer` object. +* Generating state: - * The existing example hooks in [hooks.ts](./src/hooks.ts) are for an `ERC20` contract. + * Edit the custom hook function `createInitialCheckpoint` (triggered on watch-contract, checkpoint: `true`) in [hooks.ts](./src/hooks.ts) to save an initial checkpoint `IPLDBlock` using the `Indexer` object. + + * Edit the custom hook function `createStateDiff` (triggered on a block) in [hooks.ts](./src/hooks.ts) to save the state in an `IPLDBlock` using the `Indexer` object. The default state (if exists) is updated. + +* The existing example hooks in [hooks.ts](./src/hooks.ts) are for an `ERC20` contract. ## Run @@ -74,7 +78,26 @@ GQL console: http://localhost:3008/graphql * To watch a contract: ```bash - yarn watch:contract --address --kind {{contractName}} --starting-block [block-number] + yarn watch:contract --address --kind --checkpoint --starting-block [block-number] + ``` + + * `address`: Address or identifier of the contract to be watched. + * `kind`: Kind of the contract. + * `checkpoint`: Turn checkpointing on (`true` | `false`). + * `starting-block`: Starting block for the contract (default: `1`). + + Examples: + + Watch a contract with its address and checkpointing on: + + ```bash + yarn watch:contract --address 0x1F78641644feB8b64642e833cE4AFE93DD6e7833 --kind ERC20 --checkpoint true + ``` + + Watch a contract with its identifier and checkpointing on: + + ```bash + yarn watch:contract --address MyProtocol --kind protocol --checkpoint true ``` * To fill a block range: @@ -83,8 +106,14 @@ GQL console: http://localhost:3008/graphql yarn fill --start-block --end-block ``` + * `start-block`: Block number to start filling from. + * `end-block`: Block number till which to fill. + * To create a checkpoint for a contract: ```bash yarn checkpoint --address --block-hash [block-hash] ``` + + * `address`: Address or identifier of the contract for which to create a checkpoint. + * `block-hash`: Hash of the block at which to create the checkpoint (default: current block hash). diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index f9c938a6..7d90e654 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -34,9 +34,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch }, Mutation: { - watchContract: (_: any, { contractAddress, kind, startingBlock = 1 }: { contractAddress: string, kind: string, startingBlock: number }): Promise => { - log('watchContract', contractAddress, kind, startingBlock); - return indexer.watchContract(contractAddress, kind, startingBlock); + watchContract: (_: any, { address, kind, checkpoint, startingBlock = 1 }: { address: string, kind: string, checkpoint: boolean, startingBlock: number }): Promise => { + log('watchContract', address, kind, checkpoint, startingBlock); + return indexer.watchContract(address, kind, checkpoint, startingBlock); } }, diff --git a/packages/codegen/src/templates/watch-contract-template.handlebars b/packages/codegen/src/templates/watch-contract-template.handlebars index fce34a7c..61603fc6 100644 --- a/packages/codegen/src/templates/watch-contract-template.handlebars +++ b/packages/codegen/src/templates/watch-contract-template.handlebars @@ -41,6 +41,12 @@ const main = async (): Promise => { demandOption: true, describe: 'Kind of contract' }, + checkpoint: { + type: 'boolean', + require: true, + demandOption: true, + describe: 'Turn checkpointing on' + }, startingBlock: { type: 'number', default: 1, @@ -79,7 +85,7 @@ const main = async (): Promise => { const ethProvider = getDefaultProvider(rpcProviderEndpoint); const indexer = new Indexer(serverConfig, db, ethClient, postgraphileClient, ethProvider); - await indexer.watchContract(argv.address, argv.kind, argv.startingBlock); + await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock); await db.close(); }; diff --git a/packages/codegen/src/visitor.ts b/packages/codegen/src/visitor.ts index 15d1146d..df972666 100644 --- a/packages/codegen/src/visitor.ts +++ b/packages/codegen/src/visitor.ts @@ -60,11 +60,13 @@ export class Visitor { stateVariableDeclarationVisitor (node: any): void { // TODO Handle multiples variables in a single line. // TODO Handle array types. - const name: string = node.variables[0].name; + const variable = node.variables[0]; + const name: string = variable.name; + const stateVariableType: string = variable.typeName.type; const params: Param[] = []; - let typeName = node.variables[0].typeName; + let typeName = variable.typeName; let numParams = 0; // If the variable type is mapping, extract key as a param: @@ -79,7 +81,7 @@ export class Visitor { this._schema.addQuery(name, params, returnType); this._resolvers.addQuery(name, params, returnType); - this._indexer.addQuery(MODE_STORAGE, name, params, returnType); + this._indexer.addQuery(MODE_STORAGE, name, params, returnType, stateVariableType); this._entity.addQuery(name, params, returnType); this._database.addQuery(name, params, returnType); this._client.addQuery(name, params, returnType); diff --git a/packages/erc20-watcher/src/cli/watch-contract.ts b/packages/erc20-watcher/src/cli/watch-contract.ts index 60e0d662..26a13786 100644 --- a/packages/erc20-watcher/src/cli/watch-contract.ts +++ b/packages/erc20-watcher/src/cli/watch-contract.ts @@ -29,6 +29,11 @@ import { Indexer } from '../indexer'; demandOption: true, describe: 'Address of the deployed contract' }, + checkpoint: { + type: 'boolean', + default: false, + describe: 'Turn checkpointing on' + }, startingBlock: { type: 'number', default: 1, @@ -55,7 +60,7 @@ import { Indexer } from '../indexer'; const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, jobQueue, mode); - await indexer.watchContract(argv.address, argv.startingBlock); + await indexer.watchContract(argv.address, argv.checkpoint, argv.startingBlock); await db.close(); await jobQueue.stop(); diff --git a/packages/erc20-watcher/src/database.ts b/packages/erc20-watcher/src/database.ts index 5c1a8e8d..aa39e2dd 100644 --- a/packages/erc20-watcher/src/database.ts +++ b/packages/erc20-watcher/src/database.ts @@ -114,10 +114,10 @@ export class Database { return this._baseDatabase.saveEvents(blockRepo, eventRepo, block, events); } - async saveContract (queryRunner: QueryRunner, address: string, kind: string, startingBlock: number): Promise { + async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { const repo = queryRunner.manager.getRepository(Contract); - return this._baseDatabase.saveContract(repo, address, startingBlock, kind); + return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock); } async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise { diff --git a/packages/erc20-watcher/src/entity/Contract.ts b/packages/erc20-watcher/src/entity/Contract.ts index 83c99dcb..5d070e6a 100644 --- a/packages/erc20-watcher/src/entity/Contract.ts +++ b/packages/erc20-watcher/src/entity/Contract.ts @@ -16,6 +16,9 @@ export class Contract { @Column('varchar', { length: 8 }) kind!: string; + @Column('boolean') + checkpoint!: boolean; + @Column('integer') startingBlock!: number; } diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts index 4dac2f7d..8753705b 100644 --- a/packages/erc20-watcher/src/indexer.ts +++ b/packages/erc20-watcher/src/indexer.ts @@ -305,8 +305,8 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.isWatchedContract(address); } - async watchContract (address: string, startingBlock: number): Promise { - return this._baseIndexer.watchContract(address, CONTRACT_KIND, startingBlock); + async watchContract (address: string, checkpoint: boolean, startingBlock: number): Promise { + return this._baseIndexer.watchContract(address, CONTRACT_KIND, checkpoint, startingBlock); } async saveEventEntity (dbEvent: Event): Promise { diff --git a/packages/erc20-watcher/src/resolvers.ts b/packages/erc20-watcher/src/resolvers.ts index f5c9548e..75e02eb0 100644 --- a/packages/erc20-watcher/src/resolvers.ts +++ b/packages/erc20-watcher/src/resolvers.ts @@ -34,9 +34,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch }, Mutation: { - watchToken: async (_: any, { token, startingBlock = 1 }: { token: string, startingBlock: number }): Promise => { - log('watchToken', token, startingBlock); - await indexer.watchContract(token, startingBlock); + watchToken: async (_: any, { token, checkpoint = false, startingBlock = 1 }: { token: string, checkpoint: boolean, startingBlock: number }): Promise => { + log('watchToken', token, checkpoint, startingBlock); + await indexer.watchContract(token, checkpoint, startingBlock); return true; } diff --git a/packages/erc20-watcher/src/schema.ts b/packages/erc20-watcher/src/schema.ts index c3f02326..6ff3a4a6 100644 --- a/packages/erc20-watcher/src/schema.ts +++ b/packages/erc20-watcher/src/schema.ts @@ -158,6 +158,7 @@ type Mutation { # Actively watch and index data for the token. watchToken( token: String! + checkpoint: Boolean startingBlock: Int ): Boolean! } diff --git a/packages/uni-watcher/src/cli/watch-contract.ts b/packages/uni-watcher/src/cli/watch-contract.ts index c289080a..16e37a83 100644 --- a/packages/uni-watcher/src/cli/watch-contract.ts +++ b/packages/uni-watcher/src/cli/watch-contract.ts @@ -35,6 +35,11 @@ import { Indexer } from '../indexer'; demandOption: true, describe: 'Kind of contract (factory|pool|nfpm)' }, + checkpoint: { + type: 'boolean', + default: false, + describe: 'Turn checkpointing on' + }, startingBlock: { type: 'number', default: 1, @@ -62,7 +67,7 @@ import { Indexer } from '../indexer'; const indexer = new Indexer(db, ethClient, postgraphileClient, ethProvider, jobQueue); await indexer.init(); - await indexer.watchContract(argv.address, argv.kind, argv.startingBlock); + await indexer.watchContract(argv.address, argv.kind, argv.checkpoint, argv.startingBlock); await db.close(); await jobQueue.stop(); diff --git a/packages/uni-watcher/src/database.ts b/packages/uni-watcher/src/database.ts index 3de5c081..34b61ca5 100644 --- a/packages/uni-watcher/src/database.ts +++ b/packages/uni-watcher/src/database.ts @@ -51,10 +51,10 @@ export class Database implements DatabaseInterface { return this._baseDatabase.getContracts(repo); } - async saveContract (queryRunner: QueryRunner, address: string, kind: string, startingBlock: number): Promise { + async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { const repo = queryRunner.manager.getRepository(Contract); - return this._baseDatabase.saveContract(repo, address, startingBlock, kind); + return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock); } async createTransactionRunner (): Promise { diff --git a/packages/uni-watcher/src/entity/Contract.ts b/packages/uni-watcher/src/entity/Contract.ts index 66a521e6..d9662c33 100644 --- a/packages/uni-watcher/src/entity/Contract.ts +++ b/packages/uni-watcher/src/entity/Contract.ts @@ -20,6 +20,9 @@ export class Contract { @Column('varchar', { length: 8 }) kind!: string; + @Column('boolean') + checkpoint!: boolean; + @Column('integer') startingBlock!: number; } diff --git a/packages/uni-watcher/src/indexer.ts b/packages/uni-watcher/src/indexer.ts index 35454324..03402e4d 100644 --- a/packages/uni-watcher/src/indexer.ts +++ b/packages/uni-watcher/src/indexer.ts @@ -102,7 +102,7 @@ export class Indexer implements IndexerInterface { switch (re.event.__typename) { case 'PoolCreatedEvent': { const poolContract = ethers.utils.getAddress(re.event.pool); - await this.watchContract(poolContract, KIND_POOL, dbEvent.block.blockNumber); + await this.watchContract(poolContract, KIND_POOL, false, dbEvent.block.blockNumber); } } } @@ -353,8 +353,8 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.isWatchedContract(address); } - async watchContract (address: string, kind: string, startingBlock: number): Promise { - return this._baseIndexer.watchContract(address, kind, startingBlock); + async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { + return this._baseIndexer.watchContract(address, kind, checkpoint, startingBlock); } cacheContract (contract: Contract): void { diff --git a/packages/uni-watcher/test/init.ts b/packages/uni-watcher/test/init.ts index b3f9a352..f934247d 100644 --- a/packages/uni-watcher/test/init.ts +++ b/packages/uni-watcher/test/init.ts @@ -30,7 +30,7 @@ const deployFactoryContract = async (indexer: Indexer, signer: Signer): Promise< assert(factory.address, 'Factory contract not deployed.'); // Watch factory contract. - await indexer.watchContract(factory.address, 'factory', 100); + await indexer.watchContract(factory.address, 'factory', false, 100); return factory; }; @@ -45,7 +45,7 @@ const deployNFPMContract = async (indexer: Indexer, signer: Signer, factory: Con assert(nfpm.address, 'NFPM contract not deployed.'); // Watch NFPM contract. - await indexer.watchContract(nfpm.address, 'nfpm', 100); + await indexer.watchContract(nfpm.address, 'nfpm', false, 100); }; const main = async () => { diff --git a/packages/util/index.ts b/packages/util/index.ts index 1f394723..734f7a3b 100644 --- a/packages/util/index.ts +++ b/packages/util/index.ts @@ -13,3 +13,4 @@ export * from './src/types'; export * from './src/indexer'; export * from './src/job-runner'; export * from './src/graph-decimal'; +export * from './src/ipldHelper'; diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 7dea03d5..d47cd736 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -538,13 +538,13 @@ export class Database { .getMany(); } - async saveContract (repo: Repository, address: string, startingBlock: number, kind?: string): Promise { + async saveContract (repo: Repository, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { const contract = await repo .createQueryBuilder() .where('address = :address', { address }) .getOne(); - const entity = repo.create({ address, kind, startingBlock }); + const entity = repo.create({ address, kind, checkpoint, startingBlock }); // If contract already present, overwrite fields. if (contract) { diff --git a/packages/util/src/indexer.ts b/packages/util/src/indexer.ts index 794d319f..0792ddf4 100644 --- a/packages/util/src/indexer.ts +++ b/packages/util/src/indexer.ts @@ -308,7 +308,7 @@ export class Indexer { return this._watchedContracts[address]; } - async watchContract (address: string, kind: string, startingBlock: number): Promise { + async watchContract (address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise { assert(this._db.saveContract); const dbTx = await this._db.createTransactionRunner(); @@ -316,7 +316,7 @@ export class Indexer { const contractAddress = ethers.utils.getAddress(address); try { - const contract = await this._db.saveContract(dbTx, contractAddress, kind, startingBlock); + const contract = await this._db.saveContract(dbTx, contractAddress, kind, checkpoint, startingBlock); this.cacheContract(contract); await dbTx.commitTransaction(); diff --git a/packages/util/src/ipldHelper.ts b/packages/util/src/ipldHelper.ts new file mode 100644 index 00000000..b21706a6 --- /dev/null +++ b/packages/util/src/ipldHelper.ts @@ -0,0 +1,16 @@ +import _ from 'lodash'; + +export const updateStateForElementaryType = (initialObject: any, stateVariable: string, value: string): any => { + const object = _.cloneDeep(initialObject); + const path = ['state', stateVariable]; + + return _.set(object, path, value); +}; + +export const updateStateForMappingType = (initialObject: any, stateVariable: string, keys: string[], value: string): any => { + const object = _.cloneDeep(initialObject); + keys.unshift('state', stateVariable); + + // Use _.setWith() with Object as customizer as _.set() treats numeric value in path as an index to an array. + return _.setWith(object, keys, value, Object); +}; diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 5aae2475..5003bbb9 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -48,6 +48,7 @@ export interface ContractInterface { address: string; startingBlock: number; kind: string; + checkpoint: boolean; } export interface IndexerInterface { @@ -101,5 +102,5 @@ export interface DatabaseInterface { saveEventEntity (queryRunner: QueryRunner, entity: EventInterface): Promise; removeEntities (queryRunner: QueryRunner, entity: new () => Entity, findConditions?: FindManyOptions | FindConditions): Promise; getContracts?: () => Promise - saveContract?: (queryRunner: QueryRunner, contractAddress: string, kind: string, startingBlock: number) => Promise + saveContract?: (queryRunner: QueryRunner, contractAddress: string, kind: string, checkpoint: boolean, startingBlock: number) => Promise }