Support DB caching for multiple and array return types in eth_call queries (#372)

* Generate indexer and entities with DB caching for multiple return types

* Generate database file code for multiple return types

* Remove returnType from client codegen as not required

* Changes for fixing lint warnings

* Handle mapping type values in storage mode

* Generate entities for array return type queries

* Refactor visitor for DB caching of array return types

* Fix saving bigint array type in DB

packages/codegen/src/client.ts

@@ -12,7 +12,6 @@ import { gqlGenerate } from 'gql-generator';
 
 import { getGqlForSol, getTsForGql } from './utils/type-mappings';
 import { Param } from './utils/types';
-import { getBaseType } from './utils/helpers';
 
 const TEMPLATE_FILE = './templates/client-template.handlebars';
 
@@ -29,22 +28,17 @@ export class Client {
    * Stores the query to be passed to the template.
    * @param name Name of the query.
    * @param params Parameters to the query.
-   * @param returnType Return type for the query.
    */
-  addQuery (name: string, params: Array<Param>, typeName: any): void {
+  addQuery (name: string, params: Array<Param>): void {
     // Check if the query is already added.
     if (this._queries.some(query => query.name === name)) {
       return;
     }
 
-    const returnType = getBaseType(typeName);
-    assert(returnType);
-
     const queryObject = {
       name,
       getQueryName: '',
-      params: _.cloneDeep(params),
+      params: _.cloneDeep(params)
-      returnType
     };
 
     queryObject.getQueryName = (name.charAt(0) === '_')
@@ -60,12 +54,6 @@ export class Client {
       return param;
     });
 
-    const gqlReturnType = getGqlForSol(returnType);
-    assert(gqlReturnType);
-    const tsReturnType = getTsForGql(gqlReturnType);
-    assert(tsReturnType);
-    queryObject.returnType = tsReturnType;
-
     this._queries.push(queryObject);
   }
 

packages/codegen/src/database.ts

@@ -9,9 +9,10 @@ import Handlebars from 'handlebars';
 import { Writable } from 'stream';
 import _ from 'lodash';
 
+import { VariableDeclaration } from '@solidity-parser/parser/dist/src/ast-types';
+
 import { getGqlForSol, getTsForGql } from './utils/type-mappings';
 import { Param } from './utils/types';
-import { getBaseType } from './utils/helpers';
 
 const TEMPLATE_FILE = './templates/database-template.handlebars';
 
@@ -32,22 +33,19 @@ export class Database {
    * @param params Parameters to the query.
    * @param returnType Return type for the query.
    */
-  addQuery (name: string, params: Array<Param>, typeName: any): void {
+  addQuery (name: string, params: Array<Param>, returnParameters: VariableDeclaration[]): void {
     // Check if the query is already added.
     if (this._queries.some(query => query.name === name)) {
       return;
     }
 
-    const returnType = getBaseType(typeName);
-    assert(returnType);
-
     const queryObject = {
       name,
       entityName: '',
       getQueryName: '',
       saveQueryName: '',
       params: _.cloneDeep(params),
-      returnType
+      returnParameters
     };
 
     // eth_call mode: Capitalize first letter of entity name (balanceOf -> BalanceOf, getBalanceOf, saveBalanceOf).
@@ -73,12 +71,6 @@ export class Database {
       return param;
     });
 
-    const gqlReturnType = getGqlForSol(returnType);
-    assert(gqlReturnType);
-    const tsReturnType = getTsForGql(gqlReturnType);
-    assert(tsReturnType);
-
-    queryObject.returnType = tsReturnType;
     this._queries.push(queryObject);
   }
 

packages/codegen/src/entity.ts

@@ -9,10 +9,12 @@ import yaml from 'js-yaml';
 import Handlebars from 'handlebars';
 import { Writable } from 'stream';
 
+import { VariableDeclaration } from '@solidity-parser/parser/dist/src/ast-types';
+
 import { getPgForTs, getTsForGql, getGqlForSol } from './utils/type-mappings';
 import { Param } from './utils/types';
 import { getFieldType } from './utils/subgraph';
-import { getBaseType } from './utils/helpers';
+import { getBaseType, isArrayType } from './utils/helpers';
 
 const TEMPLATE_FILE = './templates/entity-template.handlebars';
 const TABLES_DIR = './data/entities';
@@ -32,7 +34,7 @@ export class Entity {
    * @param params Parameters to the query.
    * @param returnType Return type for the query.
    */
-  addQuery (name: string, params: Array<Param>, typeName: any): void {
+  addQuery (name: string, params: Array<Param>, returnParameters: VariableDeclaration[]): void {
     // Check if the query is already added.
     if (this._entities.some(entity => entity.className.toLowerCase() === name.toLowerCase())) {
       return;
@@ -138,23 +140,46 @@ export class Entity {
       })
     );
 
-    const baseType = getBaseType(typeName);
+    entityObject.columns = entityObject.columns.concat(
-    assert(baseType);
+      returnParameters.map((returnParameter, index) => {
+        let typeName = returnParameter.typeName;
+        assert(typeName);
 
-    const gqlReturnType = getGqlForSol(baseType);
+        // Handle Mapping type for state variable queries
-    assert(gqlReturnType);
+        while (typeName.type === 'Mapping') {
-    const tsReturnType = getTsForGql(gqlReturnType);
+          typeName = typeName.valueType;
-    assert(tsReturnType);
+        }
-    const pgReturnType = getPgForTs(tsReturnType);
-    assert(pgReturnType);
 
-    entityObject.columns.push({
+        const baseType = getBaseType(typeName);
-      name: 'value',
+        assert(baseType);
-      pgType: pgReturnType,
+        const gqlReturnType = getGqlForSol(baseType);
-      tsType: tsReturnType,
+        assert(gqlReturnType);
-      columnType: 'Column',
+        let tsReturnType = getTsForGql(gqlReturnType);
-      columnOptions: []
+        assert(tsReturnType);
-    });
+        const pgReturnType = getPgForTs(tsReturnType);
+        assert(pgReturnType);
+
+        const columnOptions = [];
+        const isArray = isArrayType(typeName);
+
+        if (isArray) {
+          tsReturnType = tsReturnType.concat('[]');
+
+          columnOptions.push({
+            option: 'array',
+            value: true
+          });
+        }
+
+        return {
+          name: returnParameters.length > 1 ? `value${index}` : 'value',
+          pgType: pgReturnType,
+          tsType: tsReturnType,
+          columnType: 'Column',
+          columnOptions
+        };
+      })
+    );
 
     entityObject.columns.push({
       name: 'proof',

packages/codegen/src/indexer.ts

@@ -57,9 +57,6 @@ export class Indexer {
       return;
     }
 
-    // Disable DB caching if more than 1 return params.
-    let disableCaching = returnParameters.length > 1;
-
     const returnTypes = returnParameters.map(returnParameter => {
       let typeName = returnParameter.typeName;
       assert(typeName);
@@ -78,7 +75,6 @@ export class Indexer {
 
       const isArray = isArrayType(typeName);
       if (isArray) {
-        disableCaching = true;
         tsReturnType = tsReturnType.concat('[]');
       }
 
@@ -94,8 +90,7 @@ export class Indexer {
       returnTypes,
       mode,
       stateVariableType,
-      contract,
+      contract
-      disableCaching
     };
 
     if (name.charAt(0) === '_') {

packages/codegen/src/templates/database-template.handlebars

@@ -105,11 +105,9 @@ export class Database implements DatabaseInterface {
   {{#if (reservedNameCheck query.entityName) }}
   // eslint-disable-next-line @typescript-eslint/ban-types
   {{/if}}
-  async {{query.saveQueryName}} ({ blockHash, blockNumber, contractAddress
+  async {{query.saveQueryName}} ({ blockHash, blockNumber, contractAddress,{{#each query.params}} {{this.name~}},{{/each}}{{#each query.returnParameters}}{{~#if (compare query.returnParameters.length 1 operator=">")}} value{{@index}},{{else}} value,{{/if}}{{/each}} proof }: DeepPartial<{{query.entityName}}>): Promise<{{query.entityName}}> {
-  {{~#each query.params}}, {{this.name~}} {{/each}}, value, proof }: DeepPartial<{{query.entityName}}>): Promise<{{query.entityName}}> {
     const repo = this._conn.getRepository({{query.entityName}});
-    const entity = repo.create({ blockHash, blockNumber, contractAddress
+    const entity = repo.create({ blockHash, blockNumber, contractAddress,{{#each query.params}} {{this.name~}},{{/each}}{{#each query.returnParameters}}{{~#if (compare query.returnParameters.length 1 operator=">")}} value{{@index}},{{else}} value,{{/if}}{{/each}} proof });
-    {{~#each query.params}}, {{this.name~}} {{/each}}, value, proof });
     return repo.save(entity);
   }
 

packages/codegen/src/templates/indexer-template.handlebars

@@ -178,14 +178,21 @@ export class Indexer implements IndexerInterface {
     {{else~}}
     ): Promise<ValueResult> {
     {{/if}}
-    {{#unless query.disableCaching}}
     const entity = await this._db.{{query.getQueryName}}({ blockHash, contractAddress
     {{~#each query.params}}, {{this.name~}} {{~/each}} });
     if (entity) {
       log('{{query.name}}: db hit.');
 
       return {
+        {{#if (compare query.returnTypes.length 1 operator=">")}}
+        value: {
+          {{#each query.returnTypes}}
+          value{{@index}}: entity.value{{@index}}{{#unless @last}},{{/unless}}
+          {{/each}}
+        },
+        {{else}}
         value: entity.value,
+        {{/if}}
         proof: JSON.parse(entity.proof)
       };
     }
@@ -193,7 +200,6 @@ export class Indexer implements IndexerInterface {
     const { block: { number } } = await this._ethClient.getBlockByHash(blockHash);
     const blockNumber = ethers.BigNumber.from(number).toNumber();
 
-    {{/unless}}
     log('{{query.name}}: db miss, fetching from upstream server');
 
     {{#if (compare query.mode @root.constants.MODE_ETH_CALL)}}
@@ -248,11 +254,8 @@ export class Indexer implements IndexerInterface {
     );
     {{/if}}
 
-    {{#unless disableCaching}}
+    await this._db.{{query.saveQueryName}}({ blockHash, blockNumber, contractAddress,{{~#each query.params}} {{this.name~}},{{/each}}{{#each query.returnTypes}}{{~#if (compare query.returnTypes.length 1 operator=">")}} value{{@index}}: value.value{{@index}},{{else}} value: result.value,{{/if}}{{/each}} proof: JSONbigNative.stringify(result.proof) });
-    await this._db.{{query.saveQueryName}}({ blockHash, blockNumber, contractAddress
-    {{~#each query.params}}, {{this.name~}} {{/each}}, value: result.value, proof: JSONbigNative.stringify(result.proof) });
 
-    {{/unless}}
     {{#if query.stateVariableType}}
     {{#if (compare query.stateVariableType 'Mapping')}}
     if (diff) {

packages/codegen/src/visitor.ts

@@ -69,42 +69,30 @@ export class Visitor {
       return { name: item.name, type: item.typeName.name };
     });
 
-    let errorMessage = '';
+    // Check for unhandled return type params
+    node.returnParameters.forEach(returnParameter => {
+      assert(returnParameter.typeName);
+      const isTypeHandled = ['ElementaryTypeName', 'ArrayTypeName'].includes(returnParameter.typeName.type);
 
-    const typeName = node.returnParameters[0].typeName;
+      if (!isTypeHandled) {
-    assert(typeName);
+        const errorMessage = `No support in codegen for type ${returnParameter.typeName.type} from method "${node.name}"`;
 
-    switch (typeName.type) {
+        if (this._continueOnError) {
-      case 'ElementaryTypeName':
+          console.log(errorMessage);
-        this._entity.addQuery(name, params, typeName);
+          return;
-        this._database.addQuery(name, params, typeName);
+        }
-        this._client.addQuery(name, params, typeName);
-        // falls through
 
-      case 'ArrayTypeName':
+        throw new Error(errorMessage);
-        this._schema.addQuery(name, params, node.returnParameters);
-        this._resolvers.addQuery(name, params);
-
-        assert(this._contract);
-        this._indexer.addQuery(this._contract.name, MODE_ETH_CALL, name, params, node.returnParameters);
-        break;
-
-      case 'UserDefinedTypeName':
-        errorMessage = `No support in codegen for user defined return type from method "${node.name}"`;
-        break;
-
-      default:
-        errorMessage = `No support in codegen for return type "${typeName.type}" from method "${node.name}"`;
-    }
-
-    if (errorMessage !== '') {
-      if (this._continueOnError) {
-        console.log(errorMessage);
-        return;
       }
+    });
 
-      throw new Error(errorMessage);
+    this._schema.addQuery(name, params, node.returnParameters);
-    }
+    this._resolvers.addQuery(name, params);
+    assert(this._contract);
+    this._indexer.addQuery(this._contract.name, MODE_ETH_CALL, name, params, node.returnParameters);
+    this._entity.addQuery(name, params, node.returnParameters);
+    this._database.addQuery(name, params, node.returnParameters);
+    this._client.addQuery(name, params);
   }
 
   /**
@@ -149,12 +137,11 @@ export class Visitor {
       case 'ElementaryTypeName': {
         this._schema.addQuery(name, params, [variable]);
         this._resolvers.addQuery(name, params);
-        this._entity.addQuery(name, params, typeName);
-        this._database.addQuery(name, params, typeName);
-        this._client.addQuery(name, params, typeName);
-
         assert(this._contract);
         this._indexer.addQuery(this._contract.name, MODE_STORAGE, name, params, [variable], stateVariableType);
+        this._entity.addQuery(name, params, [variable]);
+        this._database.addQuery(name, params, [variable]);
+        this._client.addQuery(name, params);
 
         break;
       }