Implement data source templates (#106)

* Implement data source templates

* Remove optional param from wasm instantiate

* Add all types for solidity to typescript mapping

* Set contract address in context for data source templates

* Implement block handlers for data source templates

packages/graph-node/src/call-handler.test.ts

@@ -71,7 +71,10 @@ describe('call handler in mapping code', () => {
       db,
       indexer,
       provider,
-      { block: dummyEventData.block },
+      {
+        block: dummyEventData.block,
+        contractAddress: dummyGraphData.dataSource.address
+      },
       filePath,
       dummyGraphData
     );

packages/graph-node/src/crypto.test.ts

@@ -9,7 +9,7 @@ import { utils } from 'ethers';
 import { BaseProvider } from '@ethersproject/providers';
 
 import { instantiate } from './loader';
-import { getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
+import { getDummyGraphData, getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
 import { Database } from './database';
 import { Indexer } from '../test/utils/indexer';
 
@@ -26,14 +26,18 @@ describe('crypto host api', () => {
   });
 
   it('should load the subgraph example wasm', async () => {
+    const dummyGraphData = getDummyGraphData();
     const filePath = path.resolve(__dirname, '../test/subgraph/example1/build/Example1/Example1.wasm');
+
     const instance = await instantiate(
       db,
       indexer,
       provider,
       {},
-      filePath
+      filePath,
+      dummyGraphData
     );
+
     exports = instance.exports;
     const { _start } = exports;
 

packages/graph-node/src/eden.test.ts

@@ -89,7 +89,10 @@ describe('eden wasm loader tests', async () => {
         db,
         indexer,
         provider,
-        { block: dummyEventData.block },
+        {
+          block: dummyEventData.block,
+          contractAddress
+        },
         filePath,
         data
       ));
@@ -205,7 +208,10 @@ describe('eden wasm loader tests', async () => {
       ({ exports } = await instantiate(db,
         indexer,
         provider,
-        { block: dummyEventData.block },
+        {
+          block: dummyEventData.block,
+          contractAddress
+        },
         filePath,
         data
       ));
@@ -319,7 +325,10 @@ describe('eden wasm loader tests', async () => {
         db,
         indexer,
         provider,
-        { block: dummyEventData.block },
+        {
+          block: dummyEventData.block,
+          contractAddress
+        },
         filePath,
         data
       ));

packages/graph-node/src/eth-abi.test.ts

@@ -8,7 +8,7 @@ import { expect } from 'chai';
 import { BaseProvider } from '@ethersproject/providers';
 
 import { instantiate } from './loader';
-import { getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
+import { getDummyGraphData, getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
 import { Database } from './database';
 import { Indexer } from '../test/utils/indexer';
 
@@ -26,14 +26,18 @@ describe('ethereum ABI encode decode', () => {
   });
 
   it('should load the subgraph example wasm', async () => {
+    const dummyGraphData = getDummyGraphData();
     const filePath = path.resolve(__dirname, '../test/subgraph/example1/build/Example1/Example1.wasm');
+
     const instance = await instantiate(
       db,
       indexer,
       provider,
       {},
-      filePath
+      filePath,
+      dummyGraphData
     );
+
     exports = instance.exports;
     const { _start } = exports;
 

packages/graph-node/src/eth-call.test.ts

@@ -50,7 +50,10 @@ describe('eth-call wasm tests', () => {
       db,
       indexer,
       provider,
-      { block: dummyEventData.block },
+      {
+        block: dummyEventData.block,
+        contractAddress
+      },
       filePath,
       data
     );

packages/graph-node/src/json.test.ts

@@ -7,7 +7,7 @@ import path from 'path';
 import { BaseProvider } from '@ethersproject/providers';
 
 import { instantiate } from './loader';
-import { getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
+import { getDummyGraphData, getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
 import { Database } from './database';
 import { Indexer } from '../test/utils/indexer';
 
@@ -24,14 +24,18 @@ describe('json host api', () => {
   });
 
   it('should load the subgraph example wasm', async () => {
+    const dummyGraphData = getDummyGraphData();
     const filePath = path.resolve(__dirname, '../test/subgraph/example1/build/Example1/Example1.wasm');
+
     const instance = await instantiate(
       db,
       indexer,
       provider,
       {},
-      filePath
+      filePath,
+      dummyGraphData
     );
+
     exports = instance.exports;
     const { _start } = exports;
 

packages/graph-node/src/loader.test.ts

@@ -9,7 +9,7 @@ import { utils } from 'ethers';
 import { BaseProvider } from '@ethersproject/providers';
 
 import { instantiate } from './loader';
-import { getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
+import { getDummyGraphData, getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
 import { Database } from './database';
 import { Indexer } from '../test/utils/indexer';
 
@@ -21,19 +21,23 @@ describe('wasm loader tests', () => {
   let indexer: Indexer;
   let provider: BaseProvider;
   let module: WebAssembly.Module;
+  let dummyGraphData: any;
 
   before(async () => {
     db = getTestDatabase();
     indexer = getTestIndexer();
     provider = getTestProvider();
+    dummyGraphData = getDummyGraphData();
 
     const filePath = path.resolve(__dirname, WASM_FILE_PATH);
+
     const instance = await instantiate(
       db,
       indexer,
       provider,
       {},
-      filePath
+      filePath,
+      dummyGraphData
     );
 
     exports = instance.exports;
@@ -110,7 +114,8 @@ describe('wasm loader tests', () => {
       indexer,
       provider,
       {},
-      module
+      module,
+      dummyGraphData
     );
 
     exports = instance.exports;

packages/graph-node/src/loader.ts

@@ -38,14 +38,14 @@ type idOfType = (TypeId: number) => number
 
 export interface GraphData {
   abis?: {[key: string]: ContractInterface};
-  dataSource?: {
-    address: string,
+  dataSource: {
     network: string;
   };
 }
 
 export interface Context {
   block?: Block
+  contractAddress?: string
 }
 
 const log = debug('vulcanize:graph-node');
@@ -56,7 +56,7 @@ export const instantiate = async (
   provider: BaseProvider,
   context: Context,
   filePathOrModule: string | WebAssembly.Module,
-  data: GraphData = {}
+  data: GraphData
 ): Promise<loader.ResultObject & { exports: any }> => {
   const { abis = {}, dataSource } = data;
 
@@ -108,8 +108,8 @@ export const instantiate = async (
 
         // Create an auto-diff.
         assert(indexer.createDiffStaged);
-        assert(dataSource?.address);
-        await indexer.createDiffStaged(dataSource.address, context.block.blockHash, diffData);
+        assert(context.contractAddress);
+        await indexer.createDiffStaged(context.contractAddress, context.block.blockHash, diffData);
       },
 
       'log.log': (level: number, msg: number) => {
@@ -203,8 +203,9 @@ export const instantiate = async (
           const res = await __newArray(arrayEthereumValueId, resultPtrArray);
 
           return res;
-        } catch (err) {
-          console.log('eth_call error', err);
+        } catch (err: any) {
+          log('eth_call error', err.message);
+
           return null;
         }
       },
@@ -608,8 +609,8 @@ export const instantiate = async (
     },
     datasource: {
       'dataSource.address': async () => {
-        assert(dataSource);
-        const addressStringPtr = await __newString(dataSource.address);
+        assert(context.contractAddress);
+        const addressStringPtr = await __newString(context.contractAddress);
         return Address.fromString(addressStringPtr);
       },
       'dataSource.context': async () => {
@@ -621,6 +622,15 @@ export const instantiate = async (
       'dataSource.network': async () => {
         assert(dataSource);
         return __newString(dataSource.network);
+      },
+      'dataSource.create': async (name: number, params: number) => {
+        const [addressStringPtr] = __getArray(params);
+        const addressString = __getString(addressStringPtr);
+        const contractKind = __getString(name);
+
+        assert(indexer.watchContract);
+        assert(context.block);
+        await indexer.watchContract(utils.getAddress(addressString), contractKind, true, Number(context.block.blockNumber));
       }
     },
     json: {

packages/graph-node/src/numbers.test.ts

@@ -10,7 +10,7 @@ import { GraphDecimal } from '@vulcanize/util';
 import { BaseProvider } from '@ethersproject/providers';
 
 import { instantiate } from './loader';
-import { getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
+import { getDummyGraphData, getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
 import { Database } from './database';
 import { Indexer } from '../test/utils/indexer';
 import {
@@ -37,13 +37,16 @@ describe('numbers wasm tests', () => {
     indexer = getTestIndexer();
     provider = getTestProvider();
 
+    const dummyGraphData = getDummyGraphData();
     const filePath = path.resolve(__dirname, EXAMPLE_WASM_FILE_PATH);
+
     const instance = await instantiate(
       db,
       indexer,
       provider,
       {},
-      filePath
+      filePath,
+      dummyGraphData
     );
     exports = instance.exports;
     const { _start } = exports;

packages/graph-node/src/type-conversion.test.ts

@@ -9,7 +9,7 @@ import { utils, BigNumber } from 'ethers';
 import { BaseProvider } from '@ethersproject/providers';
 
 import { instantiate } from './loader';
-import { getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
+import { getDummyGraphData, getTestDatabase, getTestIndexer, getTestProvider } from '../test/utils';
 import { Database } from './database';
 import { Indexer } from '../test/utils/indexer';
 
@@ -26,13 +26,16 @@ describe('typeConversion wasm tests', () => {
     indexer = getTestIndexer();
     provider = getTestProvider();
 
+    const dummyGraphData = getDummyGraphData();
     const filePath = path.resolve(__dirname, EXAMPLE_WASM_FILE_PATH);
+
     const instance = await instantiate(
       db,
       indexer,
       provider,
       {},
-      filePath
+      filePath,
+      dummyGraphData
     );
     exports = instance.exports;
     const { _start } = exports;

packages/graph-node/src/watcher.ts

@@ -47,12 +47,12 @@ export class GraphWatcher {
   }
 
   async init () {
-    const { dataSources } = await getSubgraphConfig(this._subgraphPath);
-    this._dataSources = dataSources;
+    const { dataSources, templates = [] } = await getSubgraphConfig(this._subgraphPath);
+    this._dataSources = dataSources.concat(templates);
 
-    // Create wasm instance and contract interface for each dataSource in subgraph yaml.
+    // Create wasm instance and contract interface for each dataSource and template in subgraph yaml.
     const dataPromises = this._dataSources.map(async (dataSource: any) => {
-      const { source: { address, abi }, mapping, network } = dataSource;
+      const { source: { abi }, mapping, network } = dataSource;
       const { abis, file } = mapping;
 
       const abisMap = abis.reduce((acc: {[key: string]: ContractInterface}, abi: any) => {
@@ -68,7 +68,6 @@ export class GraphWatcher {
       const data = {
         abis: abisMap,
         dataSource: {
-          address,
           network
         }
       };
@@ -94,8 +93,8 @@ export class GraphWatcher {
       // TODO: Check api version https://github.com/graphprotocol/graph-node/blob/6098daa8955bdfac597cec87080af5449807e874/runtime/wasm/src/module/mod.rs#L533
       instance.exports._start();
 
-      const { source: { address } } = dataSource;
-      acc[address] = data[index];
+      const { name } = dataSource;
+      acc[name] = data[index];
 
       return acc;
     }, {});
@@ -110,10 +109,13 @@ export class GraphWatcher {
     for (const dataSource of this._dataSources) {
       const { source: { address, startBlock }, name } = dataSource;
 
-      const watchedContract = await this._indexer.isWatchedContract(address);
+      // Skip for templates as they are added dynamically.
+      if (address) {
+        const watchedContract = await this._indexer.isWatchedContract(address);
 
-      if (!watchedContract) {
-        await this._indexer.watchContract(address, name, true, startBlock);
+        if (!watchedContract) {
+          await this._indexer.watchContract(address, name, true, startBlock);
+        }
       }
     }
   }
@@ -128,15 +130,21 @@ export class GraphWatcher {
     const blockData = this._context.block;
     assert(blockData);
 
+    assert(this._indexer && this._indexer.isWatchedContract);
+    const watchedContract = await this._indexer.isWatchedContract(contract);
+    assert(watchedContract);
+
     // Get dataSource in subgraph yaml based on contract address.
-    const dataSource = this._dataSources.find(dataSource => dataSource.source.address === contract);
+    const dataSource = this._dataSources.find(dataSource => dataSource.name === watchedContract.kind);
 
     if (!dataSource) {
-      log(`Subgraph doesnt have configuration for contract ${contract}`);
+      log(`Subgraph doesn't have configuration for contract ${contract}`);
       return;
     }
 
-    const { instance, contractInterface } = this._dataSourceMap[contract];
+    this._context.contractAddress = contract;
+
+    const { instance, contractInterface } = this._dataSourceMap[watchedContract.kind];
     assert(instance);
     const { exports: instanceExports } = instance;
 
@@ -172,7 +180,7 @@ export class GraphWatcher {
     // Create ethereum event to be passed to the wasm event handler.
     const ethereumEvent = await createEvent(instanceExports, contract, data);
 
-    await this._handleMemoryError(instanceExports[eventHandler.handler](ethereumEvent), dataSource.source.address);
+    await this._handleMemoryError(instanceExports[eventHandler.handler](ethereumEvent), dataSource.name);
   }
 
   async handleBlock (blockHash: string) {
@@ -191,27 +199,47 @@ export class GraphWatcher {
         // https://stackoverflow.com/a/40453962
         // https://github.com/AssemblyScript/assemblyscript/pull/1268#issue-618411291
         // https://github.com/WebAssembly/memory64/blob/main/proposals/memory64/Overview.md#motivation
-        await this._reInitWasm(dataSource.source.address);
+        await this._reInitWasm(dataSource.name);
       }
 
-      // Check if block handler(s) are configured and start block has been reached.
-      if (!dataSource.mapping.blockHandlers || blockData.blockNumber < dataSource.source.startBlock) {
+      // Check if block handler(s) are configured.
+      if (!dataSource.mapping.blockHandlers) {
         continue;
       }
 
-      const { instance } = this._dataSourceMap[dataSource.source.address];
+      const { instance } = this._dataSourceMap[dataSource.name];
       assert(instance);
       const { exports: instanceExports } = instance;
 
       // Create ethereum block to be passed to a wasm block handler.
       const ethereumBlock = await createBlock(instanceExports, blockData);
 
-      // Call all the block handlers one after the another for a contract.
-      const blockHandlerPromises = dataSource.mapping.blockHandlers.map(async (blockHandler: any): Promise<void> => {
-        await instanceExports[blockHandler.handler](ethereumBlock);
-      });
+      let contractAddressList: string[] = [];
 
-      await this._handleMemoryError(Promise.all(blockHandlerPromises), dataSource.source.address);
+      if (dataSource.source.address) {
+        // Check if start block has been reached.
+        if (blockData.blockNumber >= dataSource.source.startBlock) {
+          contractAddressList.push(dataSource.source.address);
+        }
+      } else {
+        // Data source templates will have multiple watched contracts.
+        assert(this._indexer?.getContractsByKind);
+        const watchedContracts = this._indexer.getContractsByKind(dataSource.name);
+
+        contractAddressList = watchedContracts.filter(contract => blockData.blockNumber >= contract.startingBlock)
+          .map(contract => contract.address);
+      }
+
+      for (const contractAddress of contractAddressList) {
+        this._context.contractAddress = contractAddress;
+
+        // Call all the block handlers one after another for a contract.
+        const blockHandlerPromises = dataSource.mapping.blockHandlers.map(async (blockHandler: any): Promise<void> => {
+          await instanceExports[blockHandler.handler](ethereumBlock);
+        });
+
+        await this._handleMemoryError(Promise.all(blockHandlerPromises), dataSource.name);
+      }
     }
   }
 
@@ -228,20 +256,20 @@ export class GraphWatcher {
   }
 
   /**
-   * Method to reinstantiate WASM instance for specified contract address.
-   * @param contractAddress
+   * Method to reinstantiate WASM instance for specified dataSource.
+   * @param dataSourceName
    */
-  async _reInitWasm (contractAddress: string): Promise<void> {
-    const { data, instance } = this._dataSourceMap[contractAddress];
+  async _reInitWasm (dataSourceName: string): Promise<void> {
+    const { data, instance } = this._dataSourceMap[dataSourceName];
 
     assert(instance);
     const { module } = instance;
-    delete this._dataSourceMap[contractAddress].instance;
+    delete this._dataSourceMap[dataSourceName].instance;
 
     assert(this._indexer);
 
     // Reinstantiate with existing module.
-    this._dataSourceMap[contractAddress].instance = await instantiate(
+    this._dataSourceMap[dataSourceName].instance = await instantiate(
       this._database,
       this._indexer,
       this._ethProvider,
@@ -252,17 +280,17 @@ export class GraphWatcher {
 
     // Important to call _start for built subgraphs on instantiation!
     // TODO: Check api version https://github.com/graphprotocol/graph-node/blob/6098daa8955bdfac597cec87080af5449807e874/runtime/wasm/src/module/mod.rs#L533
-    this._dataSourceMap[contractAddress].instance!.exports._start();
+    this._dataSourceMap[dataSourceName].instance!.exports._start();
   }
 
-  async _handleMemoryError (handlerPromise: Promise<any>, contractAddress: string): Promise<void> {
+  async _handleMemoryError (handlerPromise: Promise<any>, dataSourceName: string): Promise<void> {
     try {
       await handlerPromise;
     } catch (error) {
       if (error instanceof WebAssembly.RuntimeError && error instanceof Error) {
         if (error.message === 'unreachable') {
           // Reintantiate WASM for out of memory error.
-          this._reInitWasm(contractAddress);
+          this._reInitWasm(dataSourceName);
         }
       }