Automate steps in codegen to build subgraph (#478)

* Use shelljs to run graph-cli codegen command

* Replace graph-cli and graph-ts with cerc-io forks

* Add steps to build subgraph

* Add config packageManager for different subgraph repo setup

* Copy over the subgraph build to generated watcher

* Remove TODO

* Change subgraph path in generated watcher config

packages/codegen/README.md

@@ -60,11 +60,28 @@ Steps:
   # Flatten the input contract file(s) [true | false] (default: true).
   flatten: true
 
-  # Path to the subgraph build (optional).
-  # Can set empty contracts array when using subgraphPath.
-  # Subgraph WASM files should be compiled using @cerc-io/graph-cli
-  # graph-cli and graph-ts dependencies in the target subgraph repo can be replaced with forked cerc-io packages
-  subgraphPath: ../graph-node/test/subgraph/example1/build
+  # Config for subgraph (optional)
+  # Can set empty contracts array if this config is set
+  subgraph:
+    # Path to subgraph repo directory containing package.json
+    directory: ../graph-node/test/subgraph/example1
+
+    # Package manager that is used in subgraph repo for dependencies
+    packageManager: yarn
+
+    # Path to subgraph manifest/config file
+    configFile: ../graph-node/test/subgraph/example1/subgraph.yaml
+
+    # Networks config file path used when building subgraph (optional)
+    # networkFilePath:
+
+    # Network configuration to use from the networks config file (optional)
+    # network:
+
+    # Path to the subgraph build (optional)
+    # Subgraph build WASM files should be compiled using @cerc-io/graph-cli
+    # If this is set codegen does not use the build generated from subgraph.directory and subgraph.configFile
+    # buildPath: ../graph-node/test/subgraph/example1/build
 
   # NOTE: When passed an *URL* as contract path, it is assumed that it points to an already flattened contract file.
   ```

packages/codegen/package.json

@@ -22,6 +22,7 @@
   "dependencies": {
     "@cerc-io/util": "^0.2.73",
     "@graphql-tools/load-files": "^6.5.2",
+    "@npmcli/package-json": "^5.0.0",
     "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
     "@solidity-parser/parser": "^0.13.2",
     "ethers": "^5.4.4",
@@ -33,6 +34,7 @@
     "lodash": "^4.17.21",
     "node-fetch": "^2",
     "pluralize": "^8.0.0",
+    "shelljs": "^0.8.5",
     "solc": "0.8.13",
     "ts-node": "^10.2.1",
     "typescript": "^5.0.2",
@@ -43,7 +45,9 @@
     "@types/js-yaml": "^4.0.3",
     "@types/lodash": "^4.14.168",
     "@types/node": "^16.9.0",
+    "@types/npmcli__package-json": "^4.0.3",
     "@types/pluralize": "^0.0.29",
+    "@types/shelljs": "^0.8.15",
     "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",

packages/codegen/src/generate-code.ts

@@ -17,7 +17,7 @@ import { parse, visit } from '@solidity-parser/parser';
 import { ASTNode } from '@solidity-parser/parser/dist/src/ast-types';
 import { KIND_ACTIVE, KIND_LAZY } from '@cerc-io/util';
 
-import { MODE_ETH_CALL, MODE_STORAGE, MODE_ALL, MODE_NONE, DEFAULT_PORT } from './utils/constants';
+import { MODE_ETH_CALL, MODE_STORAGE, MODE_ALL, MODE_NONE, DEFAULT_PORT, ASSET_DIR } from './utils/constants';
 import { Visitor } from './visitor';
 import { exportServer } from './server';
 import { exportConfig } from './config';
@@ -35,14 +35,12 @@ import { exportCheckpoint } from './checkpoint';
 import { exportState } from './export-state';
 import { importState } from './import-state';
 import { exportInspectCID } from './inspect-cid';
-import { getSubgraphConfig } from './utils/subgraph';
+import { buildSubgraph, getSubgraphConfig } from './utils/subgraph';
 import { exportIndexBlock } from './index-block';
 import { exportSubscriber } from './subscriber';
 import { exportReset } from './reset';
 import { filterInheritedContractNodes, writeFileToStream } from './utils/helpers';
 
-const ASSET_DIR = path.resolve(__dirname, 'assets');
-
 const main = async (): Promise<void> => {
   const argv = await yargs(hideBin(process.argv))
     .option('config-file', {
@@ -67,7 +65,7 @@ const main = async (): Promise<void> => {
     })
     .argv;
 
-  const config = getConfig(path.resolve(argv['config-file']));
+  const config = await getConfig(path.resolve(argv['config-file']));
 
   // Create an array of flattened contract strings.
   const contracts: any[] = [];
@@ -214,6 +212,11 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any, overW
 
   visitor.visitSubgraph(config.subgraphPath, config.subgraphConfig);
 
+  if (config.subgraphPath && outputDir) {
+    // Copy over the subgraph build to generated watcher
+    fs.cpSync(config.subgraphPath, path.join(outputDir, 'subgraph-build'), { recursive: true });
+  }
+
   outStream = outputDir
     ? fs.createWriteStream(path.join(outputDir, 'src/schema.gql'))
     : process.stdout;
@@ -386,7 +389,7 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any, overW
   }
 }
 
-function getConfig (configFile: string): any {
+async function getConfig (configFile: string): Promise<any> {
   assert(fs.existsSync(configFile), `Config file not found at ${configFile}`);
 
   // Read config.
@@ -411,12 +414,21 @@ function getConfig (configFile: string): any {
     return contract;
   });
 
-  let subgraphPath: any;
+  let subgraphPath: string | undefined;
   let subgraphConfig;
 
-  if (inputConfig.subgraphPath) {
-    // Resolve path.
-    subgraphPath = inputConfig.subgraphPath.replace(/^~/, os.homedir());
+  if (inputConfig.subgraph) {
+    if (inputConfig.subgraph.directory) {
+      await buildSubgraph(configFile, inputConfig.subgraph);
+      subgraphPath = path.resolve(inputConfig.subgraph.directory, 'build');
+    }
+
+    if (inputConfig.subgraph.buildPath) {
+      // Resolve path.
+      subgraphPath = inputConfig.subgraph.buildPath.replace(/^~/, os.homedir()) as string;
+    }
+
+    assert(subgraphPath, 'Config subgraph.directory or subgraph.buildPath must be specified');
     subgraphConfig = getSubgraphConfig(subgraphPath);
 
     // Add contracts missing for dataSources and templates in subgraph config.
@@ -425,6 +437,7 @@ function getConfig (configFile: string): any {
       .forEach((dataSource: any) => {
         if (!contracts.some((contract: any) => contract.kind === dataSource.name)) {
           const abi = dataSource.mapping.abis.find((abi: any) => abi.name === dataSource.source.abi);
+          assert(subgraphPath);
           const abiPath = path.resolve(subgraphPath, abi.file);
 
           contracts.push({

packages/codegen/src/templates/config-template.handlebars

@@ -14,7 +14,7 @@
   enableState = true
 
   {{#if (subgraphPath)}}
-  subgraphPath = "{{subgraphPath}}"
+  subgraphPath = "./subgraph-build"
 
   # Interval to restart wasm instance periodically
   wasmRestartBlocksInterval = 20

packages/codegen/src/utils/constants.ts

@@ -2,9 +2,13 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
+import path from 'path';
+
 export const MODE_ETH_CALL = 'eth_call';
 export const MODE_STORAGE = 'storage';
 export const MODE_ALL = 'all';
 export const MODE_NONE = 'none';
 
 export const DEFAULT_PORT = 3008;
+
+export const ASSET_DIR = path.resolve(__dirname, '../assets');

packages/codegen/src/utils/subgraph.ts

@@ -2,9 +2,16 @@ import path from 'path';
 import assert from 'assert';
 import fs from 'fs';
 import yaml from 'js-yaml';
+import shell from 'shelljs';
 
+import PackageJson from '@npmcli/package-json';
 import { loadFilesSync } from '@graphql-tools/load-files';
 
+import { ASSET_DIR } from './constants';
+
+const GRAPH_TS_VERSION = '0.27.0-watcher-ts-0.1.2';
+const GRAPH_CLI_VERSION = '0.32.0-watcher-ts-0.1.3';
+
 export function parseSubgraphSchema (subgraphPath: string, subgraphConfig: any): any {
   const subgraphSchemaPath = path.join(path.resolve(subgraphPath), subgraphConfig.schema?.file ?? './schema.graphql');
 
@@ -52,6 +59,72 @@ export function getSubgraphConfig (subgraphPath: string): any {
   return yaml.load(fs.readFileSync(subgraphConfigPath, 'utf8')) as any;
 }
 
+export async function buildSubgraph (
+  codegenConfigPath: string,
+  subgraphConfig: {
+    directory: string,
+    packageManager: string,
+    configFile: string,
+    networkFilePath?: string,
+    network?: string
+  }
+): Promise<void> {
+  const subgraphDirectory = path.resolve(codegenConfigPath, subgraphConfig.directory);
+  const codegenWorkingDir = process.cwd();
+  // Change directory to subgraph repo
+  shell.cd(subgraphDirectory);
+
+  // Replace graph-cli & graph-ts in package.json with cerc-io forks
+  const pkgJson = await PackageJson.load(subgraphDirectory);
+  const { content } = pkgJson;
+
+  if (content.dependencies) {
+    // Remove graph tools from direct dependencies
+    delete content.dependencies['@graphprotocol/graph-ts'];
+    delete content.dependencies['@graphprotocol/graph-cli'];
+  }
+
+  if (!content.devDependencies) {
+    content.devDependencies = {};
+  }
+
+  content.devDependencies['@graphprotocol/graph-ts'] = `npm:@cerc-io/graph-ts@${GRAPH_TS_VERSION}`;
+  delete content.devDependencies['@graphprotocol/graph-cli'];
+  content.devDependencies['@cerc-io/graph-cli'] = GRAPH_CLI_VERSION;
+  pkgJson.update(content);
+  await pkgJson.save();
+
+  // Create .npmrc for cerc-io packages
+  fs.copyFileSync(path.join(ASSET_DIR, '.npmrc'), path.join(subgraphDirectory, '.npmrc'));
+
+  const packageManager = subgraphConfig.packageManager;
+  // Install dependencies
+  const { code: installCode } = shell.exec(`${packageManager} install --force`);
+  assert(installCode === 0, 'Installing dependencies exited with error');
+
+  const subgraphConfigPath = path.resolve(codegenConfigPath, subgraphConfig.configFile);
+
+  // Run graph-cli codegen
+  const { code: codegenCode } = shell.exec(`${packageManager === 'npm' ? 'npx' : packageManager} graph codegen ${subgraphConfigPath}`);
+  assert(codegenCode === 0, 'Subgraph codegen command exited with error');
+
+  // Run graph-cli build
+  let buildCommand = `${packageManager === 'npm' ? 'npx' : packageManager} graph build ${subgraphConfigPath}`;
+
+  if (subgraphConfig.networkFilePath) {
+    const subgraphNetworkFilePath = path.resolve(codegenConfigPath, subgraphConfig.networkFilePath);
+    assert(subgraphConfig.network, 'Config subgraph.network should be set if using networkFilePath');
+    const subgraphNetwork = subgraphConfig.network;
+    buildCommand = `${buildCommand} --network-file ${subgraphNetworkFilePath} --network ${subgraphNetwork}`;
+  }
+
+  const { code: buildCode } = shell.exec(buildCommand);
+  assert(buildCode === 0, 'Subgraph build command exited with error');
+
+  // Change directory back to codegen
+  shell.cd(codegenWorkingDir);
+}
+
 function parseType (typeNode: any): any {
   // Check if 'NamedType' is reached.
   if (typeNode.kind !== 'NamedType') {