build: add more last tests for ci-cd

.github/workflows/ci-cd-trigger.yml

@@ -101,97 +101,7 @@ jobs:
 
       # See affected apps
       - name: See affected apps
-        run: |
-          affected="$(yarn nx print-affected --base=${{ env.NX_BASE }} --head=${{ env.NX_HEAD }} --select=projects)"
-          branch_slug="$(echo '${{ github.head_ref || github.ref_name }}' | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )"
-
-          echo ">>>> debug"
-          echo "NX_BASE: ${{ env.NX_BASE }}"
-          echo "NX_HEAD: ${{ env.NX_HEAD }}"
-          echo "Affected: ${affected}"
-          echo "Branch slug: ${branch_slug}"
-          echo ">>>> eof debug"
-
-          projects_array=()
-
-          preview_governance="not deployed"
-          preview_trading="not deployed"
-          preview_explorer="not deployed"
-          preview_tools="not deployed"
-
-          # parse if affected is any of three main applications, if none - use all of them
-          if echo "$affected" | grep -q governance; then
-            echo "Governance is affected"
-            projects_array+=("governance")
-            preview_governance=$(printf "https://%s.%s.vega.rocks" "governance" "$branch_slug")
-          fi
-          if echo "$affected" | grep -q trading; then
-            echo "Trading is affected"
-            projects_array+=("trading")
-            preview_trading=$(printf "https://%s.%s.vega.rocks" "trading" "$branch_slug")
-          fi
-          if echo "$affected" | grep -q explorer; then
-            echo "Explorer is affected"
-            projects_array+=("explorer")
-            preview_explorer=$(printf "https://%s.%s.vega.rocks" "explorer" "$branch_slug")
-          fi
-          if [[ ${#projects_array[@]} -eq 0 ]]; then
-            projects_array=("governance" "trading" "explorer")
-            preview_governance=$(printf "https://%s.%s.vega.rocks" "governance" "$branch_slug")
-            preview_trading=$(printf "https://%s.%s.vega.rocks" "trading" "$branch_slug")
-            preview_explorer=$(printf "https://%s.%s.vega.rocks" "explorer" "$branch_slug")
-          fi
-
-          # applications parsed before this loop are applicable for running e2e-tests
-          projects_e2e_array=()
-          for project in "${projects_array[@]}"; do
-            projects_e2e_array+=("${project}-e2e")
-          done
-          # all applications below this loop are not applicable for running e2e-test
-
-          # check if pull request event to deploy tools
-          if [[ "${{ github.event_name }}" = "pull_request" ]]; then
-            if echo "$affected" | grep -q multisig-signer; then
-              echo "Tools are affected"
-              echo "Deploying tools on preview"
-              preview_tools=$(printf "https://%s.%s.vega.rocks" "tools" "$branch_slug")
-
-              projects_array+=("multisig-signer")
-            fi
-          # those apps deploy only from develop to mainnet
-          elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then
-            if echo "$affected" | grep -q multisig-signer; then
-              echo "Tools are affected"
-              echo "Deploying tools on s3"
-
-              projects_array+=("multisig-signer")
-            fi
-            if echo "$affected" | grep -q static; then
-              echo "static is affected"
-              echo "Deploying static on s3"
-
-              projects_array+=("static")
-            fi
-            if echo "$affected" | grep -q ui-toolkit; then
-              echo "ui-toolkit is affected"
-              echo "Deploying ui-toolkit on s3"
-
-              projects_array+=("ui-toolkit")
-            fi
-          fi
-
-          echo "Projects: ${projects_array[@]}"
-          echo "Projects E2E: ${projects_e2e_array[@]}"
-          projects_json=$(jq -M --compact-output --null-input '$ARGS.positional' --args -- "${projects_array[@]}")
-          projects_e2e_json=$(jq -M --compact-output --null-input '$ARGS.positional' --args -- "${projects_e2e_array[@]}")
-
-          echo PROJECTS_E2E=$projects_e2e_json >> $GITHUB_ENV
-          echo PROJECTS=$projects_json >> $GITHUB_ENV
-
-          echo PREVIEW_GOVERNANCE=$preview_governance >> $GITHUB_ENV
-          echo PREVIEW_TRADING=$preview_trading >> $GITHUB_ENV
-          echo PREVIEW_EXPLORER=$preview_explorer >> $GITHUB_ENV
-          echo PREVIEW_TOOLS=$preview_tools >> $GITHUB_ENV
+        run: node tools/scripts/ci-cd-trigger-affected.js
 
     outputs:
       projects: ${{ env.PROJECTS }}

.github/workflows/publish-dist.yml

@@ -64,52 +64,7 @@ jobs:
       # https://docs.github.com/en/actions/learn-github-actions/contexts
       - name: Define dist variables
         if: ${{ github.event_name == 'push' }}
-        run: |
-          envName=''
-          domain="vega.rocks"
-          bucketName=''
-
-          if [[ "${{ github.ref }}" =~ .*release/.* ]]; then
-            envName="$(echo ${{ github.ref }} | rev | cut -d '/' -f 1 | rev)"
-          elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then
-            envName="stagnet1"
-            if [[ "${{ matrix.app }}" = "multisig-signer" ]]; then
-              envName="mainnet"
-              bucketName="tools.vega.xyz"
-            fi
-            if [[ "${{ matrix.app }}" = "static" ]]; then
-              envName="mainnet"
-              bucketName="static.vega.xyz"
-            fi
-            if [[ "${{ matrix.app }}" = "ui-toolkit" ]]; then
-              envName="mainnet"
-              bucketName="ui.vega.rocks"
-            fi
-          elif [[ "${{ github.ref }}" =~ .*main$ ]]; then
-            envName="mainnet"
-          fi
-
-          if [[ "${envName}" = "mainnet" ]]; then
-            domain="vega.xyz"
-            if [[ -z "${bucketName}" ]]; then
-              bucketName="${{ matrix.app }}.${domain}"
-            fi
-          elif [[ "${envName}" = "testnet" ]]; then
-            domain="fairground.wtf"
-            if [[ -z "${bucketName}" ]]; then
-              bucketName="${{ matrix.app }}.${domain}"
-            fi
-          fi
-
-          if [[ -z "${bucketName}" ]]; then
-            bucketName="${{ matrix.app }}.${envName}.${domain}"
-          fi
-
-          echo "bucket name: ${bucketName}"
-          echo "env name: ${envName}"
-
-          echo BUCKET_NAME=${bucketName} >> $GITHUB_ENV
-          echo ENV_NAME=${envName} >> $GITHUB_ENV
+        run: node tools/scripts/publish-dist-set-vars.js --app=${{ matrix.app }}
 
       - name: Build local dist
         run: |

jest.config.ts

@@ -1,5 +1,5 @@
 const { getJestProjects } = require('@nrwl/jest');
 
 export default {
-  projects: getJestProjects(),
+  projects: [...getJestProjects(), '<rootDir>/tools/scripts'],
 };

tools/scripts/.gitignore

@@ -0,0 +1 @@
+node_modules

tools/scripts/ci-cd-trigger-affected.js

@@ -0,0 +1,264 @@
+/**
+ * Using `nx affected` as a starting point, decide which test suites must
+ * run and where required generate preview links for the affected apps.
+ *
+ * @author Miko
+ * @author Edd
+ */
+const execSync = require('child_process').execSync;
+const { AppsThatDeployToMainnetFromDevelop } = require('./config');
+const {
+  output,
+  getBranch,
+  fail,
+  IS_TEST,
+  validateAppName,
+  IS_PULL_REQUEST,
+  specialCasePrefix,
+} = require('./lib/ci-functions');
+
+const MAIN_TOOLS = ['governance', 'trading', 'explorer'];
+
+// Add a valid nx project here if you want it to deploy on all PRs,
+// but it has no corresponding -e2e project
+// TODO: Do this automatically by checking if tool-e2e exists in project
+const TESTLESS_TOOLS_THAT_DEPLOY_ON_ALL_PRS = ['multisig-signer'];
+
+/**
+ * Preview links are deployed to S3 buckset, and domain names map on to
+ * those S3 buckets. To see which bucket is used to store an application,
+ * check out `publish-dist-set-vars.js`. This function generates the predictable
+ * URL for a given app and branch.
+ *
+ * @param {string} app
+ * @param {string} branch
+ * @returns
+ */
+function getDeployPreviewLink(app, branch) {
+  if (!validateAppName(app)) {
+    return fail('Cannot generate preview link for unknown app: ' + app);
+  }
+  if (!branch || branch.trim().length === 0) {
+    return fail(
+      `Invalid branch name specified for '${app}' preview link: ${branch}`
+    );
+  }
+
+  return `https://${app}.${branch}.vega.rocks`;
+}
+
+function getAffected() {
+  if (!process.env.NX_BASE || !process.env.NX_HEAD) {
+    fail('Environment variables NX_BASE and NX_HEAD must be set');
+  }
+  const affected = [];
+  const affectedOutput = execSync(
+    `yarn nx print-affected --base=${process.env.NX_BASE} --head=${process.env.NX_HEAD} --select=projects`
+  );
+
+  if (affectedOutput.status && affectedOutput.status !== 0) {
+    fail(`Could not parse output of nx print-affected: ${affectedOutput}`);
+  }
+
+  // [0] is the yarn output, [1] is the command line, [2] is the output we want
+  const affectedAppsString = affectedOutput.toString().split('\n')[2];
+  if (
+    !affectedAppsString ||
+    affectedAppsString.trim() === '' ||
+    affectedAppsString.indexOf(',') === -1
+  ) {
+    console.debug(
+      'Nothing is affected. Nothing to do, unless there are special cases.'
+    );
+  } else {
+    affectedAppsString.split(',').forEach((app) => {
+      const a = app.trim();
+      if (validateAppName(a)) {
+        affected.push(a);
+      }
+    });
+  }
+
+  return affected;
+}
+/**
+ * Ensures that E2E test run pipelines are triggered
+ *
+ * @param {string[]} affected
+ * @returns
+ */
+function testRunsToTrigger(affected) {
+  let projects_e2e = [];
+
+  MAIN_TOOLS.forEach((tool) => {
+    if (affected.includes(tool)) {
+      projects_e2e.push(`${tool}-e2e`);
+    }
+  });
+  // By default, trigger everything
+  if (projects_e2e.length === 0) {
+    if (!IS_TEST) {
+      console.log(
+        `${specialCasePrefix} no major apps are affected, but let's run the tests anyway`
+      );
+    }
+
+    projects_e2e = MAIN_TOOLS.map((tool) => `${tool}-e2e`);
+  }
+
+  return projects_e2e;
+}
+
+function generateDeployPreviewLinks(affected, branch) {
+  let countMajorAppsAffected = 0;
+
+  let res = {};
+  if (affected.includes('governance')) {
+    res['preview_governance'] = getDeployPreviewLink('governance', branch);
+    countMajorAppsAffected++;
+  }
+  if (affected.includes('trading')) {
+    res['preview_trading'] = getDeployPreviewLink('trading', branch);
+    countMajorAppsAffected++;
+  }
+  if (affected.includes('explorer')) {
+    res['preview_explorer'] = getDeployPreviewLink('explorer', branch);
+    countMajorAppsAffected++;
+  }
+
+  // We only need the multisig-signer
+  if (IS_PULL_REQUEST && affected.includes('multisig-signer')) {
+    environmentVariablesToSet['preview_tools'] = getDeployPreviewLink(
+      'multisig-signer',
+      branch
+    );
+  }
+
+  // By default, a deploy for everything is created
+  if (countMajorAppsAffected === 0) {
+    if (!IS_TEST) {
+      console.log(
+        `${specialCasePrefix} no major apps are affected, but let's output preview links anyway`
+      );
+    }
+
+    res['preview_governance'] = getDeployPreviewLink('governance', branch);
+    res['preview_trading'] = getDeployPreviewLink('trading', branch);
+    res['preview_explorer'] = getDeployPreviewLink('explorer', branch);
+  }
+
+  return res;
+}
+
+/**
+ * Just some handy output for when you're debugging a pipeline.
+ * Disable it by setting DEBUG=false in the pipeline environment.
+ *
+ * @param {string[]} affected list of nx:affected apps
+ * @param {string} branch
+ */
+function debugOutput(affected, branch) {
+  if (process.env.DEBUG && process.env.DEBUG !== 'false') {
+    console.group('>>>> debug');
+    console.debug(`NX_BASE: ${process.env.NX_BASE}`);
+    console.debug(`NX_HEAD: ${process.env.NX_HEAD}`);
+    console.debug(`Affected: ${affected}`);
+    console.debug(`Branch slug: ${branch}`);
+    console.debug(`Is pull request: ${IS_PULL_REQUEST}`);
+    console.groupEnd();
+    console.debug('>>>> eof debug');
+  }
+}
+
+/**
+ * Using the list of projects that were affected or selected for e2e testing, we generate
+ * a list of projects that should be deployed.
+ *
+ * @see testRunsToTrigger
+ * @param {string[]} projectsToTest a list of affected/unaffected projects that testRunsToTrigger selected
+ * @param {string} branch the string name of the branch. `develop` has some special cases
+ * @param {string[]} affected
+ * @returns string[] a list of projects that should be deployed
+ */
+function mapTestsToDeploy(projectsToTest, branch, affected, isPullRequest) {
+  // Catches fully qualified refs (e.g. `refs/head/develop`) and short name (`develop`)
+  const BRANCH_IS_DEVELOP = branch.match(/.*develop$/);
+
+  let projectsToDeploy = new Set(
+    projectsToTest.map((p) => p.replace(/-e2e/g, ''))
+  );
+
+  //
+  // Special Cases
+  if (isPullRequest) {
+    TESTLESS_TOOLS_THAT_DEPLOY_ON_ALL_PRS.forEach((tool) => {
+      if (affected.includes(tool)) {
+        if (!IS_TEST) {
+          console.log(
+            `${specialCasePrefix} ${tool} is affected, but deploys from develop. Adding to deploy queue...`
+          );
+        }
+
+        projectsToDeploy.add(tool);
+      }
+    });
+  }
+
+  if (BRANCH_IS_DEVELOP) {
+    // On merge/push to develop, updated these 3 things automatically
+    // Only validated app names get here
+    AppsThatDeployToMainnetFromDevelop.forEach((tool) => {
+      if (affected.includes(tool)) {
+        if (!IS_TEST) {
+          console.log(
+            `${specialCasePrefix} ${tool} is not affected, but deploys from develop. Adding to deploy queue...`
+          );
+        }
+        projectsToDeploy.add(tool);
+      }
+    });
+  }
+
+  return Array.from(projectsToDeploy);
+}
+
+if (!IS_TEST) {
+  const branch = getBranch();
+  const affected = getAffected();
+  debugOutput(affected, branch);
+
+  // The list of projects here will be passed to the step `cypress` to have their smoke/regression tests run.
+  const projectsToTest = testRunsToTrigger(affected);
+
+  // The list of apps that will have their deploy previews updated
+  const projectsToDeploy = mapTestsToDeploy(
+    projectsToTest,
+    branch,
+    affected,
+    IS_PULL_REQUEST
+  );
+
+  // Environment variables are used to pass the result of this script on to subsequent CI steps
+  const environmentVariablesToSet = generateDeployPreviewLinks(
+    affected,
+    branch
+  );
+
+  try {
+    output('PROJECTS_E2E', JSON.stringify(projectsToTest));
+    output('PROJECTS', JSON.stringify(projectsToDeploy));
+
+    Object.entries(environmentVariablesToSet).forEach(([key, value]) => {
+      output(key, value);
+    });
+  } catch (e) {
+    fail('Error stringifying/exporting output', e);
+  }
+}
+
+module.exports = {
+  getDeployPreviewLink,
+  testRunsToTrigger,
+  generateDeployPreviewLinks,
+  mapTestsToDeploy,
+};

tools/scripts/ci-cd-trigger-affected.spec.js

@@ -0,0 +1,200 @@
+const {
+  testRunsToTrigger,
+  getDeployPreviewLink,
+  generateDeployPreviewLinks,
+  mapTestsToDeploy,
+} = require('./ci-cd-trigger-affected');
+
+describe('getDeployPreviewLink', () => {
+  test('should return the deploy preview link if the app name is valid and branch is provided', () => {
+    const app = 'governance';
+    const branch = 'feature-branch';
+
+    expect(getDeployPreviewLink(app, branch)).toBe(
+      'https://governance.feature-branch.vega.rocks'
+    );
+  });
+
+  test('should fail if the app name is not valid', () => {
+    const app = 'invalid-app';
+    const branch = 'feature-branch';
+
+    expect(getDeployPreviewLink(app, branch)).toStrictEqual(false);
+  });
+
+  test('should fail if the branch is not provided or empty', () => {
+    const app = 'governance';
+    const branch = '';
+
+    expect(getDeployPreviewLink(app, branch)).toStrictEqual(false);
+  });
+});
+
+describe('testRunsToTrigger', () => {
+  test('should return an array with "governance-e2e" if "governance" is affected', () => {
+    const affected = ['governance'];
+    expect(testRunsToTrigger(affected)).toEqual(['governance-e2e']);
+  });
+
+  test('should return an array with "trading-e2e" if "trading" is affected', () => {
+    const affected = ['trading'];
+    expect(testRunsToTrigger(affected)).toEqual(['trading-e2e']);
+  });
+
+  test('should return an array with "explorer-e2e" if "explorer" is affected', () => {
+    const affected = ['explorer'];
+    expect(testRunsToTrigger(affected)).toEqual(['explorer-e2e']);
+  });
+
+  test('should return an array with all three projects if all are affected', () => {
+    const affected = ['governance', 'trading', 'explorer'];
+    expect(testRunsToTrigger(affected)).toEqual([
+      'governance-e2e',
+      'trading-e2e',
+      'explorer-e2e',
+    ]);
+  });
+
+  test('should return an array with all three projects if no specific projects are affected', () => {
+    const affected = ['other'];
+    expect(testRunsToTrigger(affected)).toEqual([
+      'governance-e2e',
+      'trading-e2e',
+      'explorer-e2e',
+    ]);
+  });
+});
+
+describe('generateDeployPreviewLinks', () => {
+  test('should return an object with a link for "governance" app if it is affected', () => {
+    const affected = ['governance'];
+    const branch = 'feature-branch';
+    const output = generateDeployPreviewLinks(affected, branch);
+    expect(output).toEqual({
+      preview_governance: 'https://governance.feature-branch.vega.rocks',
+    });
+  });
+
+  test('should return an object with a link for "trading" app if it is affected', () => {
+    const affected = ['trading'];
+    const branch = 'feature-branch';
+    const output = generateDeployPreviewLinks(affected, branch);
+    expect(output).toEqual({
+      preview_trading: 'https://trading.feature-branch.vega.rocks',
+    });
+  });
+
+  test('should return an object with a link for "explorer" app if it is affected', () => {
+    const affected = ['explorer'];
+    const branch = 'feature-branch';
+    const output = generateDeployPreviewLinks(affected, branch);
+    expect(output).toEqual({
+      preview_explorer: 'https://explorer.feature-branch.vega.rocks',
+    });
+  });
+
+  test('should return an object with links for all apps if all are affected', () => {
+    const affected = ['governance', 'trading', 'explorer'];
+    const branch = 'feature-branch';
+    const output = generateDeployPreviewLinks(affected, branch);
+    expect(output).toEqual({
+      preview_governance: 'https://governance.feature-branch.vega.rocks',
+      preview_trading: 'https://trading.feature-branch.vega.rocks',
+      preview_explorer: 'https://explorer.feature-branch.vega.rocks',
+    });
+  });
+
+  test('should return an object with links for all apps if no specific apps are affected', () => {
+    const affected = ['other'];
+    const branch = 'feature-branch';
+    const output = generateDeployPreviewLinks(affected, branch);
+    expect(output).toEqual({
+      preview_governance: 'https://governance.feature-branch.vega.rocks',
+      preview_trading: 'https://trading.feature-branch.vega.rocks',
+      preview_explorer: 'https://explorer.feature-branch.vega.rocks',
+    });
+  });
+
+  test('should return an object with all links if no apps are affected', () => {
+    const affected = [];
+    const branch = 'feature-branch';
+    const output = generateDeployPreviewLinks(affected, branch);
+    expect(output).toEqual({
+      preview_governance: 'https://governance.feature-branch.vega.rocks',
+      preview_trading: 'https://trading.feature-branch.vega.rocks',
+      preview_explorer: 'https://explorer.feature-branch.vega.rocks',
+    });
+  });
+});
+
+describe('mapTestsToDeploy', () => {
+  test('should add affected "TESTLESS_TOOLS_THAT_DEPLOY_ON_ALL_PRS" to deploy queue on pull request', () => {
+    const projectsToTest = ['project1-e2e', 'project2-e2e'];
+    const affected = ['project1', 'project2', 'multisig-signer'];
+    const branch = 'main';
+
+    const result = mapTestsToDeploy(projectsToTest, branch, affected, true);
+
+    expect(result).toContain('project1');
+    expect(result).toContain('project2');
+    expect(result).toContain('multisig-signer');
+  });
+
+  test('even if a special case is already affected', () => {
+    const projectsToTest = [];
+    const affected = ['multisig-signer'];
+    const branch = 'main';
+
+    const result = mapTestsToDeploy(projectsToTest, branch, affected, true);
+
+    expect(result).toContain('multisig-signer');
+    expect(result.length).toStrictEqual(1);
+  });
+
+  test('IS_PULL_REQUEST special case should not be deployed on non-prs', () => {
+    const projectsToTest = [];
+    const affected = ['multisig-signer'];
+    const branch = 'main';
+
+    const result = mapTestsToDeploy(projectsToTest, branch, affected, false);
+
+    expect(result).not.toContain('multisig-signer');
+    expect(result.length).toStrictEqual(0);
+  });
+
+  test('should add affected "AppsThatDeployToMainnetFromDevelop" to deploy queue on develop branch', () => {
+    const projectsToTest = ['libs/accounts', 'libs-apollo-client'];
+    const branch = 'develop';
+    const affected = ['multisig-signer', 'static', 'ui-toolkit'];
+
+    const result = mapTestsToDeploy(projectsToTest, branch, affected, false);
+
+    expect(result).toContain('multisig-signer');
+    expect(result).toContain('static');
+    expect(result).toContain('ui-toolkit');
+  });
+
+  test('should not add affected "AppsThatDeployToMainnetFromDevelop" to deploy queue on any other branch', () => {
+    const projectsToTest = ['trading', 'explorer'];
+    const branch = 'main';
+    const affected = ['multisig-signer', 'static', 'ui-toolkit'];
+
+    const result = mapTestsToDeploy(projectsToTest, branch, affected, false);
+
+    expect(result).not.toContain('multisig-signer');
+    expect(result).not.toContain('static');
+    expect(result).not.toContain('ui-toolkit');
+  });
+
+  test('should not add any other item to the deploy queue', () => {
+    const projectsToTest = ['trading-e2e', 'explorer-e2e'];
+    const branch = 'develop';
+    const affected = ['trading', 'explorer'];
+
+    const result = mapTestsToDeploy(projectsToTest, branch, affected);
+
+    expect(result.length).toEqual(2);
+    expect(result).toContain('trading');
+    expect(result).toContain('explorer');
+  });
+});

tools/scripts/config.js

@@ -0,0 +1,14 @@
+// Add a valid nx project here if you want it to deploy and be tested
+// for all commits to develop. Apps must be valid NX projects.
+const AppsThatDeployToMainnetFromDevelop = new Set([
+  'multisig-signer',
+  'static',
+  'ui-toolkit',
+]);
+
+const AppsThatDoNotDeployToMainnet = new Set(['trading']);
+
+module.exports = {
+  AppsThatDeployToMainnetFromDevelop,
+  AppsThatDoNotDeployToMainnet,
+};

tools/scripts/config.spec.js

@@ -0,0 +1,62 @@
+const isString = require('lodash/isString');
+const {
+  AppsThatDeployToMainnetFromDevelop,
+  AppsThatDoNotDeployToMainnet,
+} = require('./config');
+const { validateAppName } = require('./publish-dist-set-vars');
+
+describe('AppsThatDeployToMainnetFromDevelop configuration test', () => {
+  test('all apps that deploy to mainnet from develop should be valid', () => {
+    AppsThatDeployToMainnetFromDevelop.forEach((app) => {
+      expect(isString(app)).toBe(
+        true,
+        `Invalid non-string in AppsThatDeployToMainnetFromDevelop`
+      );
+      expect(app.length).toBeGreaterThan(
+        0,
+        `Invalid empty string in AppsThatDeployToMainnetFromDevelop`
+      );
+      expect(app.endsWith('-e2e')).toBe(false, 'Do not deploy e2e apps');
+      expect(validateAppName(app)).toBe(
+        true,
+        `App name "${app}" is not in nx project`
+      );
+    });
+  });
+
+  test('should contain "multisig-signer"', () => {
+    expect(AppsThatDeployToMainnetFromDevelop.has('static')).toBe(true);
+  });
+
+  test('should contain "static"', () => {
+    expect(AppsThatDeployToMainnetFromDevelop.has('static')).toBe(true);
+  });
+
+  test('should contain "ui-toolkit"', () => {
+    expect(AppsThatDeployToMainnetFromDevelop.has('ui-toolkit')).toBe(true);
+  });
+});
+
+describe('AppsThatDoNotDeployToMainnet configuration test', () => {
+  test('all apps that deploy to mainnet from develop should be valid', () => {
+    AppsThatDoNotDeployToMainnet.forEach((app) => {
+      expect(isString(app)).toBe(
+        true,
+        'invalid non-string in AppsThatDoNotDeployToMainnet'
+      );
+      expect(app.length).toBeGreaterThan(
+        0,
+        'invalid empty string in AppsThatDoNotDeployToMainnet'
+      );
+      expect(app.endsWith('-e2e')).toBe(false, '-e2e suites never deploy');
+      expect(validateAppName(app)).toBe(
+        true,
+        `App name "${app}" is not in nx project`
+      );
+    });
+  });
+
+  test('should contain "trading"', () => {
+    expect(AppsThatDoNotDeployToMainnet.has('trading')).toBe(true);
+  });
+});

tools/scripts/lib/ci-functions.js

@@ -0,0 +1,124 @@
+const { appendFileSync, readFileSync } = require('fs');
+const isString = require('lodash/isString');
+const path = require('path');
+
+const IS_TEST = process.env.NODE_ENV === 'test';
+const IS_PULL_REQUEST = process.env.GITHUB_EVENT_NAME === 'pull_request';
+const errorPrefix = '❌ [fail]';
+const specialCasePrefix = '🌟 [special-case]';
+
+/**
+ * Returns a trimmed, sanitised version of a full github ref
+ *
+ * @param {string} ref
+ * @returns {string} trimmed, sanitised ref
+ */
+function getBranchNameFromGithubRef(ref) {
+  try {
+    const filteredRef = ref
+      .replace(/[^a-zA-Z0-9]+/g, '-')
+      .replace(/^-+|-+$/g, '')
+      .substring(0, 50);
+    if (filteredRef.length === 0) {
+      return fail('Could not get branch name from GITHUB_REF_NAME' + ref);
+    }
+
+    return filteredRef;
+  } catch (e) {
+    return fail('Error parsing branch name from GITHUB_REF_NAME:' + ref, e);
+  }
+}
+
+/**
+ * Derives the branch name from the environment
+ *
+ * @returns {string} The branch name
+ */
+function getBranch() {
+  if (!process.env.GITHUB_HEAD_REF && !process.env.GITHUB_REF_NAME) {
+    fail(
+      'Environment variables GITHUB_HEAD_REF or GITHUB_REF_NAME must be set'
+    );
+  }
+
+  const branch =
+    process.env.GITHUB_HEAD_REF ||
+    getBranchNameFromGithubRef(process.env.GITHUB_REF_NAME);
+  if (!branch) {
+    fail('Could not get branch name from GITHUB_HEAD_REF or GITHUB_REF_NAME');
+  }
+  return branch;
+}
+
+/**
+ * Simple wrapper function to either set a Github environment variable
+ * or just log it out for testing
+ */
+function output(key, value) {
+  const isRunningOnGithub = process.env.CI && process.env.GITHUB_ENV;
+  if (isRunningOnGithub) {
+    // share env vars across steps https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-of-writing-an-environment-variable-to-github_env
+    appendFileSync(process.env.GITHUB_ENV, `${key}=${value}`);
+  } else {
+    console.log(`${key} = ${value}`);
+
+    // Just here for testing
+    return [key, value];
+  }
+}
+
+/**
+ * Log out an error message and quit with an error code
+ */
+function fail(message, e) {
+  if (IS_TEST) {
+    return false;
+  }
+
+  console.error(`${errorPrefix}${message}`);
+
+  if (e) {
+    console.dir(e);
+  }
+
+  process.exit(1);
+}
+
+/**
+ * Validates that the
+ * @param {string} app the name of the NX application or lib
+ * @returns
+ */
+function validateAppName(app) {
+  if (!app || !isString(app) || app.trim() === '') {
+    return fail('Empty app name');
+  }
+
+  const NX_WORKSPACE_FILE = 'workspace.json';
+  const appDir = path.dirname(__dirname);
+
+  const f = `${appDir}/../../${NX_WORKSPACE_FILE}`;
+  const workspace = JSON.parse(readFileSync(f), { encoding: 'utf-8' });
+  const projects = [...Object.keys(workspace.projects)];
+
+  if (!app) {
+    return fail('`app` parameter was not set (should be an NX app/lib name)');
+  }
+
+  if (projects.indexOf(app.trim()) == -1) {
+    return fail(`Project "${app}" does not exist in workspace.json`);
+  }
+
+  return true;
+}
+
+module.exports = {
+  getBranchNameFromGithubRef,
+  getBranch,
+  output,
+  fail,
+  validateAppName,
+  IS_TEST,
+  IS_PULL_REQUEST,
+  specialCasePrefix,
+};

tools/scripts/lib/ci-functions.spec.js

@@ -0,0 +1,51 @@
+const {
+  validateAppName,
+  getBranchNameFromGithubRef,
+} = require('./ci-functions');
+const {
+  AppsThatDeployToMainnetFromDevelop,
+} = require('../publish-dist-set-vars');
+
+describe('validAppName', () => {
+  test(`getValidAppName returns true for major apps, because they exist in '../../workspace.json'`, () => {
+    expect(validateAppName('trading')).toBe(true);
+    expect(validateAppName('explorer')).toBe(true);
+    expect(validateAppName('governance')).toBe(true);
+  });
+
+  test(`getValidAppName returns true for all AppsThatDeployToMainnet, because they exist in '../../workspace.json'`, () => {
+    AppsThatDeployToMainnetFromDevelop.forEach((a) => {
+      expect(validateAppName(a)).toBe(true);
+    });
+  });
+
+  test(`getValidAppName returns false for apps that have invalid names or are not in workspace.json, '../../workspace.json'`, () => {
+    expect(validateAppName('')).toBe(false);
+    expect(validateAppName('❌')).toBe(false);
+    expect(validateAppName('random-made-up-app')).toBe(false);
+  });
+});
+
+describe('getBranchNameFromGithubRef', () => {
+  test('Handles empty refs', () => {
+    // False is returned by failure
+    expect(getBranchNameFromGithubRef()).toStrictEqual(false);
+    expect(getBranchNameFromGithubRef('❌')).toStrictEqual(false);
+    expect(getBranchNameFromGithubRef(false)).toStrictEqual(false);
+    expect(getBranchNameFromGithubRef('')).toStrictEqual(false);
+  });
+
+  test('Replaces /s with -s', () => {
+    expect(getBranchNameFromGithubRef('refs/heads/fix/123-test')).toStrictEqual(
+      'refs-heads-fix-123-test'
+    );
+  });
+
+  test('Trims a long ref to 50 characters', () => {
+    expect(
+      getBranchNameFromGithubRef(
+        'refs/heads/this-is-a-very-long-branch-name-that-is-over-50-characters-long'
+      )
+    ).toStrictEqual('refs-heads-this-is-a-very-long-branch-name-that-is');
+  });
+});

tools/scripts/publish-dist-set-vars.js

@@ -0,0 +1,177 @@
+/**
+ * A script that determines which S3 storage bucket an application build
+ * should be deployed to, given an application name and a branch or environment
+ * name.
+ *
+ * This is used in a github action to update dev, testnet, stagnet and some mainnet
+ * deploys for the apps in the frontend monorepo.
+ *
+ * You should be able to get away with just updating the configuration variables at the
+ * top of the script
+ *
+ * You can run the test suite with `npm run test:human` (nice formatting) or just `npm test)`
+ *
+ * In CI this is run by `publish-dist.yml`
+ * @author Miko
+ * @author Edd
+ */
+const {
+  validateAppName,
+  output,
+  fail,
+  IS_TEST,
+  specialCasePrefix,
+} = require('./lib/ci-functions');
+
+const {
+  AppsThatDeployToMainnetFromDevelop,
+  AppsThatDoNotDeployToMainnet,
+} = require('./config');
+
+const domains = {
+  mainnet: 'vega.xyz',
+  testnet: 'fairground.wtf',
+  default: 'vega.rocks',
+};
+
+const S3BucketNameForApp = {
+  'multisig-signer': 'tools.vega.xyz',
+  static: 'static.vega.xyz',
+  'ui-toolkit': 'ui.vega.rocks',
+};
+const DEFAULT_ENVIRONMENT = 'stagnet1';
+
+/**
+ * Extract the app name from the command line
+ * @returns {string} App name
+ */
+function getApp() {
+  const app = process.argv.pop();
+  try {
+    return validateAppName(app);
+  } catch (e) {
+    return fail(
+      `Could not read workspace configuration (workspace.json in the project root)`,
+      e
+    );
+  }
+}
+
+/**
+ * Extract the git ref from the environment
+ * @returns {string} Git reference
+ */
+function getRef() {
+  const ref = process.env.GITHUB_REF;
+
+  if (!ref) {
+    return fail(
+      `Error: 'ref' was not set (should be a fully-qualified git ref set in GITHUB_REF)`
+    );
+  }
+
+  return ref;
+}
+
+/**
+ * Derives the S3 bucket name from the environment
+ *
+ * @param {string} app The application name
+ * @param {string} envName The git reference
+ * @see getApp
+ * @see getRef
+ * @returns {string} full S3 storage bucket name
+ */
+function getBucketName(app, envName) {
+  if (AppsThatDoNotDeployToMainnet.has(app) && envName === 'mainnet') {
+    return fail(`${specialCasePrefix} ${app} does not deploy to mainnet`);
+  }
+
+  if (AppsThatDeployToMainnetFromDevelop.has(app)) {
+    return S3BucketNameForApp[app];
+  }
+
+  // mainnet / fairground apps do not contain environment name in the DNS
+  if (envName === 'mainnet') {
+    return `${app}.${domains.mainnet}`;
+  } else if (envName === 'testnet') {
+    return `${app}.${domains.testnet}`;
+  }
+
+  return `${app}.${envName}.${domains.default}`;
+}
+
+/**
+ * Determines an environment for commits on develop. Required because
+ * some applications have special cases.
+ *
+ * @param {string} environment name
+ * @returns {string} environment name
+ */
+function getEnvironmentFromDevelop(app) {
+  if (AppsThatDeployToMainnetFromDevelop.has(app)) {
+    return 'mainnet';
+  }
+
+  return DEFAULT_ENVIRONMENT;
+}
+
+function getEnvironmentFromGitRef(ref) {
+  // we take last part of release/* branch as environment name
+  try {
+    let envName = ref.split('/').slice(-1).toString();
+
+    if (envName.trim() === '') {
+      throw new Error('Ref name returned empty');
+    }
+
+    return envName;
+  } catch (e) {
+    return fail(`Could not parse release name from release ref "${ref}"`, e);
+  }
+}
+
+function getEnv(app, ref) {
+  if (ref.indexOf('release/') != -1) {
+    return getEnvironmentFromGitRef(ref);
+  } else if (ref.endsWith('develop')) {
+    // normal apps are deployed from develop to stagnet1
+    return getEnvironmentFromDevelop(app);
+  } else if (ref.endsWith('main')) {
+    return 'mainnet';
+  } else {
+    // Github should not have triggered in this case
+    fail(
+      `Environment name could not be determined from ref (got "${ref}"), expected something with 'main' or 'develop' or 'release'`
+    );
+  }
+}
+
+/* Runs all the default commands unless we're testing the app */
+if (!IS_TEST) {
+  // ============================================
+  // = Check that app is set and is a valid app =
+  // ============================================
+  // app is taken from gha templating ${{matrix.app}}`
+  const app = getApp();
+  // ref is taken from the environment variable GITHUB_REF
+  const ref = getRef();
+
+  const envName = getEnv(app, ref);
+  const bucketName = getBucketName(app, envName);
+
+  if (AppsThatDoNotDeployToMainnet.has(app) && envName === 'mainnet') {
+    fail(`${specialCasePrefix} ${app} does not deploy to ${envName}`);
+  }
+
+  output('BUCKET_NAME', bucketName);
+  output('ENV_NAME', envName);
+}
+
+module.exports = {
+  validateAppName,
+  AppsThatDeployToMainnetFromDevelop,
+  S3BucketNameForApp,
+  getBucketName,
+  getEnvironmentFromDevelop,
+};

tools/scripts/publish-dist-set-vars.spec.js

@@ -0,0 +1,79 @@
+const {
+  AppsThatDeployToMainnetFromDevelop,
+  S3BucketNameForApp,
+  getBucketName,
+  getEnvironmentFromDevelop,
+} = require('./publish-dist-set-vars.js');
+
+describe('Tests for publish-dist-set-vars', () => {
+  test('Every app that deploys to mainnet from develop has an explicit bucket set', () => {
+    AppsThatDeployToMainnetFromDevelop.forEach((a) => {
+      expect(!!S3BucketNameForApp[a]).toBe(true);
+    });
+  });
+
+  test('getBucketName works for known and unknown envs for major apps', () => {
+    expect(getBucketName('trading', 'mainnet')).toBe(false);
+    expect(getBucketName('trading', 'testnet')).toBe('trading.fairground.wtf');
+    expect(getBucketName('trading', 'dev')).toBe('trading.dev.vega.rocks');
+    expect(getBucketName('trading', 'magical-test-network')).toBe(
+      'trading.magical-test-network.vega.rocks'
+    );
+
+    expect(getBucketName('explorer', 'mainnet')).toBe('explorer.vega.xyz');
+    expect(getBucketName('explorer', 'testnet')).toBe(
+      'explorer.fairground.wtf'
+    );
+    expect(getBucketName('explorer', 'dev')).toBe('explorer.dev.vega.rocks');
+    expect(getBucketName('explorer', 'magical-test-network')).toBe(
+      'explorer.magical-test-network.vega.rocks'
+    );
+
+    expect(getBucketName('governance', 'mainnet')).toBe('governance.vega.xyz');
+    expect(getBucketName('governance', 'testnet')).toBe(
+      'governance.fairground.wtf'
+    );
+    expect(getBucketName('governance', 'dev')).toBe(
+      'governance.dev.vega.rocks'
+    );
+    expect(getBucketName('governance', 'magical-test-network')).toBe(
+      'governance.magical-test-network.vega.rocks'
+    );
+  });
+
+  test('getBucketName works for hypothetical apps', () => {
+    expect(getBucketName('testy-mctestface', 'mainnet')).toBe(
+      'testy-mctestface.vega.xyz'
+    );
+    expect(getBucketName('dolphinsdolphinsdolphins', 'testnet')).toBe(
+      'dolphinsdolphinsdolphins.fairground.wtf'
+    );
+    expect(getBucketName('kittens', 'dev')).toBe('kittens.dev.vega.rocks');
+    expect(getBucketName('this-is-not-an-app', 'magical-test-network')).toBe(
+      'this-is-not-an-app.magical-test-network.vega.rocks'
+    );
+  });
+
+  test('Every app that deploys to mainnet from develop should be special cased', () => {
+    AppsThatDeployToMainnetFromDevelop.forEach((a) => {
+      const knownBucket = S3BucketNameForApp[a];
+      ['mainnet', 'testnet', 'dev', 'magical-test-network'].forEach((n) => {
+        expect(getBucketName(a, n)).toEqual(knownBucket);
+      });
+    });
+  });
+
+  test('getEnvironmentFromDevelop returns the default environment for most apps, or mainnet for special cases', () => {
+    expect(getEnvironmentFromDevelop('trading')).toEqual('stagnet1');
+    expect(getEnvironmentFromDevelop('liquidity-provision-dashboard')).toEqual(
+      'stagnet1'
+    );
+    expect(getEnvironmentFromDevelop('static')).toEqual('mainnet');
+  });
+
+  test('Every app that deploys to mainnet from develop has an explicit bucket set', () => {
+    AppsThatDeployToMainnetFromDevelop.forEach((a) => {
+      expect(!!S3BucketNameForApp[a]).toBe(true);
+    });
+  });
+});