feat(trading,governance,explorer): dist on s3 (#3332)

.github/workflows/add_issue_new_projects.yml

@@ -13,7 +13,7 @@ env:
 
 jobs:
   add_issue:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: 'Add issue to project board'
         run: |

.github/workflows/ci-cd-trigger.yml

@@ -1,10 +1,9 @@
-name: PR Validations
+name: CI/CD
 
 on:
   push:
     branches:
-      - develop
-      - main
+      - release/*
   pull_request:
     types:
       - opened
@@ -12,42 +11,55 @@ on:
       - synchronize
       - ready_for_review
 jobs:
-  pr:
-    runs-on: ubuntu-latest
+  lint-test-build:
+    runs-on: ubuntu-22.04
+    name: '(CI) lint + unit test + build'
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Check node version
-        id: node-version
-        run: |
-          npmVersion=$(cat .nvmrc | head -n 1)
-          echo ::set-output name=npmVersion::${npmVersion}
-
       - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ steps.node-version.outputs.npmVersion }}
+          node-version-file: '.nvmrc'
+          # https://stackoverflow.com/questions/61010294/how-to-cache-yarn-packages-in-github-actions
+          cache: yarn
+
+      - name: Install root dependencies
+        run: yarn install --frozen-lockfile
 
       - name: Derive appropriate SHAs for base and head for `nx affected` commands
         uses: nrwl/nx-set-shas@v3
         with:
           main-branch-name: develop
 
-      - name: Restore node_modules from cache
-        uses: actions/cache@v3
-        with:
-          path: '**/node_modules'
-          key: node_modules-${{ hashFiles('**/yarn.lock') }}
+      - name: Check formatting
+        run: yarn nx format:check
 
-      - name: Install root dependencies
-        run: yarn install --frozen-lockfile
+      - name: Lint affected
+        run: yarn nx affected:lint --max-warnings=0
 
+      - name: Build affected spec
+        run: yarn nx affected --target=build-spec
+
+      - name: Test affected
+        run: yarn nx affected:test
+
+      - name: Build affected
+        run: yarn nx affected:build
+
+      # See affected apps
       - name: See affected apps
         run: |
-          affected="$(yarn nx print-affected --base=${{ env.NX_BASE }} --head=HEAD --select=projects)"
+          echo ">>>> debug"
+          echo "NX Version: $nx_version"
+          echo "NX_BASE: ${{ env.NX_BASE }}"
+          echo "NX_HEAD: ${{ env.NX_HEAD }}"
+          echo ">>>> eof debug"
+
+          affected="$(yarn nx print-affected --base=${{ env.NX_BASE }} --head=${{ env.NX_HEAD }} --select=projects)"
           echo -n "Affected projects: $affected"
 
           projects_e2e=""
@@ -64,32 +76,33 @@ jobs:
       projects: ${{ env.PROJECTS }}
       projects-e2e: ${{ env.PROJECTS_E2E }}
 
-  run-cypress:
-    needs: pr
-    if: ${{ needs.pr.outputs.projects != '[]' }}
+  cypress:
+    needs: lint-test-build
+    name: '(CI) cypress'
+    if: ${{ needs.lint-test-build.outputs.projects != '[]' }}
     uses: ./.github/workflows/cypress-run.yml
     secrets: inherit
     with:
-      projects: ${{ needs.pr.outputs.projects-e2e }}
+      projects: ${{ needs.lint-test-build.outputs.projects-e2e }}
       tags: '@smoke @regression'
 
-  run-docker-build:
-    needs: pr
-    if: ${{ needs.pr.outputs.projects != '[]' }}
-    uses: ./.github/workflows/publish-docker-containers.yml
+  publish-dist:
+    needs: lint-test-build
+    name: '(CD) publish dist'
+    if: ${{ needs.lint-test-build.outputs.projects != '[]' }}
+    uses: ./.github/workflows/publish-dist.yml
     secrets: inherit
     with:
-      projects: ${{ needs.pr.outputs.projects }}
+      projects: ${{ needs.lint-test-build.outputs.projects }}
 
   # Report single result at the end, to avoid mess with required checks in PR
-  result:
+  cypress-result:
     if: ${{ always() }}
-    needs: run-cypress
-    runs-on: ubuntu-latest
-    name: Cypress result
+    needs: cypress
+    runs-on: ubuntu-22.04
     steps:
       - run: |
-          result="${{ needs.run-cypress.result }}"
+          result="${{ needs.cypress.result }}"
           if [[ $result == "success" || $result == "skipped" ]]; then
             exit 0
           else

.github/workflows/cypress-live-test.yml

@@ -13,7 +13,7 @@ on:
 jobs:
   cypress-run:
     name: Run Cypress Trading tests -- live environment
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/cypress-run.yml

@@ -1,4 +1,4 @@
-name: Cypress Run
+name: (CI) Cypress Run
 on:
   workflow_call:
     inputs:

.github/workflows/generate-queries.yml

@@ -8,22 +8,25 @@ on:
 jobs:
   master:
     name: Generate Queries
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Setup node
+        uses: actions/setup-node@v3
         with:
-          fetch-depth: 0
-      - name: Use Node.js 16
-        id: Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: 16.15.1
+          node-version-file: '.nvmrc'
+          # https://stackoverflow.com/questions/61010294/how-to-cache-yarn-packages-in-github-actions
+          cache: yarn
+
       - name: Install root dependencies
-        run: yarn install
+        run: yarn install --frozen-lockfile
+
       - name: Generate queries
         run: node ./scripts/get-queries.js
+
       - uses: actions/upload-artifact@v2
         with:
           name: queries

.github/workflows/lint_pr.yml

@@ -3,21 +3,28 @@ name: Verify PR title
 
 on:
   pull_request:
-    types: [opened, ready_for_review, reopened, edited, synchronize]
+    types:
+      - opened
+      - ready_for_review
+      - reopened
+      - edited
+      - synchronize
 jobs:
   lint_pr:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Setup node
+        uses: actions/setup-node@v3
         with:
-          fetch-depth: 0
-      - name: Use Node.js 16
-        id: Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: 16.15.1
+          node-version-file: '.nvmrc'
+          # https://stackoverflow.com/questions/61010294/how-to-cache-yarn-packages-in-github-actions
+          cache: yarn
+
       - name: Install root dependencies
-        run: yarn install
+        run: yarn install --frozen-lockfile
+
       - name: Check PR title
         run: echo "${{ github.event.pull_request.title }}" | npx commitlint --config ./commitlint.config-ci.js

.github/workflows/process-tranches.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   master:
     name: Generate Queries
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Checkout

.github/workflows/publish-dist.yml

@@ -1,4 +1,4 @@
-name: Docker build
+name: (CD) Publish docker + s3
 
 on:
   workflow_call:
@@ -8,13 +8,13 @@ on:
         type: string
 
 jobs:
-  master:
+  publish-dist:
     strategy:
       fail-fast: false
       matrix:
         app: ${{ fromJSON(inputs.projects) }}
     name: ${{ matrix.app }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out code
         uses: actions/checkout@v3
@@ -29,41 +29,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      # https://docs.github.com/en/actions/learn-github-actions/contexts
-      # https://github.com/actions/checkout#Checkout-pull-request-HEAD-commit-instead-of-merge-commit
-      - name: Determine Docker Image tag
-        id: tags
-        run: |
-          npmVersion=$(cat .nvmrc | head -n 1)
-          versionTag=${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || github.event.pull_request.head.sha }}
-          echo ::set-output name=npmVersion::${npmVersion}
-          echo ::set-output name=version::${versionTag}
-
-      - name: Print config
-        run: |
-          git rev-parse --verify HEAD
-          git status
-          echo "steps.tags.outputs.version=${{ steps.tags.outputs.version }}"
-
-      - name: Build and export to local Docker
-        uses: docker/build-push-action@v3
-        with:
-          load: true
-          build-args: |
-            APP=${{ matrix.app }}
-            NODE_VERSION=${{ steps.tags.outputs.npmVersion }}
-          tags: |
-            ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local
-
-      - name: Sanity check docker image
-        run: |
-          echo "Check .env file"
-          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local cat .env
-          echo "Check ipfs-hash"
-          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local cat ipfs-hash
-          echo "List html directory"
-          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local ls -lah
-
       - name: Log in to the Container registry
         uses: docker/login-action@v2
         with:
@@ -71,17 +36,66 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push
+      # https://docs.github.com/en/actions/learn-github-actions/contexts
+      - name: Check node version
+        id: tags
+        run: |
+          nodeVersion=$(cat .nvmrc | head -n 1)
+          echo ::set-output name=nodeVersion::${nodeVersion}
+
+          if [[ "${{ github.event_name }}" = "push" ]]; then
+            envName="$(echo ${{ github.ref }} | rev | cut -d '/' -f 1 | rev)"
+            bucketName="${{ github.event.repository.name }}-$envName"
+            echo ::set-output name=bucketName::${bucketName}
+            echo ::set-output name=envName::${envName}
+          fi
+
+      - name: Build and export to local Docker
         id: docker_build
         uses: docker/build-push-action@v3
+        with:
+          load: true
+          build-args: |
+            APP=${{ matrix.app }}
+            NODE_VERSION=${{ steps.tags.outputs.nodeVersion }}
+            ENV_NAME=${{ steps.tags.outputs.envName || '' }}
+          tags: |
+            ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local
+
+      - name: Sanity check docker image
+        run: |
+          echo "Check ipfs-hash"
+          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local cat ipfs-hash
+
+          echo "List html directory"
+          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local ls -lah
+
+          echo "Copy dist to local filesystem"
+          docker create --name=dist ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local
+          docker cp dist:/usr/share/nginx/html dist
+
+          echo "Check local dist"
+          ls -al dist
+
+      - name: Publish dist as docker image
+        uses: docker/build-push-action@v3
+        if: ${{ github.event_name == 'pull_request' }}
         with:
           push: true
           build-args: |
             APP=${{ matrix.app }}
-            NODE_VERSION=${{ steps.tags.outputs.npmVersion }}
+            NODE_VERSION=${{ steps.tags.outputs.nodeVersion }}
           tags: |
-            ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:latest
-            ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:${{ steps.tags.outputs.version }}
+            ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:${{ github.event.pull_request.head.sha || github.sha }}
+
+      # - uses: shallwefootball/s3-upload-action@master
+      #   if: ${{ github.event_name == 'push' }}
+      #   name: Upload dist S3
+      #   with:
+      #     aws_key_id: ${{ secrets.AWS_KEY_ID }}
+      #     aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+      #     aws_bucket: ${{ steps.tags.outputs.bucketName }}
+      #     source_dir: 'dist'
 
       - name: Add preview label
         uses: actions-ecosystem/action-add-labels@v1

.github/workflows/publish-npm.yml

@@ -19,29 +19,27 @@ on:
 jobs:
   publish:
     name: Build & Publish - Tag
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       contents: 'read'
       actions: 'read'
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: User Node.js 16
-        id: Node
+
+      - name: Setup node
         uses: actions/setup-node@v3
         with:
-          node-version: 16.15.1
-      - name: Restore node_modules from cache
-        uses: actions/cache@v3
-        with:
-          path: '**/node_modules'
-          key: node_modules-${{ hashFiles('**/yarn.lock') }}
+          node-version-file: '.nvmrc'
+          # https://stackoverflow.com/questions/61010294/how-to-cache-yarn-packages-in-github-actions
+          cache: yarn
+
       - name: Install root dependencies
         run: yarn install --frozen-lockfile
+
       - name: Build project
         run: yarn nx build ${{inputs.project}}
+
       - name: Publish project to @vegaprotocol
         uses: JS-DevTools/npm-publish@v1
         with:

.github/workflows/test.yml

@@ -1,46 +0,0 @@
-name: Unit tests & build
-
-on:
-  push:
-    branches:
-      - develop
-      - main
-  pull_request:
-jobs:
-  pr:
-    name: Test and lint - PR
-    runs-on: ubuntu-latest
-    permissions:
-      contents: 'read'
-      actions: 'read'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Derive appropriate SHAs for base and head for `nx affected` commands
-        uses: nrwl/nx-set-shas@v2
-        with:
-          main-branch-name: ${{ github.base_ref }}
-      - name: Use Node.js 16
-        id: Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16.15.1
-      - name: Restore node_modules from cache
-        uses: actions/cache@v3
-        with:
-          path: '**/node_modules'
-          key: node_modules-${{ hashFiles('**/yarn.lock') }}
-      - name: Install root dependencies
-        run: yarn install --frozen-lockfile
-      - name: Check formatting
-        run: yarn nx format:check
-      - name: Lint affected
-        run: yarn nx affected:lint --max-warnings=0
-      - name: Test affected
-        run: yarn nx affected:test
-      - name: Build affected
-        run: yarn nx affected:build
-      - name: Build affected spec
-        run: yarn nx affected --target=build-spec

Dockerfile

@@ -4,6 +4,7 @@ FROM --platform=amd64 node:${NODE_VERSION}-alpine3.16 as build
 WORKDIR /app
 # Argument to allow building of different apps
 ARG APP
+ARG ENV_NAME=""
 RUN apk add --update --no-cache \
   python3 \
   make \
@@ -18,16 +19,10 @@ RUN sh ./docker-build.sh
 # if this fails you need to docker pull nginx:1.23-alpine and pin new SHA
 # this is to ensure that we run always same version of alpine to make sure ipfs is indempotent
 FROM --platform=amd64 nginx:1.23-alpine@sha256:6318314189b40e73145a48060bff4783a116c34cc7241532d0d94198fb2c9629
-ARG APP
 # configuration of system
-RUN apk add --no-cache bash go-ipfs
 EXPOSE 80
-COPY entrypoint.sh /entrypoint.sh
-CMD ["/entrypoint.sh"]
-
 # Copy dist
 WORKDIR /usr/share/nginx/html
 COPY nginx/nginx.conf /etc/nginx/conf.d/default.conf
 COPY --from=build /app/dist/apps/${APP} /usr/share/nginx/html
-COPY ./apps/${APP}/.env .env
-RUN ipfs init && echo "$(ipfs add -rQ .)" > ipfs-hash
+RUN apk add --no-cache go-ipfs; ipfs init && echo "$(ipfs add -rQ .)" > ipfs-hash; apk del go-ipfs

docker-build.sh

@@ -1,10 +1,22 @@
-#!/bin/sh -eux
+#!/bin/bash -ex
+
 export PATH="/app/node_modules/.bin:$PATH"
+
+flags="--network-timeout 100000 --pure-lockfile"
+
+if [[ ! -z "${ENV_NAME}" ]]; then
+  flags="--env=${ENV_NAME} $flags"
+fi
+
 if [ "${APP}" = "trading" ]; then
-  yarn nx export ${APP} --network-timeout 100000 --pure-lockfile
+  yarn nx export ${APP} $flags
   mv /app/dist/apps/trading/exported/ /app/tmp
   rm -rf /app/dist/apps/trading
   mv /app/tmp /app/dist/apps/trading
 else
-  yarn nx build ${APP} --network-timeout 100000 --pure-lockfile
+  yarn nx build ${APP} $flags
 fi
+
+env_vars_file="/app/dist/apps/${APP}/.env"
+# make sure there are no exposed .env files
+rm $env_vars_file || echo "No env vars file"

entrypoint.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-set -e
-
-# Recreate config file
-env_file=/usr/share/nginx/html/assets/env-config.js
-mkdir -p $(dirname $env_file)
-rm -rf $env_file || echo "no file to delete"
-touch $env_file
-
-env_vars_file=/usr/share/nginx/html/.env
-sed -i '/^#/d' $env_vars_file # remove comment lines
-sed -i '/^$/d' $env_vars_file # remove empty lines
-
-# Add assignment
-echo "window._env_ = {" >> $env_file
-
-# Read each line in .env file
-# Each line represents key=value pairs
-while read -r line || [[ -n "$line" ]];
-do
-  # Split env variables by character `=`
-  if printf '%s\n' "$line" | grep -q -e '='; then
-    varname=$(printf '%s\n' "$line" | sed -e 's/=.*//')
-    varvalue=$(printf '%s\n' "$line" | sed -e 's/^[^=]*=//')
-  fi
-
-  # Read value of current variable if exists as Environment variable
-  value=$(printf '%s\n' "${!varname}")
-  # Otherwise use value from .env file
-  [[ -z $value ]] && value=${varvalue}
-
-  # Append configuration property to JS file if non-empty
-  if [ ! -z "$varname" ]; then
-    echo "  $varname: \"$value\"," >> $env_file
-  fi
-done < $env_vars_file
-
-rm $env_vars_file
-
-echo "}" >> $env_file
-
-# start serving
-nginx -g 'daemon off;'