feat(ci): establish build based on docker to produce ipfs-hashes (#3612)

.github/workflows/ci-cd-trigger.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - release/*
       - develop
+    tags:
+      - v*
   pull_request:
     types:
       - opened
@@ -97,15 +99,13 @@ jobs:
       - name: See affected apps
         run: |
           echo ">>>> debug"
-          echo "NX Version: $nx_version"
           echo "NX_BASE: ${{ env.NX_BASE }}"
           echo "NX_HEAD: ${{ env.NX_HEAD }}"
           echo ">>>> eof debug"
 
           affected="$(yarn nx print-affected --base=${{ env.NX_BASE }} --head=${{ env.NX_HEAD }} --select=projects)"
-          echo -n "Affected projects: $affected"
 
-          branch_slug="$(echo ${{ github.head_ref || github.ref_name }} | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )"
+          branch_slug="$(echo '${{ github.head_ref || github.ref_name }}' | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )"
           projects_e2e=""
           preview_governance="not deployed"
           preview_trading="not deployed"

.github/workflows/publish-dist.yml

@@ -30,13 +30,21 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Log in to the Container registry
+      - name: Log in to the Container registry (ghcr)
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Log in to the Container registry (docker hub)
+        uses: docker/login-action@v2
+        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+        with:
+          # registry: registry.hub.docker.com
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -54,30 +62,24 @@ jobs:
       - name: Define variables
         run: |
           envName=''
-          dockerfile="dist.Dockerfile"
           if [[ "${{ github.event_name }}" = "push" ]]; then
             domain="vega.rocks"
             if [[ "${{ github.ref }}" =~ .*release/.* ]]; then
               envName="$(echo ${{ github.ref }} | rev | cut -d '/' -f 1 | rev)"
               if [[ "${{ github.ref }}" =~ .*mainnet.* ]]; then
                 domain="vega.community"
-                if [[ "${{ matrix.app }}" = "trading" ]]; then
-                  dockerfile="ipfs.Dockerfile"
-                fi
               fi
             elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then
               envName="stagnet1"
+            elif [[ "${{ matrix.app}}" = "trading" ]] && [[ ${{ startsWith(github.ref, 'refs/tags/v') && 'true' || 'false' }} = "true" ]]; then
+              envName="mainnet"
             fi
             bucketName="${{ matrix.app }}.${envName}.${domain}"
             echo BUCKET_NAME=${bucketName} >> $GITHUB_ENV
           fi
-          nodeVersion=$(cat .nvmrc | head -n 1)
           echo ENV_NAME=${envName} >> $GITHUB_ENV
-          echo NODE_VERSION=${nodeVersion} >> $GITHUB_ENV
-          echo DOCKERFILE=docker/${dockerfile} >> $GITHUB_ENV
 
       - name: Build local dist
-        if: ${{ env.DOCKERFILE != 'docker/ipfs.Dockerfile' }}
         run: |
           flags=""
           if [[ ! -z "${{ env.ENV_NAME }}" ]]; then
@@ -98,62 +100,59 @@ jobs:
 
       - name: Build and export to local Docker
         id: docker_build
-        if: ${{ github.event_name == 'pull_request' || ( env.DOCKERFILE == 'docker/ipfs.Dockerfile' && github.event_name == 'push' ) }}
         uses: docker/build-push-action@v3
         with:
           context: .
-          file: ${{ env.DOCKERFILE }}
+          file: docker/node-outside-docker.Dockerfile
           load: true
           build-args: |
             APP=${{ matrix.app }}
-            NODE_VERSION=${{ env.NODE_VERSION }}
             ENV_NAME=${{ env.ENV_NAME }}
           tags: |
             ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local
 
       - name: Image digest
-        if: ${{ github.event_name == 'pull_request' || ( env.DOCKERFILE == 'docker/ipfs.Dockerfile' && github.event_name == 'push' ) }}
+        if: ${{ github.event_name == 'pull_request' }}
         run: echo ${{ steps.docker_build.outputs.digest }}
 
       - name: Sanity check docker image
-        if: ${{ github.event_name == 'pull_request' || ( env.DOCKERFILE == 'docker/ipfs.Dockerfile' && github.event_name == 'push' ) }}
         run: |
           echo "Check ipfs-hash"
-          if [[ "${{ env.DOCKERFILE }}" = "docker/ipfs.Dockerfile" ]];  then
-            docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local cat /ipfs-hash
-          fi
-
+          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local cat /ipfs-hash
+          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local cat /ipfs-hash > ipfs-hash
           echo "List html directory"
-          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local sh -c 'apk add --update tree; tree .'
+          docker run --rm ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local sh -c 'apk add --update tree; tree /usr/share/nginx/html'
 
-      - name: Copy dist to local filesystem
-        if: ${{ env.DOCKERFILE == 'docker/ipfs.Dockerfile' && github.event_name == 'push' }}
-        run: |
-          docker create --name=dist ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:local
-          docker cp dist:/usr/share/nginx/html dist
-
-          echo "check local dist files"
-          tree dist/html
-          mv dist/html dist-result
-
-      - name: Publish dist as docker image
+      - name: Publish dist as docker image (ghcr)
         uses: docker/build-push-action@v3
-        if: ${{ github.event_name == 'pull_request' }}
+        if: ${{ github.event_name == 'pull_request' || (matrix.app == 'trading' && github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/v') ) }}
         with:
           context: .
-          file: ${{ env.DOCKERFILE }}
+          file: docker/node-outside-docker.Dockerfile
           push: true
           build-args: |
             APP=${{ matrix.app }}
-            NODE_VERSION=${{ env.NODE_VERSION }}
             ENV_NAME=${{ env.ENV_NAME }}
           tags: |
             ghcr.io/vegaprotocol/frontend/${{ matrix.app }}:${{ github.event.pull_request.head.sha || github.sha }}
 
+      - name: Publish dist as docker image (docker hub)
+        uses: docker/build-push-action@v3
+        if: ${{ matrix.app == 'trading' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
+        with:
+          context: .
+          file: docker/node-outside-docker.Dockerfile
+          push: true
+          build-args: |
+            APP=${{ matrix.app }}
+            ENV_NAME=${{ env.ENV_NAME }}
+          tags: |
+            vegaprotocol/${{ matrix.app }}:${{ github.ref_name }}
+
       # bucket creation in github.com/vegaprotocol/terraform//frontend
       - name: Publish dist to s3
         uses: jakejarvis/s3-sync-action@master
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/v') }}
         with:
           args: --acl private --follow-symlinks --delete
         env:
@@ -169,3 +168,9 @@ jobs:
         with:
           labels: ${{ matrix.app }}-preview
           number: ${{ github.event.number }}
+
+      - name: Add ipfs hash to release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/v')
+        with:
+          files: ipfs-hash

.gitignore

@@ -2,6 +2,7 @@
 
 # compiled output
 /dist
+/dist-result
 /tmp
 /out-tsc
 /tools/executors/**/*.js

Makefile

@@ -0,0 +1,20 @@
+.PHONY: latest-release
+latest-release:
+  gh release list | head -n 1 | awk '{print $1}'
+
+.PHONY: show-latest-release
+show-latest-release:
+  gh release view `gh release list | head -n 1 | awk '{print $1}'`
+
+.PHONY: recalculate-ipfs
+recalculate-ipfs:
+  echo "ipfs hash inside the image"
+  docker run --rm ${TAG} cat /ipfs-hash
+  echo "recalculating ipfs hash"
+  docker run --rm ${TAG} ipfs add -rw /usr/share/nginx/html
+
+.PHONY: eject-ipfs-hash
+unpack:
+  docker create --name=dist ${TAG}
+  docker cp dist:/usr/share/nginx/html dist
+  docker rm dist

README.md

@@ -103,25 +103,68 @@ In CI linting, formatting and also run. These checks can be seen in the [CI work
 
 Visit the [Nx Documentation](https://nx.dev/getting-started/intro) to learn more.
 
-# Docker & Vegacapsule
+# 🐋 Hosting a console
 
-## Docker
+To host a console there are two possible build scenarios for running the frontends: nx performed **outside** or **inside** docker build. For specific build instructions follow [build instructions](#build-instructions).
 
-The [Dockerfile](./dockerfiles) for running the frontends is pretty basic, merely building the application with the APP arg that is passed in and serving the application from [nginx](./nginx/nginx.conf). The only complexity that exists is that there is a script which allows the passing of run time environment variables to the containers. See configuration below for how to do this.
-
-You can build any of the containers locally with the following command:
-
-```bash
-docker build --dockerfile dockerfiles/Dockerfile.cra . --build-arg APP=[YOUR APP] --tag=[TAG]
-```
-
-In order to run a container:
+In order to run a container on port 3000:
 
 ```bash
 docker run -p 3000:80 [TAG]
 ```
 
-Images ending with `.dist` are to pack locally created transpiled HTML files into nginx container for non-compatible with yarn architectures like M1 Mac
+## Build instructions
+
+The [`docker`](./docker) subfolder has some docker configurations for easily setting up your own hosted version of console either for the web, or ready for pinning on IPFS
+
+### nx build outside the docker
+
+Packaging prepared dist into [`nginx`](https://hub.docker.com/_/nginx)([server configuration](./nginx/nginx.conf)) docker image involves building the application on docker host machine from source.
+
+As a prerequisite you need to perform build of `dist` directory and move its content for specific application to `dist-result` directory. Use following script to do it with a single command:
+
+```bash
+./docker/prepare-dist.sh
+```
+
+You can build any of the containers locally with the following command:
+
+```bash
+docker build --dockerfile docker/node-outside-docker.Dockerfile . --tag=[TAG]
+```
+
+### nx build inside the docker
+
+Using multistage dockerfile dist is compiled using [node](https://hub.docker.com/_/node) image and later packed to nginx as in [dist build](#dist-build) example.
+
+```bash
+docker build --build-arg APP=[YOUR APP] --build-arg NODE_VERSION=$(cat .nvmrc) --build-arg ENV_NAME=mainnet -t [TAG] -f docker/node-inside-docker.Dockerfile .
+```
+
+### Computing ipfs-hash of the build
+
+At the moment this feature is important only for `trading` (console) releases.
+
+Each docker build finishes with hash calculation for dist directory. Resulting hash is added to file named as `/ipfs-hash`. Once docker image is produced you can run following commad to display ipfs-hash:
+
+```bash
+make recalculate-ipfs TAG=vegaprotocol/trading:{YOUR_VERSION}
+```
+
+**updating hash:** recompiling dist directory (even if there are no changed to source code) results in different hash computed by ipfs command.
+
+### Verifying ipfs-hash of existing current application version
+
+An IPFS CID will be attached to every [release](https://github.com/vegaprotocol/frontend-monorepo/releases). If you are intending to pin an application on IPFS, you can check that your build matches by running the following steps:
+
+1. Show latest release by runnning: `make latest-release`. You need to configure [`gh`](https://cli.github.com/) for this step to work, otherwise please provide release manually from [github](https://github.com/vegaprotocol/frontend-monorepo/releases) or [dockerhub](https://hub.docker.com/r/vegaprotocol/trading)
+2. Set RELEASE environment variable to value that you want to validate: `export RELEASE=$(make latest-release)` or `export RELEASE=vXX.XX.XX`
+3. Set TAG environment variable to image that you want to validate: `export TAG=vegaprotocol/trading:$RELEASE`
+4. Download docker image with the desired release `docker pull $TAG`.
+5. Recalculate hash: `make recalculate-ipfs`
+6. You should see exactly same hash produced by ipfs command as one placed with the release notes: `make show-latest-release`
+7. If you want to extract dist from docker image to your local filesystem you can run following command: `make unpack`
+8. Now `dist` directory contains valid application build. **it is not possible to calculate same ipfs hash on files that are result of copy operation**
 
 ## Config
 

docker/node-inside-docker.Dockerfile

@@ -0,0 +1,31 @@
+# Build container
+ARG NODE_VERSION
+FROM --platform=amd64 node:${NODE_VERSION}-alpine3.16 as build
+WORKDIR /app
+# Argument to allow building of different apps
+ARG APP
+ARG ENV_NAME=""
+RUN apk add --update --no-cache \
+  python3==3.10.11-r0 \
+  make==4.3-r0 \
+  gcc==11.2.1_git20220219-r2 \
+  g++==11.2.1_git20220219-r2
+COPY . ./
+RUN yarn --network-timeout 100000 --pure-lockfile
+# work around for different build process in trading
+RUN sh docker/docker-build.sh
+
+# Server environment
+# if this fails you need to docker pull nginx:1.23-alpine and pin new SHA
+# this is to ensure that we run always same version of alpine to make sure ipfs is indempotent
+FROM --platform=amd64 nginx:1.23-alpine@sha256:6318314189b40e73145a48060bff4783a116c34cc7241532d0d94198fb2c9629
+# configuration of system
+EXPOSE 80
+# Copy dist
+COPY docker/nginx.conf /etc/nginx/conf.d/default.conf
+RUN rm -rf /usr/share/nginx/html/*
+COPY --from=build /app/dist/apps/${APP}/* /usr/share/nginx/html
+RUN apk add --no-cache go-ipfs==0.16.0-r6 \
+      && ipfs init \
+      && echo "$(ipfs add -rwQ /usr/share/nginx/html)" > /ipfs-hash \
+      && echo "ipfs hash of this build: $(cat /ipfs-hash)"

docker/node-outside-docker.Dockerfile

@@ -0,0 +1,10 @@
+FROM --platform=amd64 nginx:1.23-alpine@sha256:6318314189b40e73145a48060bff4783a116c34cc7241532d0d94198fb2c9629
+EXPOSE 80
+COPY docker/nginx.conf /etc/nginx/conf.d/default.conf
+RUN rm -rf /usr/share/nginx/html/*
+COPY ./dist-result/ /usr/share/nginx/html/
+RUN apk add --no-cache go-ipfs==0.16.0-r6 \
+      && ipfs init \
+      && echo "$(ipfs add -rwQ /usr/share/nginx/html)" > /ipfs-hash \
+      && echo "ipfs hash of this build: $(cat /ipfs-hash)"
+

docker/prepare-dist.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+yarn --pure-lockfile
+app={$1:-trading}
+flags="--env=${$2:-mainnet}"
+yarn install
+if [ "${app}" = "trading" ]; then
+  yarn nx export trading $flags
+  DIST_LOCATION=dist/apps/trading/exported
+else
+  yarn nx build ${app} $flags
+  DIST_LOCATION=dist/apps/${app}
+fi
+cp -r $DIST_LOCATION dist-result