feat(ci): migrate ci-cd trigger to python (#4330)

.github/workflows/ci-cd-trigger.yml

@@ -81,6 +81,22 @@ jobs:
         with:
           main-branch-name: develop
 
+      # See affected apps
+      - name: See affected apps
+        run: |
+          branch_slug="$(echo '${{ github.head_ref || github.ref_name }}' | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )"
+          python3 tools/ci/check-affected.py --github-ref="${{ github.ref }}" --branch-slug="$branch_slug" --event-name="${{ github.event_name }}"
+
+      - name: Verify script result
+        run: |
+          echo "Check outputs from script"
+          echo "projects: ${{ env.PROJECTS }}"
+          echo "projects-e2e: ${{ env.PROJECTS_E2E }}"
+          echo "preview_governance: ${{ env.PREVIEW_GOVERNANCE }}"
+          echo "preview_trading: ${{ env.PREVIEW_TRADING }}"
+          echo "preview_explorer: ${{ env.PREVIEW_EXPLORER }}"
+          echo "preview_tools: ${{ env.PREVIEW_TOOLS }}"
+
       - name: Check formatting
         run: yarn nx format:check
 
@@ -96,126 +112,6 @@ jobs:
       - name: Build affected
         run: yarn nx affected:build || (yarn install && yarn nx affected:build)
 
-      # See affected apps
-      - name: See affected apps
-        run: |
-          affected="$(yarn nx print-affected --base=${{ env.NX_BASE }} --head=${{ env.NX_HEAD }} --select=projects)"
-          branch_slug="$(echo '${{ github.head_ref || github.ref_name }}' | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )"
-
-          echo ">>>> debug"
-          echo "NX_BASE: ${{ env.NX_BASE }}"
-          echo "NX_HEAD: ${{ env.NX_HEAD }}"
-          echo "Affected: ${affected}"
-          echo "Branch slug: ${branch_slug}"
-          echo "Current ref: ${{ github.ref }}"
-          echo ">>>> eof debug"
-
-          projects_array=()
-
-          preview_governance="not deployed"
-          preview_trading="not deployed"
-          preview_explorer="not deployed"
-          preview_tools="not deployed"
-
-          # parse if affected is any of three main applications, if none - use all of them
-          if echo "$affected" | grep -q governance; then
-            echo "Governance is affected"
-            projects_array+=("governance")
-            preview_governance=$(printf "https://%s.%s.vega.rocks" "governance" "$branch_slug")
-          fi
-          if echo "$affected" | grep -q trading; then
-            echo "Trading is affected"
-            projects_array+=("trading")
-            preview_trading=$(printf "https://%s.%s.vega.rocks" "trading" "$branch_slug")
-          fi
-          if echo "$affected" | grep -q explorer; then
-            echo "Explorer is affected"
-            projects_array+=("explorer")
-            preview_explorer=$(printf "https://%s.%s.vega.rocks" "explorer" "$branch_slug")
-          fi
-          if [[ ${#projects_array[@]} -eq 0 ]]; then
-            projects_array=("governance" "trading" "explorer")
-            preview_governance=$(printf "https://%s.%s.vega.rocks" "governance" "$branch_slug")
-            preview_trading=$(printf "https://%s.%s.vega.rocks" "trading" "$branch_slug")
-            preview_explorer=$(printf "https://%s.%s.vega.rocks" "explorer" "$branch_slug")
-          fi
-
-          # applications parsed before this loop are applicable for running e2e-tests
-          projects_e2e_array=()
-          for project in "${projects_array[@]}"; do
-            projects_e2e_array+=("${project}-e2e")
-          done
-          # all applications below this loop are not applicable for running e2e-test
-
-          # check if pull request event to deploy tools
-          if [[ "${{ github.event_name }}" = "pull_request" ]]; then
-            if echo "$affected" | grep -q multisig-signer; then
-              echo "Tools are affected"
-              echo "Deploying tools on preview"
-              preview_tools=$(printf "https://%s.%s.vega.rocks" "tools" "$branch_slug")
-
-              projects_array+=("multisig-signer")
-            fi
-          # those apps deploy only from develop to mainnet
-          elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then
-            if echo "$affected" | grep -q multisig-signer; then
-              echo "Tools are affected"
-              echo "Deploying tools on s3"
-
-              projects_array+=("multisig-signer")
-            fi
-            if echo "$affected" | grep -q static; then
-              echo "static is affected"
-              echo "Deploying static on s3"
-
-              projects_array+=("static")
-            fi
-            if echo "$affected" | grep -q ui-toolkit; then
-              echo "ui-toolkit is affected"
-              echo "Deploying ui-toolkit on s3"
-
-              projects_array+=("ui-toolkit")
-            fi
-          fi
-
-          # if branch starts with release/ and ends with trading / governance or explorer - overwrite the array of affected projects with fixed single application
-          if [[ "${{ github.ref }}" == *release* ]]; then
-            echo ">> This is a relase branch"
-            case "${{ github.ref }}" in
-              *trading)
-                echo ">> Only trading will be deployed"
-                projects_array=(trading)
-                projects_e2e_array=(trading)
-              ;;
-              *governance)
-                echo ">> Only governance will be deployed"
-                projects_array=(governance)
-                projects_e2e_array=(governance)
-              ;;
-              *explorer)
-                echo ">> Only explorer will be deployed"
-                projects_array=(explorer)
-                projects_e2e_array=(explorer)
-              ;;
-              *)
-                echo ">> All apps will be deployed"
-              ;;
-            esac
-          fi
-
-          echo "Projects: ${projects_array[@]}"
-          echo "Projects E2E: ${projects_e2e_array[@]}"
-          projects_json=$(jq -M --compact-output --null-input '$ARGS.positional' --args -- "${projects_array[@]}")
-          projects_e2e_json=$(jq -M --compact-output --null-input '$ARGS.positional' --args -- "${projects_e2e_array[@]}")
-
-          echo PROJECTS_E2E=$projects_e2e_json >> $GITHUB_ENV
-          echo PROJECTS=$projects_json >> $GITHUB_ENV
-
-          echo PREVIEW_GOVERNANCE=$preview_governance >> $GITHUB_ENV
-          echo PREVIEW_TRADING=$preview_trading >> $GITHUB_ENV
-          echo PREVIEW_EXPLORER=$preview_explorer >> $GITHUB_ENV
-          echo PREVIEW_TOOLS=$preview_tools >> $GITHUB_ENV
-
     outputs:
       projects: ${{ env.PROJECTS }}
       projects-e2e: ${{ env.PROJECTS_E2E }}

.github/workflows/publish-dist.yml

@@ -97,52 +97,13 @@ jobs:
       - name: Define dist variables
         if: ${{ github.event_name == 'push' }}
         run: |
-          envName=''
+          python3 tools/ci/define-dist-variables.py --github-ref="${{ github.ref }}" --app="${{ matrix.app }}"
-          domain="vega.rocks"
-          bucketName=''
 
-          if [[ "${{ github.ref }}" =~ .*release/.* ]]; then
+      - name: Verify script result
-            # remove prefixing release/ and take the first string limited by - which is supposed to be name of the environment for releasing (format: release/testnet-trading)
+        if: ${{ github.event_name == 'push' }}
-            envName="$(echo ${{ github.ref }} | sed -e "s|refs/heads/release/||" | cut -d '-' -f 1 )"
+        run: |
-          elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then
+          echo "BUCKET_NAME=${{ env.BUCKET_NAME }}"
-            envName="stagnet1"
+          echo "ENV_NAME=${{ env.ENV_NAME }}"
-            if [[ "${{ matrix.app }}" = "multisig-signer" ]]; then
-              envName="mainnet"
-              bucketName="tools.vega.xyz"
-            fi
-            if [[ "${{ matrix.app }}" = "static" ]]; then
-              envName="mainnet"
-              bucketName="static.vega.xyz"
-            fi
-            if [[ "${{ matrix.app }}" = "ui-toolkit" ]]; then
-              envName="mainnet"
-              bucketName="ui.vega.rocks"
-            fi
-          elif [[ "${{ github.ref }}" =~ .*mainnet$ ]]; then
-            envName="mainnet"
-          fi
-
-          if [[ "${envName}" = "mainnet" ]]; then
-            domain="vega.xyz"
-            if [[ -z "${bucketName}" ]]; then
-              bucketName="${{ matrix.app }}.${domain}"
-            fi
-          elif [[ "${envName}" = "testnet" ]]; then
-            domain="fairground.wtf"
-            if [[ -z "${bucketName}" ]]; then
-              bucketName="${{ matrix.app }}.${domain}"
-            fi
-          fi
-
-          if [[ -z "${bucketName}" ]]; then
-            bucketName="${{ matrix.app }}.${envName}.${domain}"
-          fi
-
-          echo "bucket name: ${bucketName}"
-          echo "env name: ${envName}"
-
-          echo BUCKET_NAME=${bucketName} >> $GITHUB_ENV
-          echo ENV_NAME=${envName} >> $GITHUB_ENV
 
       - name: Build local dist
         run: |

tools/ci/check-affected.py

@@ -0,0 +1,110 @@
+from os import environ
+from subprocess import check_output
+from argparse import ArgumentParser
+import json
+
+projects = []
+projects_e2e = []
+
+previews = {
+  'governance': 'not deployed',
+  'explorer': 'not deployed',
+  'trading': 'not deployed',
+  'tools': 'not deployed',
+}
+
+main_apps = ['governance', 'explorer', 'trading']
+
+preview_governance="not deployed"
+preview_trading="not deployed"
+preview_explorer="not deployed"
+preview_tools="not deployed"
+
+# take input from the pipeline
+parser = ArgumentParser()
+
+# let's generate slug from bash spell for now
+parser.add_argument('--branch-slug', help='slug of branch')
+parser.add_argument('--github-ref', help='current github ref')
+parser.add_argument('--event-name', help='name of event in CI')
+args = parser.parse_args()
+
+# run yarn affected command
+affected=check_output(f'yarn nx print-affected --base={environ["NX_BASE"]} --head={environ["NX_HEAD"]} --select=projects'.split()).decode('utf-8')
+
+
+# print useful information
+print(">>>> debug")
+print(f"NX_BASE: { environ['NX_BASE'] }")
+print(f"NX_HEAD: { environ['NX_HEAD'] }")
+print(f"Branch slug: {args.branch_slug}")
+print(f"Current ref: {args.github_ref}")
+print(">> Affected output")
+print(affected)
+print(">>>> eof debug")
+
+# define affection actions -> add to projects arrays and generate preview link
+def affect_app(app, preview_name=None):
+  print(f"{app} is affected")
+  projects.append(app)
+  if not preview_name:
+    preview_name=app
+  previews[app] = f'https://{preview_name}.{args.branch_slug}.vega.rocks'
+
+# check appearance in the affected string for main apps
+for app in main_apps:
+  if app in affected:
+    affect_app(app)
+
+# if non of main apps is affected - test all of them
+if not projects:
+  for app in main_apps:
+    affect_app(app)
+
+# generate e2e targets
+projects_e2e = [f'{app}-e2e' for app in projects]
+
+# check affection for multisig-signer which is deployed only from develop and pull requests
+if args.event_name == 'pull_request' or 'develop' in args.github_ref:
+  if 'multisig-signer' in affected:
+    affect_app('multisig-signer', 'tools')
+
+# now parse apps that are deployed from develop but don't have previews
+if 'develop' in args.github_ref:
+  for app in ['static', 'ui-toolkit']:
+    if app in affected:
+      projects.append(app)
+
+# if ref is in format release/{env}-{app} then only {app} is deployed
+if 'release' in args.github_ref:
+  for app in main_apps:
+    if f'{args.github_ref}'.endswith(app):
+      projects = [app]
+      projects_e2e = [f'{app}-e2e']
+
+
+projects = json.dumps(projects)
+projects_e2e = json.dumps(projects_e2e)
+
+print(f'Projects: {projects}')
+print(f'Projects E2E: {projects_e2e}')
+
+print('>> Previews')
+for preview, preview_value in previews.items():
+  print(f'{preview}: {preview_value}')
+print('>> EOF Previews')
+
+
+lines_to_write = [
+  f'PREVIEW_GOVERNANCE={previews["governance"]}',
+  f'PREVIEW_EXPLORER={previews["explorer"]}',
+  f'PREVIEW_TRADING={previews["trading"]}',
+  f'PREVIEW_TOOLS={previews["tools"]}',
+  f'PROJECTS={projects}',
+  f'PROJECTS_E2E={projects_e2e}',
+]
+env_file = environ['GITHUB_ENV']
+print(f'Line to add to GITHUB_ENV file: {env_file}')
+print(lines_to_write)
+with open(env_file, 'a') as _f:
+  _f.write('\n'.join(lines_to_write))

tools/ci/define-dist-variables.py

@@ -0,0 +1,61 @@
+from argparse import ArgumentParser
+from os import environ
+
+# take input from the pipeline
+parser = ArgumentParser()
+
+# let's generate slug from bash spell for now
+parser.add_argument('--github-ref', help='current github ref')
+parser.add_argument('--app', help='current app')
+args = parser.parse_args()
+
+env_name = ''
+domain = 'vega.rocks'
+bucket_name = ''
+
+if 'release/' in args.github_ref:
+  # remove prefixing release/ and take the first string limited by - which is supposed to be name of the environment for releasing (format: release/testnet-trading)
+  env_name = args.github_ref.replace('refs/heads/release/', '').split('-')[0]
+elif 'develop' in args.github_ref:
+  env_name = 'stagnet1'
+  apps_deployed_from_develop_to_mainnet = {
+    'multisig-signer' :'tools.vega.xyz',
+    'static': 'static.vega.xyz',
+    'ui-toolkit' : 'ui.vega.rocks',
+  }
+  if args.app in apps_deployed_from_develop_to_mainnet:
+    env_name = 'mainnet'
+    bucket_name = apps_deployed_from_develop_to_mainnet[args.app]
+# endswith to avoid confusion with mirror env
+elif args.github_ref.endswith('mainnet'):
+  env_name = 'mainnet'
+
+
+other_domains_to_deploy = {
+  'mainnet': 'vega.xyz',
+  'testnet': 'fairground.wtf',
+}
+
+if env_name in other_domains_to_deploy:
+  domain = other_domains_to_deploy[env_name]
+  if not bucket_name:
+    bucket_name = f'{args.app}.{domain}'
+
+# testing envs on vega.rocks contain env_name in the url not like testnet / mainnet
+if not bucket_name:
+  bucket_name = f'{args.app}.{env_name}.{domain}'
+
+print(f'env name: {env_name}')
+print(f'domain: {domain}')
+print(f'bucket name: {bucket_name}')
+
+lines_to_write = [
+  f'ENV_NAME={env_name}',
+  f'BUCKET_NAME={bucket_name}',
+]
+
+env_file = environ['GITHUB_ENV']
+print(f'Line to add to GITHUB_ENV file: {env_file}')
+print(lines_to_write)
+with open(env_file, 'a') as _f:
+  _f.write('\n'.join(lines_to_write))