From 90dda208911811cc9c56d1754f4f558ed23f6034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20M=C5=82odzikowski?= Date: Wed, 26 Jul 2023 16:08:57 +0200 Subject: [PATCH] feat(ci): migrate ci-cd trigger to python (#4330) --- .github/workflows/ci-cd-trigger.yml | 136 ++++------------------------ .github/workflows/publish-dist.yml | 51 ++--------- tools/ci/check-affected.py | 110 ++++++++++++++++++++++ tools/ci/define-dist-variables.py | 61 +++++++++++++ 4 files changed, 193 insertions(+), 165 deletions(-) create mode 100644 tools/ci/check-affected.py create mode 100644 tools/ci/define-dist-variables.py diff --git a/.github/workflows/ci-cd-trigger.yml b/.github/workflows/ci-cd-trigger.yml index 851bfcaf6..53afde7e9 100644 --- a/.github/workflows/ci-cd-trigger.yml +++ b/.github/workflows/ci-cd-trigger.yml @@ -81,6 +81,22 @@ jobs: with: main-branch-name: develop + # See affected apps + - name: See affected apps + run: | + branch_slug="$(echo '${{ github.head_ref || github.ref_name }}' | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )" + python3 tools/ci/check-affected.py --github-ref="${{ github.ref }}" --branch-slug="$branch_slug" --event-name="${{ github.event_name }}" + + - name: Verify script result + run: | + echo "Check outputs from script" + echo "projects: ${{ env.PROJECTS }}" + echo "projects-e2e: ${{ env.PROJECTS_E2E }}" + echo "preview_governance: ${{ env.PREVIEW_GOVERNANCE }}" + echo "preview_trading: ${{ env.PREVIEW_TRADING }}" + echo "preview_explorer: ${{ env.PREVIEW_EXPLORER }}" + echo "preview_tools: ${{ env.PREVIEW_TOOLS }}" + - name: Check formatting run: yarn nx format:check @@ -96,126 +112,6 @@ jobs: - name: Build affected run: yarn nx affected:build || (yarn install && yarn nx affected:build) - # See affected apps - - name: See affected apps - run: | - affected="$(yarn nx print-affected --base=${{ env.NX_BASE }} --head=${{ env.NX_HEAD }} --select=projects)" - branch_slug="$(echo '${{ github.head_ref || github.ref_name }}' | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | cut -c 1-50 )" - - echo ">>>> debug" - echo "NX_BASE: ${{ env.NX_BASE }}" - echo "NX_HEAD: ${{ env.NX_HEAD }}" - echo "Affected: ${affected}" - echo "Branch slug: ${branch_slug}" - echo "Current ref: ${{ github.ref }}" - echo ">>>> eof debug" - - projects_array=() - - preview_governance="not deployed" - preview_trading="not deployed" - preview_explorer="not deployed" - preview_tools="not deployed" - - # parse if affected is any of three main applications, if none - use all of them - if echo "$affected" | grep -q governance; then - echo "Governance is affected" - projects_array+=("governance") - preview_governance=$(printf "https://%s.%s.vega.rocks" "governance" "$branch_slug") - fi - if echo "$affected" | grep -q trading; then - echo "Trading is affected" - projects_array+=("trading") - preview_trading=$(printf "https://%s.%s.vega.rocks" "trading" "$branch_slug") - fi - if echo "$affected" | grep -q explorer; then - echo "Explorer is affected" - projects_array+=("explorer") - preview_explorer=$(printf "https://%s.%s.vega.rocks" "explorer" "$branch_slug") - fi - if [[ ${#projects_array[@]} -eq 0 ]]; then - projects_array=("governance" "trading" "explorer") - preview_governance=$(printf "https://%s.%s.vega.rocks" "governance" "$branch_slug") - preview_trading=$(printf "https://%s.%s.vega.rocks" "trading" "$branch_slug") - preview_explorer=$(printf "https://%s.%s.vega.rocks" "explorer" "$branch_slug") - fi - - # applications parsed before this loop are applicable for running e2e-tests - projects_e2e_array=() - for project in "${projects_array[@]}"; do - projects_e2e_array+=("${project}-e2e") - done - # all applications below this loop are not applicable for running e2e-test - - # check if pull request event to deploy tools - if [[ "${{ github.event_name }}" = "pull_request" ]]; then - if echo "$affected" | grep -q multisig-signer; then - echo "Tools are affected" - echo "Deploying tools on preview" - preview_tools=$(printf "https://%s.%s.vega.rocks" "tools" "$branch_slug") - - projects_array+=("multisig-signer") - fi - # those apps deploy only from develop to mainnet - elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then - if echo "$affected" | grep -q multisig-signer; then - echo "Tools are affected" - echo "Deploying tools on s3" - - projects_array+=("multisig-signer") - fi - if echo "$affected" | grep -q static; then - echo "static is affected" - echo "Deploying static on s3" - - projects_array+=("static") - fi - if echo "$affected" | grep -q ui-toolkit; then - echo "ui-toolkit is affected" - echo "Deploying ui-toolkit on s3" - - projects_array+=("ui-toolkit") - fi - fi - - # if branch starts with release/ and ends with trading / governance or explorer - overwrite the array of affected projects with fixed single application - if [[ "${{ github.ref }}" == *release* ]]; then - echo ">> This is a relase branch" - case "${{ github.ref }}" in - *trading) - echo ">> Only trading will be deployed" - projects_array=(trading) - projects_e2e_array=(trading) - ;; - *governance) - echo ">> Only governance will be deployed" - projects_array=(governance) - projects_e2e_array=(governance) - ;; - *explorer) - echo ">> Only explorer will be deployed" - projects_array=(explorer) - projects_e2e_array=(explorer) - ;; - *) - echo ">> All apps will be deployed" - ;; - esac - fi - - echo "Projects: ${projects_array[@]}" - echo "Projects E2E: ${projects_e2e_array[@]}" - projects_json=$(jq -M --compact-output --null-input '$ARGS.positional' --args -- "${projects_array[@]}") - projects_e2e_json=$(jq -M --compact-output --null-input '$ARGS.positional' --args -- "${projects_e2e_array[@]}") - - echo PROJECTS_E2E=$projects_e2e_json >> $GITHUB_ENV - echo PROJECTS=$projects_json >> $GITHUB_ENV - - echo PREVIEW_GOVERNANCE=$preview_governance >> $GITHUB_ENV - echo PREVIEW_TRADING=$preview_trading >> $GITHUB_ENV - echo PREVIEW_EXPLORER=$preview_explorer >> $GITHUB_ENV - echo PREVIEW_TOOLS=$preview_tools >> $GITHUB_ENV - outputs: projects: ${{ env.PROJECTS }} projects-e2e: ${{ env.PROJECTS_E2E }} diff --git a/.github/workflows/publish-dist.yml b/.github/workflows/publish-dist.yml index 2f0d6be5a..c6c268ace 100644 --- a/.github/workflows/publish-dist.yml +++ b/.github/workflows/publish-dist.yml @@ -97,52 +97,13 @@ jobs: - name: Define dist variables if: ${{ github.event_name == 'push' }} run: | - envName='' - domain="vega.rocks" - bucketName='' + python3 tools/ci/define-dist-variables.py --github-ref="${{ github.ref }}" --app="${{ matrix.app }}" - if [[ "${{ github.ref }}" =~ .*release/.* ]]; then - # remove prefixing release/ and take the first string limited by - which is supposed to be name of the environment for releasing (format: release/testnet-trading) - envName="$(echo ${{ github.ref }} | sed -e "s|refs/heads/release/||" | cut -d '-' -f 1 )" - elif [[ "${{ github.ref }}" =~ .*develop$ ]]; then - envName="stagnet1" - if [[ "${{ matrix.app }}" = "multisig-signer" ]]; then - envName="mainnet" - bucketName="tools.vega.xyz" - fi - if [[ "${{ matrix.app }}" = "static" ]]; then - envName="mainnet" - bucketName="static.vega.xyz" - fi - if [[ "${{ matrix.app }}" = "ui-toolkit" ]]; then - envName="mainnet" - bucketName="ui.vega.rocks" - fi - elif [[ "${{ github.ref }}" =~ .*mainnet$ ]]; then - envName="mainnet" - fi - - if [[ "${envName}" = "mainnet" ]]; then - domain="vega.xyz" - if [[ -z "${bucketName}" ]]; then - bucketName="${{ matrix.app }}.${domain}" - fi - elif [[ "${envName}" = "testnet" ]]; then - domain="fairground.wtf" - if [[ -z "${bucketName}" ]]; then - bucketName="${{ matrix.app }}.${domain}" - fi - fi - - if [[ -z "${bucketName}" ]]; then - bucketName="${{ matrix.app }}.${envName}.${domain}" - fi - - echo "bucket name: ${bucketName}" - echo "env name: ${envName}" - - echo BUCKET_NAME=${bucketName} >> $GITHUB_ENV - echo ENV_NAME=${envName} >> $GITHUB_ENV + - name: Verify script result + if: ${{ github.event_name == 'push' }} + run: | + echo "BUCKET_NAME=${{ env.BUCKET_NAME }}" + echo "ENV_NAME=${{ env.ENV_NAME }}" - name: Build local dist run: | diff --git a/tools/ci/check-affected.py b/tools/ci/check-affected.py new file mode 100644 index 000000000..e3ee84ff9 --- /dev/null +++ b/tools/ci/check-affected.py @@ -0,0 +1,110 @@ +from os import environ +from subprocess import check_output +from argparse import ArgumentParser +import json + +projects = [] +projects_e2e = [] + +previews = { + 'governance': 'not deployed', + 'explorer': 'not deployed', + 'trading': 'not deployed', + 'tools': 'not deployed', +} + +main_apps = ['governance', 'explorer', 'trading'] + +preview_governance="not deployed" +preview_trading="not deployed" +preview_explorer="not deployed" +preview_tools="not deployed" + +# take input from the pipeline +parser = ArgumentParser() + +# let's generate slug from bash spell for now +parser.add_argument('--branch-slug', help='slug of branch') +parser.add_argument('--github-ref', help='current github ref') +parser.add_argument('--event-name', help='name of event in CI') +args = parser.parse_args() + +# run yarn affected command +affected=check_output(f'yarn nx print-affected --base={environ["NX_BASE"]} --head={environ["NX_HEAD"]} --select=projects'.split()).decode('utf-8') + + +# print useful information +print(">>>> debug") +print(f"NX_BASE: { environ['NX_BASE'] }") +print(f"NX_HEAD: { environ['NX_HEAD'] }") +print(f"Branch slug: {args.branch_slug}") +print(f"Current ref: {args.github_ref}") +print(">> Affected output") +print(affected) +print(">>>> eof debug") + +# define affection actions -> add to projects arrays and generate preview link +def affect_app(app, preview_name=None): + print(f"{app} is affected") + projects.append(app) + if not preview_name: + preview_name=app + previews[app] = f'https://{preview_name}.{args.branch_slug}.vega.rocks' + +# check appearance in the affected string for main apps +for app in main_apps: + if app in affected: + affect_app(app) + +# if non of main apps is affected - test all of them +if not projects: + for app in main_apps: + affect_app(app) + +# generate e2e targets +projects_e2e = [f'{app}-e2e' for app in projects] + +# check affection for multisig-signer which is deployed only from develop and pull requests +if args.event_name == 'pull_request' or 'develop' in args.github_ref: + if 'multisig-signer' in affected: + affect_app('multisig-signer', 'tools') + +# now parse apps that are deployed from develop but don't have previews +if 'develop' in args.github_ref: + for app in ['static', 'ui-toolkit']: + if app in affected: + projects.append(app) + +# if ref is in format release/{env}-{app} then only {app} is deployed +if 'release' in args.github_ref: + for app in main_apps: + if f'{args.github_ref}'.endswith(app): + projects = [app] + projects_e2e = [f'{app}-e2e'] + + +projects = json.dumps(projects) +projects_e2e = json.dumps(projects_e2e) + +print(f'Projects: {projects}') +print(f'Projects E2E: {projects_e2e}') + +print('>> Previews') +for preview, preview_value in previews.items(): + print(f'{preview}: {preview_value}') +print('>> EOF Previews') + + +lines_to_write = [ + f'PREVIEW_GOVERNANCE={previews["governance"]}', + f'PREVIEW_EXPLORER={previews["explorer"]}', + f'PREVIEW_TRADING={previews["trading"]}', + f'PREVIEW_TOOLS={previews["tools"]}', + f'PROJECTS={projects}', + f'PROJECTS_E2E={projects_e2e}', +] +env_file = environ['GITHUB_ENV'] +print(f'Line to add to GITHUB_ENV file: {env_file}') +print(lines_to_write) +with open(env_file, 'a') as _f: + _f.write('\n'.join(lines_to_write)) diff --git a/tools/ci/define-dist-variables.py b/tools/ci/define-dist-variables.py new file mode 100644 index 000000000..dbf6ac778 --- /dev/null +++ b/tools/ci/define-dist-variables.py @@ -0,0 +1,61 @@ +from argparse import ArgumentParser +from os import environ + +# take input from the pipeline +parser = ArgumentParser() + +# let's generate slug from bash spell for now +parser.add_argument('--github-ref', help='current github ref') +parser.add_argument('--app', help='current app') +args = parser.parse_args() + +env_name = '' +domain = 'vega.rocks' +bucket_name = '' + +if 'release/' in args.github_ref: + # remove prefixing release/ and take the first string limited by - which is supposed to be name of the environment for releasing (format: release/testnet-trading) + env_name = args.github_ref.replace('refs/heads/release/', '').split('-')[0] +elif 'develop' in args.github_ref: + env_name = 'stagnet1' + apps_deployed_from_develop_to_mainnet = { + 'multisig-signer' :'tools.vega.xyz', + 'static': 'static.vega.xyz', + 'ui-toolkit' : 'ui.vega.rocks', + } + if args.app in apps_deployed_from_develop_to_mainnet: + env_name = 'mainnet' + bucket_name = apps_deployed_from_develop_to_mainnet[args.app] +# endswith to avoid confusion with mirror env +elif args.github_ref.endswith('mainnet'): + env_name = 'mainnet' + + +other_domains_to_deploy = { + 'mainnet': 'vega.xyz', + 'testnet': 'fairground.wtf', +} + +if env_name in other_domains_to_deploy: + domain = other_domains_to_deploy[env_name] + if not bucket_name: + bucket_name = f'{args.app}.{domain}' + +# testing envs on vega.rocks contain env_name in the url not like testnet / mainnet +if not bucket_name: + bucket_name = f'{args.app}.{env_name}.{domain}' + +print(f'env name: {env_name}') +print(f'domain: {domain}') +print(f'bucket name: {bucket_name}') + +lines_to_write = [ + f'ENV_NAME={env_name}', + f'BUCKET_NAME={bucket_name}', +] + +env_file = environ['GITHUB_ENV'] +print(f'Line to add to GITHUB_ENV file: {env_file}') +print(lines_to_write) +with open(env_file, 'a') as _f: + _f.write('\n'.join(lines_to_write))