Add CI to test web app deployment and undeployment (#174)

* Add a script to deploy test records

* Add checks for ApplicationDeploymentRecord and the deployment URL

* Add a CI workflow to run the app deployment test

* Update test deployment request record config

* Add test for deployment removal

* Update test deployment request record

* Increase max retries

* Add retries when checking if URL is up or down

* Rename test script

* Run webapp deployment test CI on PR

* Remove unnecessary jq installation step from CI

* Revert "Run webapp deployment test CI on PR"

This reverts commit 01f373501a95b95cf0abc1dee8fed639ec62daf5.

* Add a step for Slack alerts on a CI failure

* Add a workflow dispatch for manual trigger

* Update workflow title and remove trigger on main

* Document deployment test setup

---------

Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

.github/workflows/test-app-deployment.yaml

@@ -0,0 +1,39 @@
+name: Test webapp deployment
+
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+
+jobs:
+  test_app_deployment:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Install dependencies
+        run: yarn
+      - name: Test webapp deployment
+        run: ./packages/deployer/test/test-webapp-deployment-undeployment.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS_WEBHOOK }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS_WEBHOOK }}

packages/deployer/test/README.md

@@ -0,0 +1,23 @@
+# deployer test
+
+Check if the live web app deployer is in a working state
+
+- Web app repo used: https://github.com/snowball-tools-platform/test-progressive-web-app (main branch)
+- Config used: [../config.yml](../config.yml)
+- The script [test-webapp-deployment-undeployment.sh](./test-webapp-deployment-undeployment.sh) performs the following:
+  - Create / update [`ApplicationRecord`](./records/application-record.yml) and [`ApplicationDeploymentRequest`](./records/application-deployment-request.yml) records with latest meta data from the repo
+  - Fetch the latest version of `deployment-test-app` from registry and increment `ApplicationRecord` version
+  - Publish the resulting `ApplicationRecord` record
+  - Set names to the record and check name resolution
+  - Publish the `ApplicationDeploymentRequest` record
+  - Check that the deployment occurs
+    - Check that a `ApplicationDeploymentRecord` is created
+    - Check that the deployment record has correct `ApplicationRecord` id
+    - Check that the URL present in deployment record is active
+  - Create and publish a [`ApplicationDeploymentRemovalRequest`](./records/application-deployment-removal-request.yml) record
+  - Check that the deployment is removed
+    - Check that a `ApplicationDeploymentRemovalRecord` is created
+    - Check that the deployment URL goes down
+- The test script is run in a GitHub CI [workflow](../../../.github/workflows/test-app-deployment.yaml) that:
+  - Is scheduled to run everyday on the default (`main`) branch or can be triggered manually
+  - Sends Slack alerts to configured channels on failure

packages/deployer/test/records/application-deployment-removal-request.yml

@@ -0,0 +1,4 @@
+record:
+  deployment: <APPLICATION_DEPLOYMENT_RECORD_ID>
+  type: ApplicationDeploymentRemovalRequest
+  version: 1.0.0

packages/deployer/test/records/application-deployment-request.yml

@@ -0,0 +1,15 @@
+record:
+  type: ApplicationDeploymentRequest
+  version: '1.0.0'
+  name: deployment-test-app@0.1.24
+  application: crn://snowballtools/applications/deployment-test-app@0.1.24
+  dns: deployment-ci-test
+  config:
+    env:
+      CERC_TEST_WEBAPP_CONFIG1: "deployment test config 1"
+      CERC_TEST_WEBAPP_CONFIG2: "deployment test config 2"
+      CERC_WEBAPP_DEBUG: 0
+  meta:
+    note: Deployment test @ Thu 11 Apr 2024 07:29:19 AM UTC
+    repository: "https://github.com/snowball-tools-platform/test-progressive-web-app"
+    repository_ref: 05819619487a0d2dbc5453b6d1ccff3044c0dd26

packages/deployer/test/records/application-record.yml

@@ -0,0 +1,8 @@
+record:
+  type: ApplicationRecord
+  version: 0.0.1
+  repository_ref: 05819619487a0d2dbc5453b6d1ccff3044c0dd26
+  repository: ["https://github.com/snowball-tools-platform/test-progressive-web-app"]
+  app_type: webapp
+  name: deployment-test-app
+  app_version: 0.1.24

packages/deployer/test/test-webapp-deployment-undeployment.sh

@@ -0,0 +1,225 @@
+#!/bin/bash
+
+# Repository URL
+REPO_URL="https://github.com/snowball-tools-platform/test-progressive-web-app"
+
+# Get the latest commit hash from the repository
+LATEST_HASH=$(git ls-remote $REPO_URL HEAD | awk '{print $1}')
+
+# Fetch the package.json file content
+# Extract version from package.json content
+package_json=$(wget -qO- "$REPO_URL/raw/$LATEST_HASH/package.json")
+PACKAGE_VERSION=$(echo "$package_json" | jq -r '.version')
+
+# Current date and time for note
+CURRENT_DATE_TIME=$(date -u)
+
+CONFIG_FILE=packages/deployer/config.yml
+REGISTRY_BOND_ID="99c0e9aec0ac1b8187faa579be3b54f93fafb6060ac1fd29170b860df605be32"
+
+# Reference: https://git.vdb.to/cerc-io/test-progressive-web-app/src/branch/main/scripts
+
+APP_NAME=deployment-test-app
+
+# Get latest version from registry and increment application-record version
+NEW_APPLICATION_VERSION=$(yarn --silent laconic -c $CONFIG_FILE cns record list --type ApplicationRecord --all --name "$APP_NAME" 2>/dev/null | jq -r -s ".[] | sort_by(.createTime) | reverse | [ .[] | select(.bondId == \"$REGISTRY_BOND_ID\") ] | .[0].attributes.version" | awk -F. -v OFS=. '{$NF += 1 ; print}')
+
+if [ -z "$NEW_APPLICATION_VERSION" ] || [ "1" == "$NEW_APPLICATION_VERSION" ]; then
+  # Set application-record version if no previous records were found
+  NEW_APPLICATION_VERSION=0.0.1
+fi
+
+# Generate application-record.yml with incremented version
+RECORD_FILE=packages/deployer/test/records/application-record.yml
+
+cat > $RECORD_FILE <<EOF
+record:
+  type: ApplicationRecord
+  version: $NEW_APPLICATION_VERSION
+  repository_ref: $LATEST_HASH
+  repository: ["$REPO_URL"]
+  app_type: webapp
+  name: $APP_NAME
+  app_version: $PACKAGE_VERSION
+EOF
+
+# Generate application-deployment-request.yml
+REQUEST_RECORD_FILE=packages/deployer/test/records/application-deployment-request.yml
+
+cat > $REQUEST_RECORD_FILE <<EOF
+record:
+  type: ApplicationDeploymentRequest
+  version: '1.0.0'
+  name: $APP_NAME@$PACKAGE_VERSION
+  application: crn://snowballtools/applications/$APP_NAME@$PACKAGE_VERSION
+  dns: deployment-ci-test
+  config:
+    env:
+      CERC_TEST_WEBAPP_CONFIG1: "deployment test config 1"
+      CERC_TEST_WEBAPP_CONFIG2: "deployment test config 2"
+      CERC_WEBAPP_DEBUG: 0
+  meta:
+    note: Deployment test @ $CURRENT_DATE_TIME
+    repository: "$REPO_URL"
+    repository_ref: $LATEST_HASH
+EOF
+
+echo "Record files generated successfully."
+
+# Publish ApplicationRecord
+RECORD_ID=$(yarn --silent laconic -c $CONFIG_FILE cns record publish --filename $RECORD_FILE | jq -r '.id')
+echo "ApplicationRecord published"
+echo $RECORD_ID
+
+# Set name to record
+REGISTRY_APP_CRN="crn://snowballtools/applications/$APP_NAME"
+
+sleep 2
+yarn --silent laconic -c $CONFIG_FILE cns name set "$REGISTRY_APP_CRN@${PACKAGE_VERSION}" "$RECORD_ID"
+sleep 2
+yarn --silent laconic -c $CONFIG_FILE cns name set "$REGISTRY_APP_CRN@${LATEST_HASH}" "$RECORD_ID"
+sleep 2
+# Set name if latest release
+yarn --silent laconic -c $CONFIG_FILE cns name set "$REGISTRY_APP_CRN" "$RECORD_ID"
+echo "$REGISTRY_APP_CRN set for ApplicationRecord"
+
+# Check if record exists for REGISTRY_APP_CRN
+APP_RECORD=$(yarn --silent laconic -c $CONFIG_FILE cns name resolve "$REGISTRY_APP_CRN" | jq '.[0]')
+if [ -z "$APP_RECORD" ] || [ "null" == "$APP_RECORD" ]; then
+  echo "No record found for $REGISTRY_APP_CRN."
+  exit 1
+fi
+
+sleep 2
+DEPLOYMENT_REQUEST_ID=$(yarn --silent laconic -c $CONFIG_FILE cns record publish --filename $REQUEST_RECORD_FILE | jq -r '.id')
+echo "ApplicationDeploymentRequest published"
+echo $DEPLOYMENT_REQUEST_ID
+
+# Deployment checks
+RETRY_INTERVAL=30
+MAX_RETRIES=20
+
+# Check that a ApplicationDeploymentRecord is published
+retry_count=0
+while true; do
+  deployment_records_response=$(yarn --silent laconic -c $CONFIG_FILE cns record list --type ApplicationDeploymentRecord --all --name "$APP_NAME" request $DEPLOYMENT_REQUEST_ID)
+  len_deployment_records=$(echo $deployment_records_response | jq 'length')
+
+  # Check if number of records returned is 0
+  if [ $len_deployment_records -eq 0 ]; then
+    # Check if retries are exhausted
+    if [ $retry_count -eq $MAX_RETRIES ]; then
+      echo "Retries exhausted"
+      echo "ApplicationDeploymentRecord for deployment request $DEPLOYMENT_REQUEST_ID not found, exiting"
+      exit 1
+    else
+      echo "ApplicationDeploymentRecord not found, retrying in $RETRY_INTERVAL sec..."
+      sleep $RETRY_INTERVAL
+      retry_count=$((retry_count+1))
+    fi
+  else
+    echo "ApplicationDeploymentRecord found"
+    break
+  fi
+done
+
+DEPLOYMENT_RECORD_ID=$(echo $deployment_records_response | jq -r '.[0].id')
+echo $DEPLOYMENT_RECORD_ID
+
+# Check if ApplicationDeploymentRecord has the correct record id
+fetched_application_record_id=$(echo $deployment_records_response | jq -r '.[0].attributes.application')
+if [ "$fetched_application_record_id" = "$RECORD_ID" ]; then
+  echo "ApplicationRecord id matched"
+else
+  echo "ApplicationRecord id does not match, expected: $RECORD_ID, received: $fetched_application_record_id"
+  exit 1
+fi
+
+# Check if the url present in ApplicationDeploymentRecord is active
+fetched_url=$(echo $deployment_records_response | jq -r '.[0].attributes.url')
+
+retry_count=0
+max_retries=10
+retry_interval=5
+while true; do
+  url_response=$(curl -s -o /dev/null -I -w "%{http_code}" $fetched_url)
+  if [ "$url_response" = "200" ]; then
+    echo "Deployment URL $fetched_url is active"
+    break
+  else
+    if [ $retry_count -eq $max_retries ]; then
+      echo "Retries exhausted"
+      echo "Deployment URL $fetched_url is not active, exiting"
+      exit 1
+    else
+      echo "Deployment URL $fetched_url is not active, received code $url_response, retrying in $retry_interval sec..."
+      sleep $retry_interval
+      retry_count=$((retry_count+1))
+    fi
+  fi
+done
+
+# Generate application-deployment-removal-request.yml
+REMOVAL_REQUEST_RECORD_FILE=packages/deployer/test/records/application-deployment-removal-request.yml
+
+cat > $REMOVAL_REQUEST_RECORD_FILE <<EOF
+record:
+  deployment: $DEPLOYMENT_RECORD_ID
+  type: ApplicationDeploymentRemovalRequest
+  version: 1.0.0
+EOF
+
+sleep 2
+REMOVAL_REQUEST_ID=$(yarn --silent laconic -c $CONFIG_FILE cns record publish --filename $REMOVAL_REQUEST_RECORD_FILE | jq -r '.id')
+echo "ApplicationDeploymentRemovalRequest published"
+echo $REMOVAL_REQUEST_ID
+
+# Check that an ApplicationDeploymentRemovalRecord is published
+retry_count=0
+while true; do
+  removal_records_response=$(yarn --silent laconic -c $CONFIG_FILE cns record list --type ApplicationDeploymentRemovalRecord --all request $REMOVAL_REQUEST_ID)
+  len_removal_records=$(echo $removal_records_response | jq 'length')
+
+  # Check if number of records returned is 0
+  if [ $len_removal_records -eq 0 ]; then
+    # Check if retries are exhausted
+    if [ $retry_count -eq $MAX_RETRIES ]; then
+      echo "Retries exhausted"
+      echo "ApplicationDeploymentRemovalRecord for deployment removal request $REMOVAL_REQUEST_ID not found"
+      exit 1
+    else
+      echo "ApplicationDeploymentRemovalRecord not found, retrying in $RETRY_INTERVAL sec..."
+      sleep $RETRY_INTERVAL
+      retry_count=$((retry_count+1))
+    fi
+  else
+    echo "ApplicationDeploymentRemovalRecord found"
+    REMOVAL_RECORD_ID=$(echo $removal_records_response | jq -r '.[0].id')
+    echo $REMOVAL_RECORD_ID
+    break
+  fi
+done
+
+# Check if the application url is down after deployment removal
+retry_count=0
+max_retries=10
+retry_interval=5
+while true; do
+  url_response=$(curl -s -o /dev/null -I -w "%{http_code}" $fetched_url)
+  if [ "$url_response" = "404" ]; then
+    echo "Deployment URL $fetched_url is down"
+    break
+  else
+    if [ $retry_count -eq $max_retries ]; then
+      echo "Retries exhausted"
+      echo "Deployment URL $fetched_url is still active, exiting"
+      exit 1
+    else
+      echo "Deployment URL $fetched_url is still active, received code $url_response, retrying in $retry_interval sec..."
+      sleep $retry_interval
+      retry_count=$((retry_count+1))
+    fi
+  fi
+done
+
+echo "Test successful"