From c8c8ce9788242672799e37cc11a4ff47474863ab Mon Sep 17 00:00:00 2001 From: zramsay Date: Mon, 25 Sep 2023 15:28:36 -0400 Subject: [PATCH 1/2] gitea test --- .gitea/workflows/gitea.yml | 49 +++++++++++++++++++++++++++++++++++++ .github/workflows/gitea.yml | 29 ++++++++++++++++++++++ README.md | 2 ++ app/build_npms.py | 4 +-- tests/gitea/test.sh | 33 +++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 .gitea/workflows/gitea.yml create mode 100644 .github/workflows/gitea.yml create mode 100755 tests/gitea/test.sh diff --git a/.gitea/workflows/gitea.yml b/.gitea/workflows/gitea.yml new file mode 100644 index 00000000..93d2ff74 --- /dev/null +++ b/.gitea/workflows/gitea.yml @@ -0,0 +1,49 @@ +name: Test Gitea + +on: + pull_request: + branches: '*' + push: + branches: + - main + - zach-test + - new-gitea-test + +# Needed until we can incorporate docker startup into the executor container +env: + DOCKER_HOST: unix:///var/run/dind.sock + +jobs: + test: + name: "Run Gitea package registry test" + runs-on: ubuntu-latest + steps: + - name: 'Update' + run: apt-get update + - name: 'Setup jq' + run: apt-get install jq -y + - name: 'Check jq' + run: | + which jq + jq --version + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: Start dockerd # Also needed until we can incorporate into the executor + run: | + dockerd -H $DOCKER_HOST --userland-proxy=false & + sleep 5 + - name: "Run gitea tests" + run: ./tests/gitea/test.sh + diff --git a/.github/workflows/gitea.yml b/.github/workflows/gitea.yml new file mode 100644 index 00000000..895b1b9d --- /dev/null +++ b/.github/workflows/gitea.yml @@ -0,0 +1,29 @@ +name: Gitea Test + +on: + pull_request: + branches: '*' + push: + branches: '*' + +jobs: + test: + name: "Run basic test suite" + runs-on: ubuntu-latest + steps: + - name: "Clone project repository" + uses: actions/checkout@v3 + - name: "Install Python" + uses: actions/setup-python@v4 + with: + python-version: '3.8' + - name: "Print Python version" + run: python3 --version + - name: "Install shiv" + run: pip install shiv + - name: "Generate build version file" + run: ./scripts/create_build_tag_file.sh + - name: "Build local shiv package" + run: ./scripts/build_shiv_package.sh + - name: "Run gitea tests" + run: ./tests/gitea/test.sh diff --git a/README.md b/README.md index a888895e..b7e75aaf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Stack Orchestrator +test + Stack Orchestrator allows building and deployment of a Laconic Stack on a single machine with minimial prerequisites. It is a Python3 CLI tool that runs on any OS with Python3 and Docker. The following diagram summarizes the relevant repositories in the Laconic Stack - and the relationship to Stack Orchestrator. ![The Stack](/docs/images/laconic-stack.png) diff --git a/app/build_npms.py b/app/build_npms.py index 2ffbea1b..aa862a52 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -132,8 +132,8 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): try: docker.run(builder_js_image_name, remove=True, - interactive=True, - tty=True, + interactive=False, + tty=False, user=f"{os.getuid()}:{os.getgid()}", envs=envs, # TODO: detect this host name in npm_registry_url rather than hard-wiring it diff --git a/tests/gitea/test.sh b/tests/gitea/test.sh new file mode 100755 index 00000000..c9c1a681 --- /dev/null +++ b/tests/gitea/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +# Dump environment variables for debugging +echo "Environment variables:" +env +echo "Running gitea test" +# Bit of a hack, test the most recent package +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) +# Set a non-default repo dir +export CERC_REPO_BASE_DIR=~/stack-orchestrator-gitea-test +echo "Testing this package: $TEST_TARGET_SO" +echo "Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "Version reported is: ${reported_version_string}" +echo "Cloning repositories into: $CERC_REPO_BASE_DIR" +rm -rf $CERC_REPO_BASE_DIR +mkdir -p $CERC_REPO_BASE_DIR + +$TEST_TARGET_SO --stack build-support build-containers +$TEST_TARGET_SO --stack package-registry setup-repositories +$TEST_TARGET_SO --stack package-registry build-containers +output=$($TEST_TARGET_SO --stack package-registry deploy-system up) +token=$(echo $output | grep -o 'export CERC_NPM_AUTH_TOKEN=[^ ]*' | cut -d '=' -f 2) +export CERC_NPM_AUTH_TOKEN=$token +$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories +$TEST_TARGET_SO --stack fixturenet-laconicd build-npms + +# Clean up +$TEST_TARGET_SO --stack package-registry deploy-system down +echo "Test passed" -- 2.45.2 From 067c61832197bcc0681c770caf88b515baa949c6 Mon Sep 17 00:00:00 2001 From: zramsay Date: Wed, 27 Sep 2023 17:35:27 -0400 Subject: [PATCH 2/2] squash me debug --- .gitea/workflows/gitea.yml | 4 +- README.md | 2 - .../cerc-builder-js/build-npm-package.sh | 39 ++++++++++++++----- tests/gitea/test.sh | 9 ++++- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/.gitea/workflows/gitea.yml b/.gitea/workflows/gitea.yml index 93d2ff74..5515812b 100644 --- a/.gitea/workflows/gitea.yml +++ b/.gitea/workflows/gitea.yml @@ -45,5 +45,7 @@ jobs: dockerd -H $DOCKER_HOST --userland-proxy=false & sleep 5 - name: "Run gitea tests" - run: ./tests/gitea/test.sh + run: | + export CERC_SCRIPT_DEBUG=true & + ./tests/gitea/test.sh diff --git a/README.md b/README.md index b7e75aaf..a888895e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Stack Orchestrator -test - Stack Orchestrator allows building and deployment of a Laconic Stack on a single machine with minimial prerequisites. It is a Python3 CLI tool that runs on any OS with Python3 and Docker. The following diagram summarizes the relevant repositories in the Laconic Stack - and the relationship to Stack Orchestrator. ![The Stack](/docs/images/laconic-stack.png) diff --git a/app/data/container-build/cerc-builder-js/build-npm-package.sh b/app/data/container-build/cerc-builder-js/build-npm-package.sh index 375712e5..499781bb 100755 --- a/app/data/container-build/cerc-builder-js/build-npm-package.sh +++ b/app/data/container-build/cerc-builder-js/build-npm-package.sh @@ -1,9 +1,25 @@ #!/bin/bash # Usage: build-npm-package.sh # Note: supply the registry auth token in CERC_NPM_AUTH_TOKEN +# + + if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then set -x fi + +echo "GITEA ACTIONS 1 is: $GITEA_ACTIONS" +echo "GITEA ACTIONS 2 is: ${GITEA_ACTIONS}" + +echo "token 1 is: $CERC_NPM_AUTH_TOKEN" +echo "token 2 is: ${CERC_NPM_AUTH_TOKEN}" + +if [[ "$GITEA_ACTIONS" = "true" ]]; then + echo "WORKS" +else + echo "WTF" +fi + if ! [[ $# -eq 1 || $# -eq 2 ]]; then echo "Illegal number of parameters" >&2 exit 1 @@ -28,17 +44,20 @@ npm config set @lirewine:registry ${local_npm_registry_url} local_npm_registry_url_fixed=$( echo ${local_npm_registry_url} | sed -e 's/^http[s]\{0,1\}://') npm config set -- ${local_npm_registry_url_fixed}:_authToken ${CERC_NPM_AUTH_TOKEN} # First check if the version of this package we're trying to build already exists in the registry +# but this line: "jq -r .data.dist.tarball" fails only in gitea, so skip this block +if [[ "${GITEA_ACTIONS}" != "true" ]]; then package_exists=$( yarn info --json ${package_name}@${package_publish_version} 2>/dev/null | jq -r .data.dist.tarball ) -if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then - echo "${package_publish_version} of ${package_name} already exists in the registry" - if [[ ${CERC_FORCE_REBUILD} == "true" ]]; then - # Attempt to unpublish the existing package - echo "NOTE: unpublishing existing package version since force rebuild is enabled" - npm unpublish --force ${package_name}@${package_publish_version} - else - echo "skipping build since target version already exists" - exit 0 - fi + if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then + echo "${package_publish_version} of ${package_name} already exists in the registry" + if [[ ${CERC_FORCE_REBUILD} == "true" ]]; then + # Attempt to unpublish the existing package + echo "NOTE: unpublishing existing package version since force rebuild is enabled" + npm unpublish --force ${package_name}@${package_publish_version} + else + echo "skipping build since target version already exists" + exit 0 + fi + fi fi echo "Build and publish ${package_name} version ${package_publish_version}" yarn install diff --git a/tests/gitea/test.sh b/tests/gitea/test.sh index c9c1a681..747986e9 100755 --- a/tests/gitea/test.sh +++ b/tests/gitea/test.sh @@ -1,12 +1,17 @@ #!/usr/bin/env bash set -e + +export CERC_SCRIPT_DEBUG=true if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi # Dump environment variables for debugging echo "Environment variables:" env + +echo "holla 1 $GITEA_ACTIONS" echo "Running gitea test" + # Bit of a hack, test the most recent package TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) # Set a non-default repo dir @@ -25,8 +30,10 @@ $TEST_TARGET_SO --stack package-registry build-containers output=$($TEST_TARGET_SO --stack package-registry deploy-system up) token=$(echo $output | grep -o 'export CERC_NPM_AUTH_TOKEN=[^ ]*' | cut -d '=' -f 2) export CERC_NPM_AUTH_TOKEN=$token +export GITEA_ACTIONS=true +echo "holla 2 $GITEA_ACTIONS" $TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories -$TEST_TARGET_SO --stack fixturenet-laconicd build-npms +$TEST_TARGET_SO --stack fixturenet-laconicd --verbose --debug build-npms --include laconic-sdk # Clean up $TEST_TARGET_SO --stack package-registry deploy-system down -- 2.45.2