From 2ba3ffac8481dfe7a5dc62d3ce25610af93e1183 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 4 Jan 2023 07:39:01 -0700 Subject: [PATCH 1/6] Add lirewine scope --- container-build/cerc-builder-js/build-npm-package.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/container-build/cerc-builder-js/build-npm-package.sh b/container-build/cerc-builder-js/build-npm-package.sh index 9e5094b..6161e23 100755 --- a/container-build/cerc-builder-js/build-npm-package.sh +++ b/container-build/cerc-builder-js/build-npm-package.sh @@ -14,6 +14,7 @@ if [[ -z "${NPM_AUTH_TOKEN}" ]]; then fi local_npm_registry_url=$1 package_publish_version=$2 +npm config set @lirewine:registry ${local_npm_registry_url} npm config set @cerc-io:registry ${local_npm_registry_url} npm config set -- ${local_npm_registry_url}:_authToken ${NPM_AUTH_TOKEN} echo "Build and publish version ${package_publish_version}" From ee1ecbd21759be4e930d373d7070426436303331 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 4 Jan 2023 16:16:40 -0700 Subject: [PATCH 2/6] Initial implementation of npm-build --- app/build_npms.py | 93 +++++++++++++++++++++++++++++++++++ app/data/npm-package-list.txt | 1 + cli.py | 2 + 3 files changed, 96 insertions(+) create mode 100644 app/build_npms.py create mode 100644 app/data/npm-package-list.txt diff --git a/app/build_npms.py b/app/build_npms.py new file mode 100644 index 0000000..81b215a --- /dev/null +++ b/app/build_npms.py @@ -0,0 +1,93 @@ +# Copyright © 2022 Cerc + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds or pulls containers for the system components + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +import os +from decouple import config +import subprocess +import click +import pkg_resources +from python_on_whales import docker +from .util import include_exclude_check + +@click.command() +@click.option('--include', help="only build these packages") +@click.option('--exclude', help="don\'t build these packages") +@click.pass_context +def command(ctx, include, exclude): + '''build the set of npm packages required for a complete stack''' + + quiet = ctx.obj.quiet + verbose = ctx.obj.verbose + dry_run = ctx.obj.dry_run + local_stack = ctx.obj.local_stack + + if local_stack: + dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")] + print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}') + else: + dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc")) + + if not quiet: + print(f'Dev Root is: {dev_root_path}') + + if not os.path.isdir(dev_root_path): + print('Dev root directory doesn\'t exist, creating') + + with pkg_resources.resource_stream(__name__, "data/npm-package-list.txt") as package_list_file: + packages = package_list_file.read().decode().splitlines() + + if verbose: + print(f'Packages: {packages}') + + # To build a package we need to run the cerc/build-js container with a bind mount from the project directory + # and use docker run to execute the build script inside that container + # docker.run("cerc/builder-js", ["ls", "/host"], volumes=[("", "/workspace")]) + # sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ + # build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.0-beta.1' + # NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 + # Set uid/gid + + def build_package(package): + if not quiet: + print(f"Building: {package}") + repo_dir = package + repo_full_path = os.path.join(dev_root_path, repo_dir) + build_command = ["sh", "-c", "'cd /workspace && build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.0-beta.1'"] + if not dry_run: + if verbose: + print(f"Executing: {build_command}") + build_result = docker.run("cerc/builder-js", + remove=True, + interactive=True, + tty=True, + envs={"NPM_AUTH_TOKEN": "6613572a28ebebaee20ccd90064251fa8c2b94f6"}, + add_hosts=[("host.docker.internal", "host-gateway")], + volumes=[(repo_full_path, "/workspace")]) + # TODO: check result in build_result.returncode + print(f"Result is: {build_result}") + else: + print("Skipped") + + for package in packages: + if include_exclude_check(package, include, exclude): + build_package(package) + else: + if verbose: + print(f"Excluding: {package}") diff --git a/app/data/npm-package-list.txt b/app/data/npm-package-list.txt new file mode 100644 index 0000000..6e0ead2 --- /dev/null +++ b/app/data/npm-package-list.txt @@ -0,0 +1 @@ +gem diff --git a/cli.py b/cli.py index 87de406..b3890dd 100644 --- a/cli.py +++ b/cli.py @@ -17,6 +17,7 @@ import click from app import setup_repositories from app import build_containers +from app import build_npms from app import deploy_system CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -44,4 +45,5 @@ def cli(ctx, quiet, verbose, dry_run, local_stack): cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") +cli.add_command(build_npms.command, "build-npms") cli.add_command(deploy_system.command, "deploy-system") From 073d94e2886333a0a67328f5a5f2bd5481135c09 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 4 Jan 2023 20:43:13 -0700 Subject: [PATCH 3/6] Brand token env var --- app/build_npms.py | 18 ++++++------------ app/data/npm-package-list.txt | 3 +++ container-build/cerc-builder-js/README.md | 2 +- .../cerc-builder-js/build-npm-package.sh | 8 ++++---- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/app/build_npms.py b/app/build_npms.py index 81b215a..812fb5c 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -20,7 +20,6 @@ import os from decouple import config -import subprocess import click import pkg_resources from python_on_whales import docker @@ -56,20 +55,12 @@ def command(ctx, include, exclude): if verbose: print(f'Packages: {packages}') - # To build a package we need to run the cerc/build-js container with a bind mount from the project directory - # and use docker run to execute the build script inside that container - # docker.run("cerc/builder-js", ["ls", "/host"], volumes=[("", "/workspace")]) - # sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ - # build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.0-beta.1' - # NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 - # Set uid/gid - def build_package(package): if not quiet: print(f"Building: {package}") repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) - build_command = ["sh", "-c", "'cd /workspace && build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.0-beta.1'"] + build_command = ["sh", "-c", "cd /workspace && build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.15"] if not dry_run: if verbose: print(f"Executing: {build_command}") @@ -77,9 +68,12 @@ def command(ctx, include, exclude): remove=True, interactive=True, tty=True, - envs={"NPM_AUTH_TOKEN": "6613572a28ebebaee20ccd90064251fa8c2b94f6"}, + user=f"{os.getuid()}:{os.getgid()}", + envs={"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"]}, add_hosts=[("host.docker.internal", "host-gateway")], - volumes=[(repo_full_path, "/workspace")]) + volumes=[(repo_full_path, "/workspace")], + command=build_command + ) # TODO: check result in build_result.returncode print(f"Result is: {build_result}") else: diff --git a/app/data/npm-package-list.txt b/app/data/npm-package-list.txt index 6e0ead2..3adedb5 100644 --- a/app/data/npm-package-list.txt +++ b/app/data/npm-package-list.txt @@ -1 +1,4 @@ gem +laconic-sdk +debug +laconic-registry-cli diff --git a/container-build/cerc-builder-js/README.md b/container-build/cerc-builder-js/README.md index 865d39e..da784ef 100644 --- a/container-build/cerc-builder-js/README.md +++ b/container-build/cerc-builder-js/README.md @@ -12,6 +12,6 @@ it is possible to build packages manually by invoking `docker run` , for example ``` $ docker run --rm -it --add-host host.docker.internal:host-gateway \ -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js \ - sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ + sh -c 'cd /workspace && CERC_NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.8' ``` diff --git a/container-build/cerc-builder-js/build-npm-package.sh b/container-build/cerc-builder-js/build-npm-package.sh index 6161e23..88cc974 100755 --- a/container-build/cerc-builder-js/build-npm-package.sh +++ b/container-build/cerc-builder-js/build-npm-package.sh @@ -1,6 +1,6 @@ #!/bin/bash # Usage: build-npm-package.sh -# Note: supply the registry auth token in NPM_AUTH_TOKEN +# Note: supply the registry auth token in CERC_NPM_AUTH_TOKEN if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi @@ -8,15 +8,15 @@ if [[ $# -ne 2 ]]; then echo "Illegal number of parameters" >&2 exit 1 fi -if [[ -z "${NPM_AUTH_TOKEN}" ]]; then - echo "NPM_AUTH_TOKEN is not set" >&2 +if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then + echo "CERC_NPM_AUTH_TOKEN is not set" >&2 exit 1 fi local_npm_registry_url=$1 package_publish_version=$2 npm config set @lirewine:registry ${local_npm_registry_url} npm config set @cerc-io:registry ${local_npm_registry_url} -npm config set -- ${local_npm_registry_url}:_authToken ${NPM_AUTH_TOKEN} +npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN} echo "Build and publish version ${package_publish_version}" yarn install yarn build From 68c6eaf0b43eae1ecc43c8f82e592696b11b2c74 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 4 Jan 2023 20:50:23 -0700 Subject: [PATCH 4/6] Use the right script --- app/build_npms.py | 3 ++- .../cerc-builder-js/build-npm-package-local-dependencies.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/build_npms.py b/app/build_npms.py index 812fb5c..4dbed55 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -60,7 +60,8 @@ def command(ctx, include, exclude): print(f"Building: {package}") repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) - build_command = ["sh", "-c", "cd /workspace && build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.15"] + # TODO: make the npm registry url configurable. + build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.16"] if not dry_run: if verbose: print(f"Executing: {build_command}") diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh index cb2b52a..6449bb8 100755 --- a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -9,8 +9,8 @@ if [[ $# -ne 2 ]]; then echo "Illegal number of parameters" >&2 exit 1 fi -if [[ -z "${NPM_AUTH_TOKEN}" ]]; then - echo "NPM_AUTH_TOKEN is not set" >&2 +if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then + echo "CERC_NPM_AUTH_TOKEN is not set" >&2 exit 1 fi local_npm_registry_url=$1 @@ -19,7 +19,7 @@ package_publish_version=$2 npm_scope_for_local="@cerc-io" # We need to configure the local registry npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url} -npm config set -- ${local_npm_registry_url}:_authToken ${NPM_AUTH_TOKEN} +npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN} # Find the set of dependencies from the specified scope mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]') echo "Fixing up dependencies" From 765db5937d5d859b5a38fca2158500f321f96543 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 4 Jan 2023 21:23:17 -0700 Subject: [PATCH 5/6] Default to from-repo package version --- app/build_npms.py | 4 ++-- .../build-npm-package-local-dependencies.sh | 2 +- container-build/cerc-builder-js/build-npm-package.sh | 8 ++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/build_npms.py b/app/build_npms.py index 4dbed55..f91fcf9 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -61,7 +61,7 @@ def command(ctx, include, exclude): repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) # TODO: make the npm registry url configurable. - build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.0.16"] + build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/"] if not dry_run: if verbose: print(f"Executing: {build_command}") @@ -70,7 +70,7 @@ def command(ctx, include, exclude): interactive=True, tty=True, user=f"{os.getuid()}:{os.getgid()}", - envs={"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"]}, + envs={"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"], "CERC_SCRIPT_DEBUG": "true"}, add_hosts=[("host.docker.internal", "host-gateway")], volumes=[(repo_full_path, "/workspace")], command=build_command diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh index 6449bb8..2824e00 100755 --- a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -5,7 +5,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -if [[ $# -ne 2 ]]; then +if ! [[ $# -eq 1 || $# -eq 2 ]]; then echo "Illegal number of parameters" >&2 exit 1 fi diff --git a/container-build/cerc-builder-js/build-npm-package.sh b/container-build/cerc-builder-js/build-npm-package.sh index 88cc974..c0b8f5d 100755 --- a/container-build/cerc-builder-js/build-npm-package.sh +++ b/container-build/cerc-builder-js/build-npm-package.sh @@ -4,7 +4,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -if [[ $# -ne 2 ]]; then +if ! [[ $# -eq 1 || $# -eq 2 ]]; then echo "Illegal number of parameters" >&2 exit 1 fi @@ -12,8 +12,12 @@ if [[ -z "${CERC_NPM_AUTH_TOKEN}" ]]; then echo "CERC_NPM_AUTH_TOKEN is not set" >&2 exit 1 fi +if [[ $# -eq 2 ]]; then + package_publish_version=$2 +else + package_publish_version=$( cat package.json | jq -r .version ) +fi local_npm_registry_url=$1 -package_publish_version=$2 npm config set @lirewine:registry ${local_npm_registry_url} npm config set @cerc-io:registry ${local_npm_registry_url} npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN} From ff76cdad6147b61518657f127a49cb1402d31f31 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 5 Jan 2023 07:40:05 -0700 Subject: [PATCH 6/6] Back out debug code --- app/build_npms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build_npms.py b/app/build_npms.py index f91fcf9..255ca05 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -70,7 +70,7 @@ def command(ctx, include, exclude): interactive=True, tty=True, user=f"{os.getuid()}:{os.getgid()}", - envs={"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"], "CERC_SCRIPT_DEBUG": "true"}, + envs={"CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"]}, add_hosts=[("host.docker.internal", "host-gateway")], volumes=[(repo_full_path, "/workspace")], command=build_command