From 680f227967d5d32323d6bafe72ab8bb0be054c57 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 14 Dec 2022 09:23:12 -0700 Subject: [PATCH 01/11] Add yarn fixup script for local registry --- container-build/cerc-builder-js/README.md | 11 +++++++- .../cerc-builder-js/build-npm-package.sh | 2 +- .../yarn-local-registry-fixup.sh | 27 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100755 container-build/cerc-builder-js/yarn-local-registry-fixup.sh diff --git a/container-build/cerc-builder-js/README.md b/container-build/cerc-builder-js/README.md index ea09f21e..c1a93e32 100644 --- a/container-build/cerc-builder-js/README.md +++ b/container-build/cerc-builder-js/README.md @@ -12,5 +12,14 @@ it is possible to build packages manually by invoking `docker run` , for example $ docker run -it --add-host host.docker.internal:host-gateway \ -v ${HOME}/cerc/laconic-sdk:/workspace cerc/builder-js \ sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ - /build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.2.3-test' + /build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.4' +``` + +or: + +``` +$ docker run --rm -it --add-host host.docker.internal:host-gateway \ + -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js \ + sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ + /build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.4' ``` diff --git a/container-build/cerc-builder-js/build-npm-package.sh b/container-build/cerc-builder-js/build-npm-package.sh index cc2eeb26..9e5094b2 100755 --- a/container-build/cerc-builder-js/build-npm-package.sh +++ b/container-build/cerc-builder-js/build-npm-package.sh @@ -1,7 +1,7 @@ #!/bin/bash # Usage: build-npm-package.sh # Note: supply the registry auth token in NPM_AUTH_TOKEN -if [ -n "$CERC_ENABLE_SCRIPT_DEBUG" ]; then +if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi if [[ $# -ne 2 ]]; then diff --git a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh new file mode 100755 index 00000000..b16585cf --- /dev/null +++ b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Usage: yarn-local-registry-fixup.sh +# Assumes package.json and yarn.lock are in the cwd +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +if [[ $# -ne 1 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi +target_package=$1 +versioned_target_package=$(grep ${target_package} package.json | sed -e 's#[[:space:]]\{1,\}\"\('${target_package}'\)\":[[:space:]]\{1,\}\"\(.*\)\",#\1@\2#' ) +yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null) +package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball) +package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity) +package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum) +package_resolved=${package_tarball}#${package_shasum} +escaped_package_resolved=$(printf '%s\n' "$package_resolved" | sed -e 's/[\/&]/\\&/g') +escaped_target_package=$(printf '%s\n' "$target_package" | sed -e 's/[\/&]/\\&/g') +if [ -n "$CERC_SCRIPT_VERBOSE" ]; then + echo "Tarball: ${package_tarball}" + echo "Integrity: ${package_integrity}" + echo "Shasum: ${package_shasum}" + echo "Resolved: ${package_resolved}" +fi +sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock +sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${package_integrity}'/' yarn.lock From d96546a3c592f312ab29139b83dca0371202c7d7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 14 Dec 2022 09:26:18 -0700 Subject: [PATCH 02/11] Put script into the container --- container-build/cerc-builder-js/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/container-build/cerc-builder-js/Dockerfile b/container-build/cerc-builder-js/Dockerfile index e9511933..8eab2705 100644 --- a/container-build/cerc-builder-js/Dockerfile +++ b/container-build/cerc-builder-js/Dockerfile @@ -36,6 +36,7 @@ RUN \ # RUN su node -c "npm install -g " COPY build-npm-package.sh . +COPY yarn-local-registry-fixup.sh . COPY entrypoint.sh . ENTRYPOINT ["./entrypoint.sh"] From 717391d835fc7af8a9a18138c39e1a17e82b9210 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 19 Dec 2022 09:37:24 -0700 Subject: [PATCH 03/11] Add comments --- .../cerc-builder-js/yarn-local-registry-fixup.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh index b16585cf..db04c433 100755 --- a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh +++ b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh @@ -1,6 +1,12 @@ #!/bin/bash # Usage: yarn-local-registry-fixup.sh # Assumes package.json and yarn.lock are in the cwd +# The purpose of this script is to take a project cloned from git +# and "fixup" its yarn.lock file such that specified dependency +# will be fetched from a repository other than the one used when +# yarn.lock was generated. It updates all checksums using data +# from the "new" registry (because due to embedded timestamps etc +# the same source code re-built later will not have the same checksum). if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi @@ -10,11 +16,14 @@ if [[ $# -ne 1 ]]; then fi target_package=$1 versioned_target_package=$(grep ${target_package} package.json | sed -e 's#[[:space:]]\{1,\}\"\('${target_package}'\)\":[[:space:]]\{1,\}\"\(.*\)\",#\1@\2#' ) +# Use yarn info to get URL checksums etc from the new registry yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null) +# Code below parses out the values we need package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball) package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity) package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum) package_resolved=${package_tarball}#${package_shasum} +# Some strings need to be escaped so they work when passed to sed later escaped_package_resolved=$(printf '%s\n' "$package_resolved" | sed -e 's/[\/&]/\\&/g') escaped_target_package=$(printf '%s\n' "$target_package" | sed -e 's/[\/&]/\\&/g') if [ -n "$CERC_SCRIPT_VERBOSE" ]; then @@ -23,5 +32,6 @@ if [ -n "$CERC_SCRIPT_VERBOSE" ]; then echo "Shasum: ${package_shasum}" echo "Resolved: ${package_resolved}" fi +# Use magic sed regex to replace the values in yarn.lock sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${package_integrity}'/' yarn.lock From 23b12cc8f1aa04fd04f8b75a88a2625d961a0ae5 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 19 Dec 2022 09:44:51 -0700 Subject: [PATCH 04/11] Fix typo --- container-build/cerc-builder-js/yarn-local-registry-fixup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh index db04c433..13264a23 100755 --- a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh +++ b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh @@ -3,7 +3,7 @@ # Assumes package.json and yarn.lock are in the cwd # The purpose of this script is to take a project cloned from git # and "fixup" its yarn.lock file such that specified dependency -# will be fetched from a repository other than the one used when +# will be fetched from a registry other than the one used when # yarn.lock was generated. It updates all checksums using data # from the "new" registry (because due to embedded timestamps etc # the same source code re-built later will not have the same checksum). From b6f1fd029d25b86444b94e1b363b9fa31d4c7516 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 21:10:53 -0700 Subject: [PATCH 05/11] Add npm package build with local dependencies script --- .../build-npm-package-local-dependencies.sh | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 container-build/cerc-builder-js/build-npm-package-local-dependencies.sh diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh new file mode 100755 index 00000000..51b5ede8 --- /dev/null +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Usage: build-npm-package-local-dependencies.sh +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +if [[ $# -ne 2 ]]; then + echo "Illegal number of parameters" >&2 + exit 1 +fi +if [[ -z "${NPM_AUTH_TOKEN}" ]]; then + echo "NPM_AUTH_TOKEN is not set" >&2 + exit 1 +fi +local_npm_registry_url=$1 +package_publish_version=$2 +# TODO: make this a paramater and allow a list of scopes +npm_scope_for_local="@cerc-io" +# We need to configure the local registry +npm config set @cerc-io:registry ${local_npm_registry_url} +npm config set -- ${local_npm_registry_url}:_authToken ${NPM_AUTH_TOKEN} +# Find the set of dependencies from the specified scope +mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^@cerc-io/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]') +echo "Fixing up dependencies" +for package in "${dependencies_from_scope[@]}" +do + yarn-local-registry-fixup.sh $package +done +echo "Running build" +build-npm-package.sh ${local_npm_registry_url} ${package_publish_version} From b10b6c19f912d83ff53d870ddbd719a4b1b9eaa2 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 21:14:35 -0700 Subject: [PATCH 06/11] Use variable for local package scope --- .../cerc-builder-js/build-npm-package-local-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh index 51b5ede8..84327528 100755 --- a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -16,10 +16,10 @@ package_publish_version=$2 # TODO: make this a paramater and allow a list of scopes npm_scope_for_local="@cerc-io" # We need to configure the local registry -npm config set @cerc-io:registry ${local_npm_registry_url} +npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url} npm config set -- ${local_npm_registry_url}:_authToken ${NPM_AUTH_TOKEN} # Find the set of dependencies from the specified scope -mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^@cerc-io/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]') +mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]') echo "Fixing up dependencies" for package in "${dependencies_from_scope[@]}" do From 1b4b47e9a4608cd75baf33087f3f198115a0dedd Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 21:16:20 -0700 Subject: [PATCH 07/11] Add more output --- .../cerc-builder-js/build-npm-package-local-dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh index 84327528..7edba0a9 100755 --- a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -23,6 +23,7 @@ mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | echo "Fixing up dependencies" for package in "${dependencies_from_scope[@]}" do + echo "Fixing up package ${package}" yarn-local-registry-fixup.sh $package done echo "Running build" From 47b7a47160ad81979412f00f3417568fc0127d4c Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 21:17:30 -0700 Subject: [PATCH 08/11] Add comments --- .../cerc-builder-js/build-npm-package-local-dependencies.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh index 7edba0a9..3981e238 100755 --- a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -1,5 +1,7 @@ #!/bin/bash # Usage: build-npm-package-local-dependencies.sh +# Runs build-npm-package.sh after first fixing up yarn.lock to use a local +# npm registry for all packages in a spcific scope (currently @cerc-io) if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi From 1b0407ae3c21f3f7bcc102afe8b2eb91d1a90cda Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 21:39:06 -0700 Subject: [PATCH 09/11] Add scripts to image --- container-build/cerc-builder-js/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/container-build/cerc-builder-js/Dockerfile b/container-build/cerc-builder-js/Dockerfile index 8eab2705..9f460375 100644 --- a/container-build/cerc-builder-js/Dockerfile +++ b/container-build/cerc-builder-js/Dockerfile @@ -25,8 +25,8 @@ RUN \ && npm cache clean --force > /dev/null 2>&1 # [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends jq # [Optional] Uncomment if you want to install an additional version of node using nvm # ARG EXTRA_NODE_VERSION=10 @@ -35,8 +35,11 @@ RUN \ # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " -COPY build-npm-package.sh . -COPY yarn-local-registry-fixup.sh . +RUN mkdir /scripts +COPY build-npm-package.sh /scripts +COPY yarn-local-registry-fixup.sh /scripts +COPY build-npm-package-local-dependencies.sh /scripts +ENV PATH="${PATH}:/scripts" COPY entrypoint.sh . ENTRYPOINT ["./entrypoint.sh"] From c059745af966935b30049a360ad069636372afcc Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 22:37:44 -0700 Subject: [PATCH 10/11] Working builds --- .../build-npm-package-local-dependencies.sh | 2 +- .../cerc-builder-js/yarn-local-registry-fixup.sh | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh index 3981e238..cb2b52a4 100755 --- a/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh +++ b/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh @@ -26,7 +26,7 @@ echo "Fixing up dependencies" for package in "${dependencies_from_scope[@]}" do echo "Fixing up package ${package}" - yarn-local-registry-fixup.sh $package + yarn-local-registry-fixup.sh $package ${local_npm_registry_url} done echo "Running build" build-npm-package.sh ${local_npm_registry_url} ${package_publish_version} diff --git a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh index 13264a23..e6d9b793 100755 --- a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh +++ b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Usage: yarn-local-registry-fixup.sh +# Usage: yarn-local-registry-fixup.sh # Assumes package.json and yarn.lock are in the cwd # The purpose of this script is to take a project cloned from git # and "fixup" its yarn.lock file such that specified dependency @@ -10,16 +10,24 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -if [[ $# -ne 1 ]]; then +if [[ $# -ne 2 ]]; then echo "Illegal number of parameters" >&2 exit 1 fi target_package=$1 +local_npm_registry_url=$2 +# TODO: use jq rather than sed here: versioned_target_package=$(grep ${target_package} package.json | sed -e 's#[[:space:]]\{1,\}\"\('${target_package}'\)\":[[:space:]]\{1,\}\"\(.*\)\",#\1@\2#' ) # Use yarn info to get URL checksums etc from the new registry yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null) # Code below parses out the values we need package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball) +# When running inside a container, the registry can return a URL with the wrong host name due to proxying +# so we need to check if that has happened and fix the URL if so. +if ! [[ "${package_tarball}" =~ ^${local_npm_registry_url}.* ]]; then + # HACK: I've hard-wired the host names below. Replace with proper implementation + package_tarball=$( echo ${package_tarball} | sed -e 's/localhost/host.docker.internal/g' ) +fi package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity) package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum) package_resolved=${package_tarball}#${package_shasum} @@ -33,5 +41,6 @@ if [ -n "$CERC_SCRIPT_VERBOSE" ]; then echo "Resolved: ${package_resolved}" fi # Use magic sed regex to replace the values in yarn.lock +# Note: yarn.lock is not json so we can not use jq for this sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${package_integrity}'/' yarn.lock From a5b1f6e1072f1a9eb5e77b7ab25a4ff773a56930 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 3 Jan 2023 22:38:51 -0700 Subject: [PATCH 11/11] Update readme --- container-build/cerc-builder-js/README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/container-build/cerc-builder-js/README.md b/container-build/cerc-builder-js/README.md index c1a93e32..865d39ef 100644 --- a/container-build/cerc-builder-js/README.md +++ b/container-build/cerc-builder-js/README.md @@ -8,18 +8,10 @@ using `yarn`. As a temporary measure while the necessary functionality is being added to Stack Orchestrator, it is possible to build packages manually by invoking `docker run` , for example as follows: -``` -$ docker run -it --add-host host.docker.internal:host-gateway \ - -v ${HOME}/cerc/laconic-sdk:/workspace cerc/builder-js \ - sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ - /build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.4' -``` - -or: ``` $ docker run --rm -it --add-host host.docker.internal:host-gateway \ -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js \ sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ - /build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.4' + build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.8' ```