Merge pull request #86 from cerc-io/dboreham/build-laconic-cli

Allow building of npm packages using locally published dependencies

container-build/cerc-builder-js/Dockerfile

@@ -25,8 +25,8 @@ RUN \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends jq
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -35,7 +35,11 @@ RUN \
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
-COPY build-npm-package.sh .
+RUN mkdir /scripts
+COPY build-npm-package.sh /scripts
+COPY yarn-local-registry-fixup.sh /scripts
+COPY build-npm-package-local-dependencies.sh /scripts
+ENV PATH="${PATH}:/scripts"
 
 COPY entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]

container-build/cerc-builder-js/README.md

@@ -8,9 +8,10 @@ using `yarn`.
 As a temporary measure while the necessary functionality is being added to Stack Orchestrator,
 it is possible to build packages manually by invoking `docker run` , for example as follows:
 
+
 ```
-$ docker run -it --add-host host.docker.internal:host-gateway \
-  -v ${HOME}/cerc/laconic-sdk:/workspace cerc/builder-js \
+$ docker run --rm -it --add-host host.docker.internal:host-gateway \
+  -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js  \
   sh -c 'cd /workspace && NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \
-  /build-npm-package.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 1.2.3-test'
+  build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.8'
 ```

container-build/cerc-builder-js/build-npm-package-local-dependencies.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
+# Runs build-npm-package.sh after first fixing up yarn.lock to use a local
+# npm registry for all packages in a spcific scope (currently @cerc-io)
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if [[ $# -ne 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+if [[ -z "${NPM_AUTH_TOKEN}" ]]; then
+    echo "NPM_AUTH_TOKEN is not set" >&2
+    exit 1
+fi
+local_npm_registry_url=$1
+package_publish_version=$2
+# TODO: make this a paramater and allow a list of scopes
+npm_scope_for_local="@cerc-io"
+# We need to configure the local registry 
+npm config set ${npm_scope_for_local}:registry ${local_npm_registry_url}
+npm config set -- ${local_npm_registry_url}:_authToken ${NPM_AUTH_TOKEN}
+# Find the set of dependencies from the specified scope
+mapfile -t dependencies_from_scope < <(cat package.json | jq -r '.dependencies | with_entries(if (.key|test("^'${npm_scope_for_local}'/.*$")) then ( {key: .key, value: .value } ) else empty end ) | keys[]')
+echo "Fixing up dependencies"
+for package in "${dependencies_from_scope[@]}"
+do
+    echo "Fixing up package ${package}"
+    yarn-local-registry-fixup.sh $package ${local_npm_registry_url}
+done
+echo "Running build"
+build-npm-package.sh ${local_npm_registry_url} ${package_publish_version}

container-build/cerc-builder-js/build-npm-package.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Usage: build-npm-package.sh <registry-url> <publish-with-this-version>
 # Note: supply the registry auth token in NPM_AUTH_TOKEN
-if [ -n "$CERC_ENABLE_SCRIPT_DEBUG" ]; then
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 if [[ $# -ne 2 ]]; then

container-build/cerc-builder-js/yarn-local-registry-fixup.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Usage: yarn-local-registry-fixup.sh <package-to-fix> <registry-url>
+# Assumes package.json and yarn.lock are in the cwd
+# The purpose of this script is to take a project cloned from git
+# and "fixup" its yarn.lock file such that specified dependency
+# will be fetched from a registry other than the one used when
+# yarn.lock was generated. It updates all checksums using data
+# from the "new" registry (because due to embedded timestamps etc
+# the same source code re-built later will not have the same checksum).
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+if [[ $# -ne 2 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+target_package=$1
+local_npm_registry_url=$2
+# TODO: use jq rather than sed here:
+versioned_target_package=$(grep ${target_package} package.json | sed -e 's#[[:space:]]\{1,\}\"\('${target_package}'\)\":[[:space:]]\{1,\}\"\(.*\)\",#\1@\2#' )
+# Use yarn info to get URL checksums etc from the new registry
+yarn_info_output=$(yarn info --json $versioned_target_package 2>/dev/null)
+# Code below parses out the values we need
+package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball)
+# When running inside a container, the registry can return a URL with the wrong host name due to proxying
+# so we need to check if that has happened and fix the URL if so.
+if ! [[ "${package_tarball}" =~ ^${local_npm_registry_url}.* ]]; then
+    # HACK: I've hard-wired the host names below. Replace with proper implementation
+    package_tarball=$( echo ${package_tarball} | sed -e 's/localhost/host.docker.internal/g' )
+fi
+package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity)
+package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum)
+package_resolved=${package_tarball}#${package_shasum}
+# Some strings need to be escaped so they work when passed to sed later
+escaped_package_resolved=$(printf '%s\n' "$package_resolved" | sed -e 's/[\/&]/\\&/g')
+escaped_target_package=$(printf '%s\n' "$target_package" | sed -e 's/[\/&]/\\&/g')
+if [ -n "$CERC_SCRIPT_VERBOSE" ]; then
+    echo "Tarball: ${package_tarball}"
+    echo "Integrity: ${package_integrity}"
+    echo "Shasum: ${package_shasum}"
+    echo "Resolved: ${package_resolved}"
+fi
+# Use magic sed regex to replace the values in yarn.lock
+# Note: yarn.lock is not json so we can not use jq for this
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}resolved \).*$/\1'\"${escaped_package_resolved}\"'/' yarn.lock
+sed -i -e '/^\"'${escaped_target_package}'.*\":$/ , /^\".*$/ s/^\([[:space:]]\{1,\}integrity \).*$/\1'${package_integrity}'/' yarn.lock