From 92743d7390d3e1f9017b313eabf9cf49ddd8a901 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 6 Jan 2023 13:13:56 -0700 Subject: [PATCH 1/4] Rename directory --- .../Dockerfile | 0 .../{cerc-laconic-cns-cli => cerc-laconic-registry-cli}/build.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename container-build/{cerc-laconic-cns-cli => cerc-laconic-registry-cli}/Dockerfile (100%) rename container-build/{cerc-laconic-cns-cli => cerc-laconic-registry-cli}/build.sh (100%) diff --git a/container-build/cerc-laconic-cns-cli/Dockerfile b/container-build/cerc-laconic-registry-cli/Dockerfile similarity index 100% rename from container-build/cerc-laconic-cns-cli/Dockerfile rename to container-build/cerc-laconic-registry-cli/Dockerfile diff --git a/container-build/cerc-laconic-cns-cli/build.sh b/container-build/cerc-laconic-registry-cli/build.sh similarity index 100% rename from container-build/cerc-laconic-cns-cli/build.sh rename to container-build/cerc-laconic-registry-cli/build.sh -- 2.45.2 From 6e67a59251be24f97224206025573421af5392c3 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 6 Jan 2023 13:44:52 -0700 Subject: [PATCH 2/4] Initial implementation --- app/data/container-image-list.txt | 2 +- .../cerc-laconic-registry-cli/Dockerfile | 57 ++++++++++++++++--- .../cerc-laconic-registry-cli/build.sh | 6 +- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/app/data/container-image-list.txt b/app/data/container-image-list.txt index c697fe21..48d5bc04 100644 --- a/app/data/container-image-list.txt +++ b/app/data/container-image-list.txt @@ -9,7 +9,7 @@ cerc/ipld-eth-beacon-db cerc/ipld-eth-beacon-indexer cerc/ipld-eth-server cerc/laconicd -cerc/laconic-cns-cli +cerc/laconic-registry-cli cerc/fixturenet-eth-geth cerc/fixturenet-eth-lighthouse cerc/watcher-mobymask diff --git a/container-build/cerc-laconic-registry-cli/Dockerfile b/container-build/cerc-laconic-registry-cli/Dockerfile index dab45c26..45ec0a8c 100644 --- a/container-build/cerc-laconic-registry-cli/Dockerfile +++ b/container-build/cerc-laconic-registry-cli/Dockerfile @@ -1,13 +1,54 @@ -# TODO: move this into the cerc-io/laconic-sdk (aka client) repo -FROM node:16.17.1-alpine3.16 +# Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile +# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster +ARG VARIANT=16-bullseye +FROM node:${VARIANT} -ARG NPM_AUTH_TOKEN -WORKDIR /app +ARG USERNAME=node +ARG NPM_GLOBAL=/usr/local/share/npm-global -COPY . . +# This container pulls npm packages from a local registry configured via these env vars +ARG CERC_LOCAL_NPM_URL +ARG CERC_LOCAL_NPM_TOKEN -RUN echo //npm.pkg.github.com/:_authToken=$NPM_AUTH_TOKEN > ~/.npmrc +# Add NPM global to PATH. +ENV PATH=${NPM_GLOBAL}/bin:${PATH} -RUN apk --update --no-cache add git && yarn && yarn build +RUN \ + # Configure global npm install location, use group to adapt to UID/GID changes + if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \ + && usermod -a -G npm ${USERNAME} \ + && umask 0002 \ + && mkdir -p ${NPM_GLOBAL} \ + && touch /usr/local/etc/npmrc \ + && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \ + && chmod g+s ${NPM_GLOBAL} \ + && npm config -g set prefix ${NPM_GLOBAL} \ + && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ + # Install eslint + && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + && npm cache clean --force > /dev/null 2>&1 -ENTRYPOINT ["/app/bin/laconic"] +# [Optional] Uncomment this section to install additional OS packages. +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends jq + +# [Optional] Uncomment if you want to install an additional version of node using nvm +# ARG EXTRA_NODE_VERSION=10 +# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" + +# [Optional] Uncomment if you want to install more global node modules +# RUN su node -c "npm install -g " + +# Configure the local npm registry +RUN npm config set @lirewine:registry ${CERC_LOCAL_NPM_URL} \ + && npm config set @cerc-io:registry ${CERC_LOCAL_NPM_URL} \ + && npm config set @lirewine:registry ${CERC_LOCAL_NPM_URL} \ + && npm config set -- ${CERC_LOCAL_NPM_URL}:_authToken ${CERC_LOCAL_NPM_TOKEN} + +# TODO: the image at this point could be made a base image for several different CLI images +# that install different Node-based CLI commands + +# Globally install the cli package +RUN yarn global add @cerc-io/laconic-registry-cli + +ENTRYPOINT ["laconic"] diff --git a/container-build/cerc-laconic-registry-cli/build.sh b/container-build/cerc-laconic-registry-cli/build.sh index a4a8de41..43ec55a7 100755 --- a/container-build/cerc-laconic-registry-cli/build.sh +++ b/container-build/cerc-laconic-registry-cli/build.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash -# Build cerc/laconic-cns-cli +# Build cerc/laconic-registry-cli # See: https://stackoverflow.com/a/246128/1701505 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/laconic-cns-cli:local -f ${SCRIPT_DIR}/Dockerfile --build-arg NPM_AUTH_TOKEN=$(NPM_AUTH_TOKEN) ${CERC_REPO_BASE_DIR}/laconic-cns-cli +docker build -t cerc/laconic-registry-cli:local -f ${SCRIPT_DIR}/Dockerfile \ + --add-host host.docker.internal:host-gateway \ + --build-arg CERC_LOCAL_NPM_TOKEN --build-arg CERC_LOCAL_NPM_URL ${SCRIPT_DIR} -- 2.45.2 From 318d70020e15bcd411d58e68fd2c366e7cd46ab7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 6 Jan 2023 13:54:03 -0700 Subject: [PATCH 3/4] Add build environment --- app/build_containers.py | 6 ++++++ container-build/cerc-laconic-registry-cli/Dockerfile | 12 ++++++------ container-build/cerc-laconic-registry-cli/build.sh | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/app/build_containers.py b/app/build_containers.py index f8edb0b3..43888b3d 100644 --- a/app/build_containers.py +++ b/app/build_containers.py @@ -64,6 +64,12 @@ def command(ctx, include, exclude): if verbose: print(f'Containers: {containers}') + # TODO: make this configurable + container_build_env = { + "CERC_NPM_URL": "http://host.docker.internal:3000/api/packages/cerc-io/npm/", + "CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"] + } + def process_container(container): if not quiet: print(f"Building: {container}") diff --git a/container-build/cerc-laconic-registry-cli/Dockerfile b/container-build/cerc-laconic-registry-cli/Dockerfile index 45ec0a8c..ba22a5c9 100644 --- a/container-build/cerc-laconic-registry-cli/Dockerfile +++ b/container-build/cerc-laconic-registry-cli/Dockerfile @@ -7,8 +7,8 @@ ARG USERNAME=node ARG NPM_GLOBAL=/usr/local/share/npm-global # This container pulls npm packages from a local registry configured via these env vars -ARG CERC_LOCAL_NPM_URL -ARG CERC_LOCAL_NPM_TOKEN +ARG CERC_NPM_URL +ARG CERC_NPM_AUTH_TOKEN # Add NPM global to PATH. ENV PATH=${NPM_GLOBAL}/bin:${PATH} @@ -40,10 +40,10 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # RUN su node -c "npm install -g " # Configure the local npm registry -RUN npm config set @lirewine:registry ${CERC_LOCAL_NPM_URL} \ - && npm config set @cerc-io:registry ${CERC_LOCAL_NPM_URL} \ - && npm config set @lirewine:registry ${CERC_LOCAL_NPM_URL} \ - && npm config set -- ${CERC_LOCAL_NPM_URL}:_authToken ${CERC_LOCAL_NPM_TOKEN} +RUN npm config set @lirewine:registry ${CERC_NPM_URL} \ + && npm config set @cerc-io:registry ${CERC_NPM_URL} \ + && npm config set @lirewine:registry ${CERC_NPM_URL} \ + && npm config set -- ${CERC_NPM_URL}:_authToken ${CERC_NPM_AUTH_TOKEN} # TODO: the image at this point could be made a base image for several different CLI images # that install different Node-based CLI commands diff --git a/container-build/cerc-laconic-registry-cli/build.sh b/container-build/cerc-laconic-registry-cli/build.sh index 43ec55a7..fa29fa4b 100755 --- a/container-build/cerc-laconic-registry-cli/build.sh +++ b/container-build/cerc-laconic-registry-cli/build.sh @@ -6,4 +6,4 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) docker build -t cerc/laconic-registry-cli:local -f ${SCRIPT_DIR}/Dockerfile \ --add-host host.docker.internal:host-gateway \ - --build-arg CERC_LOCAL_NPM_TOKEN --build-arg CERC_LOCAL_NPM_URL ${SCRIPT_DIR} + --build-arg CERC_NPM_AUTH_TOKEN --build-arg CERC_NPM_URL ${SCRIPT_DIR} -- 2.45.2 From 82755a27f2f34ea2a1916d562c7572a5949ffd6b Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 6 Jan 2023 15:57:14 -0700 Subject: [PATCH 4/4] Build a CLI container --- app/build_containers.py | 7 ++++--- app/build_npms.py | 4 ++-- container-build/cerc-builder-js/README.md | 4 ++-- container-build/cerc-builder-js/build-npm-package.sh | 2 +- .../cerc-builder-js/yarn-local-registry-fixup.sh | 3 ++- container-build/cerc-laconic-registry-cli/Dockerfile | 3 +++ container-build/cerc-laconic-registry-cli/build.sh | 2 +- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/app/build_containers.py b/app/build_containers.py index 43888b3d..c74df1e9 100644 --- a/app/build_containers.py +++ b/app/build_containers.py @@ -66,8 +66,9 @@ def command(ctx, include, exclude): # TODO: make this configurable container_build_env = { - "CERC_NPM_URL": "http://host.docker.internal:3000/api/packages/cerc-io/npm/", - "CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"] + "CERC_NPM_URL": "http://gitea.local:3000/api/packages/cerc-io/npm/", + "CERC_NPM_AUTH_TOKEN": os.environ["CERC_NPM_AUTH_TOKEN"], + "CERC_REPO_BASE_DIR": dev_root_path } def process_container(container): @@ -91,7 +92,7 @@ def command(ctx, include, exclude): if not dry_run: if verbose: print(f"Executing: {build_command}") - build_result = subprocess.run(build_command, shell=True, env={'CERC_REPO_BASE_DIR': dev_root_path}) + build_result = subprocess.run(build_command, shell=True, env=container_build_env) # TODO: check result in build_result.returncode print(f"Result is: {build_result}") else: diff --git a/app/build_npms.py b/app/build_npms.py index 2bce43f3..e00c0734 100644 --- a/app/build_npms.py +++ b/app/build_npms.py @@ -62,7 +62,7 @@ def command(ctx, include, exclude): repo_dir = package repo_full_path = os.path.join(dev_root_path, repo_dir) # TODO: make the npm registry url configurable. - build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/"] + build_command = ["sh", "-c", "cd /workspace && build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/"] if not dry_run: if verbose: print(f"Executing: {build_command}") @@ -73,7 +73,7 @@ def command(ctx, include, exclude): tty=True, user=f"{os.getuid()}:{os.getgid()}", envs=envs, - add_hosts=[("host.docker.internal", "host-gateway")], + add_hosts=[("gitea.local", "host-gateway")], volumes=[(repo_full_path, "/workspace")], command=build_command ) diff --git a/container-build/cerc-builder-js/README.md b/container-build/cerc-builder-js/README.md index da784efa..e77136f8 100644 --- a/container-build/cerc-builder-js/README.md +++ b/container-build/cerc-builder-js/README.md @@ -10,8 +10,8 @@ it is possible to build packages manually by invoking `docker run` , for example ``` -$ docker run --rm -it --add-host host.docker.internal:host-gateway \ +$ docker run --rm -it --add-host gitea.local:host-gateway \ -v ${HOME}/cerc/laconic-registry-cli:/workspace cerc/builder-js \ sh -c 'cd /workspace && CERC_NPM_AUTH_TOKEN=6613572a28ebebaee20ccd90064251fa8c2b94f6 \ - build-npm-package-local-dependencies.sh http://host.docker.internal:3000/api/packages/cerc-io/npm/ 0.1.8' + build-npm-package-local-dependencies.sh http://gitea.local:3000/api/packages/cerc-io/npm/ 0.1.8' ``` diff --git a/container-build/cerc-builder-js/build-npm-package.sh b/container-build/cerc-builder-js/build-npm-package.sh index 89308bc8..9c9ac580 100755 --- a/container-build/cerc-builder-js/build-npm-package.sh +++ b/container-build/cerc-builder-js/build-npm-package.sh @@ -25,7 +25,7 @@ npm config set @cerc-io:registry ${local_npm_registry_url} npm config set -- ${local_npm_registry_url}:_authToken ${CERC_NPM_AUTH_TOKEN} # First check if the version of this package we're trying to build already exists in the registry package_exists=$( yarn info --json ${package_name}@${package_publish_version} | jq -r .data.dist.tarball ) -if [[ -n "$package_exists" ]]; then +if [[ ! -z "$package_exists" && "$package_exists" != "null" ]]; then echo "${package_publish_version} of ${package_name} already exists in the registry, skipping build" exit 0 fi diff --git a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh index e6d9b793..987fcbc2 100755 --- a/container-build/cerc-builder-js/yarn-local-registry-fixup.sh +++ b/container-build/cerc-builder-js/yarn-local-registry-fixup.sh @@ -26,7 +26,8 @@ package_tarball=$(echo $yarn_info_output | jq -r .data.dist.tarball) # so we need to check if that has happened and fix the URL if so. if ! [[ "${package_tarball}" =~ ^${local_npm_registry_url}.* ]]; then # HACK: I've hard-wired the host names below. Replace with proper implementation - package_tarball=$( echo ${package_tarball} | sed -e 's/localhost/host.docker.internal/g' ) + # TODO: remove the hack when proven no longer necessary + package_tarball=$( echo ${package_tarball} | sed -e 's/localhost/gitea.local/g' ) fi package_integrity=$(echo $yarn_info_output | jq -r .data.dist.integrity) package_shasum=$(echo $yarn_info_output | jq -r .data.dist.shasum) diff --git a/container-build/cerc-laconic-registry-cli/Dockerfile b/container-build/cerc-laconic-registry-cli/Dockerfile index ba22a5c9..3341da79 100644 --- a/container-build/cerc-laconic-registry-cli/Dockerfile +++ b/container-build/cerc-laconic-registry-cli/Dockerfile @@ -48,6 +48,9 @@ RUN npm config set @lirewine:registry ${CERC_NPM_URL} \ # TODO: the image at this point could be made a base image for several different CLI images # that install different Node-based CLI commands +# DEBUG, remove +RUN yarn info @cerc-io/laconic-registry-cli + # Globally install the cli package RUN yarn global add @cerc-io/laconic-registry-cli diff --git a/container-build/cerc-laconic-registry-cli/build.sh b/container-build/cerc-laconic-registry-cli/build.sh index fa29fa4b..ea157dac 100755 --- a/container-build/cerc-laconic-registry-cli/build.sh +++ b/container-build/cerc-laconic-registry-cli/build.sh @@ -5,5 +5,5 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) docker build -t cerc/laconic-registry-cli:local -f ${SCRIPT_DIR}/Dockerfile \ - --add-host host.docker.internal:host-gateway \ + --add-host gitea.local:host-gateway \ --build-arg CERC_NPM_AUTH_TOKEN --build-arg CERC_NPM_URL ${SCRIPT_DIR} -- 2.45.2