Add support for pnpm as a webapp build tool. (#767 )

This adds support for auto-detecting pnpm as a build tool for webapps. Reviewed-on: cerc-io/stack-orchestrator#767 Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to> Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>
Check for existing tag in remote repo before building. (#764 )
2024-02-26 23:31:52 +00:00 · 2024-02-24 03:22:49 +00:00 · 2024-02-24 00:15:53 +00:00
11 changed files with 107 additions and 29 deletions
@@ -21,6 +21,8 @@
 # TODO: display the available list of containers; allow re-build of either all or specific containers

 import os
+import sys
+
 from decouple import config
 import click
 from pathlib import Path
@@ -40,12 +42,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
    '''build the specified webapp container'''

    quiet = ctx.obj.quiet
-    verbose = ctx.obj.verbose
-    dry_run = ctx.obj.dry_run
    debug = ctx.obj.debug
    local_stack = ctx.obj.local_stack
    stack = ctx.obj.stack
-    continue_on_error = ctx.obj.continue_on_error

    # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
    container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -73,7 +72,10 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
        container_build_env,
        dev_root_path,
    )
-    build_containers.process_container(build_context_1)
+    ok = build_containers.process_container(build_context_1)
+    if not ok:
+        print("ERROR: Build failed.", file=sys.stderr)
+        sys.exit(1)

    # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
    container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -94,4 +96,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
        container_build_env,
        dev_root_path,
    )
-    build_containers.process_container(build_context_2)
+    ok = build_containers.process_container(build_context_2)
+    if not ok:
+        print("ERROR: Build failed.", file=sys.stderr)
+        sys.exit(1)
@@ -26,6 +26,8 @@ RUN \
    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
    # Install semver
    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+    # Install pnpm
+    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
    && npm cache clean --force > /dev/null 2>&1

 # [Optional] Uncomment this section to install additional OS packages.
@@ -35,6 +37,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"

+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
+
 # Expose port for http
 EXPOSE 80

@@ -10,10 +10,12 @@ TRG_DIR="${3:-.next-r}"

 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ]; then
-    CERC_BUILD_TOOL=npm
-  else
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
  fi
 fi

@@ -9,7 +9,9 @@ CERC_MIN_NEXTVER=13.4.2
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
    CERC_BUILD_TOOL=yarn
  else
    CERC_BUILD_TOOL=npm
@@ -16,7 +16,9 @@ trap ctrl_c INT

 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
    CERC_BUILD_TOOL=yarn
  else
    CERC_BUILD_TOOL=npm
@@ -24,6 +24,10 @@ RUN \
    && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
    # Install eslint
    && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    # Install semver
+    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+    # Install pnpm
+    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
    && npm cache clean --force > /dev/null 2>&1

 # [Optional] Uncomment this section to install additional OS packages.
@@ -1,11 +1,12 @@
 FROM cerc/webapp-base:local as builder

 ARG CERC_BUILD_TOOL
+ARG CERC_BUILD_OUTPUT_DIR

 WORKDIR /app
 COPY . .
-RUN rm -rf node_modules build .next*
-RUN /scripts/build-app.sh /app build /data
+RUN rm -rf node_modules build dist .next*
+RUN /scripts/build-app.sh /app /data

 FROM cerc/webapp-base:local
 COPY --from=builder /data /data
@@ -7,9 +7,10 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi

 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
+
 WORK_DIR="${1:-/app}"
-OUTPUT_DIR="${2:-build}"
-DEST_DIR="${3:-/data}"
+DEST_DIR="${2:-/data}"

 if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
@@ -22,7 +23,9 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
  cd "${WORK_DIR}" || exit 1

  if [ -z "$CERC_BUILD_TOOL" ]; then
-    if [ -f "yarn.lock" ]; then
+    if [ -f "pnpm-lock.yaml" ]; then
+      CERC_BUILD_TOOL=pnpm
+    elif [ -f "yarn.lock" ]; then
      CERC_BUILD_TOOL=yarn
    else
      CERC_BUILD_TOOL=npm
@@ -33,7 +36,17 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
  $CERC_BUILD_TOOL build || exit 1

  rm -rf "${DEST_DIR}"
-  mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
+  if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
+    if [ -d "${WORK_DIR}/dist" ]; then
+      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
+    elif [ -d "${WORK_DIR}/build" ]; then
+      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
+    else
+      echo "ERROR: Unable to locate build output.  Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
+      exit 1
+    fi
+  fi
+  mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
 else
  echo "Copying static app ..."
  mv "${WORK_DIR}" "${DEST_DIR}"
@@ -33,23 +33,29 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec

 ### Ports

-Edit `network` in spec file to map container ports to same ports in host:
+Edit `uniswap-urbit-app-spec.yml` such that it looks like:

 ```yml
-...
+stack: uniswap-urbit-app
+deploy-to: compose
 network:
  ports:
-    urbit-fake-ship:
-      - '8080:80'
    proxy-server:
      - '4000:4000'
+    urbit-fake-ship:
+      - '8080:80'
    ipfs:
-      - '8081:8080'
-      - '5001:5001'
-...
+     - '4001'
+     - '8081:8080'
+     - 0.0.0.0:5001:5001
+volumes:
+  urbit_app_builds: ./data/urbit_app_builds
+  urbit_data: ./data/urbit_data
+  ipfs-import: ./data/ipfs-import
+  ipfs-data: ./data/ipfs-data
 ```

-Note: Skip the `ipfs` ports if need to use an externally running IPFS node
+Note: Skip the `ipfs` ports if using an externally running IPFS node, set via `config.env`, below.

 ### Data volumes

@@ -29,6 +29,29 @@ def _image_needs_pushed(image: str):
    return image.endswith(":local")


+def remote_image_exists(remote_repo_url: str, local_tag: str):
+    docker = DockerClient()
+    try:
+        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+        result = docker.manifest.inspect(remote_tag)
+        return True if result else False
+    except Exception:  # noqa: E722
+        return False
+
+
+def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
+    if not additional_tags:
+        return
+
+    if not remote_image_exists(remote_repo_url, local_tag):
+        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
+
+    docker = DockerClient()
+    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
+
+
 def remote_tag_for_image(image: str, remote_repo_url: str):
    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
    major_parts = image.split("/", 2)
@@ -24,6 +24,7 @@ import uuid

 import click

+from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
 from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
                                                   build_container_image, push_container_image,
@@ -43,6 +44,7 @@ def process_app_deployment_request(
    deployment_parent_dir,
    kube_config,
    image_registry,
+    force_rebuild=False,
    log_file=None
 ):
    # 1. look up application
@@ -91,7 +93,9 @@ def process_app_deployment_request(
    deployment_record = laconic.get_record(app_deployment_crn)
    deployment_dir = os.path.join(deployment_parent_dir, fqdn)
    deployment_config_file = os.path.join(deployment_dir, "config.env")
+    # TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
    deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
+    app_image_shared_tag = f"laconic-webapp/{app.id}:local"
    #   b. check for deployment directory (create if necessary)
    if not os.path.exists(deployment_dir):
        if deployment_record:
@@ -106,11 +110,20 @@ def process_app_deployment_request(
    needs_k8s_deploy = False
    # 6. build container (if needed)
    if not deployment_record or deployment_record.attributes.application != app.id:
-        # TODO: pull from request
-        extra_build_args = []
-        build_container_image(app, deployment_container_tag, extra_build_args, log_file)
-        push_container_image(deployment_dir, log_file)
        needs_k8s_deploy = True
+        # check if the image already exists
+        shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
+        if shared_tag_exists and not force_rebuild:
+            # simply add our unique tag to the existing image and we are done
+            print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file)
+            add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
+        else:
+            extra_build_args = []  # TODO: pull from request
+            build_container_image(app, deployment_container_tag, extra_build_args, log_file)
+            push_container_image(deployment_dir, log_file)
+            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
+            print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file)
+            add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)

    # 7. update config (if needed)
    if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -171,12 +184,13 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
@click.option("--include-tags", help="Only include requests with matching tags (comma-separated).", default="")
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
+@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.pass_context
 def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,  # noqa: C901
            request_id, discover, state_file, only_update_state,
            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
-            include_tags, exclude_tags, log_dir):
+            include_tags, exclude_tags, force_rebuild, log_dir):
    if request_id and discover:
        print("Cannot specify both --request-id and --discover", file=sys.stderr)
        sys.exit(2)
@@ -306,6 +320,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                    os.path.abspath(deployment_parent_dir),
                    kube_config,
                    image_registry,
+                    force_rebuild,
                    run_log_file
                )
                status = "DEPLOYED"