Allow setting build tool (npm/yarn) and next.js version. (#639)

* Allow setting build tool (npm/yarn) and next.js version.

stack_orchestrator/build/build_webapp.py

@@ -65,6 +65,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
 
 
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
+    container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
                                                                     base_container.replace("/", "-"),

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
+ARG VARIANT=20-bullseye
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -28,28 +28,15 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
-
-# [Optional] Uncomment if you want to install an additional version of node using nvm
-# ARG EXTRA_NODE_VERSION=10
-# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
-
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
-COPY /scripts /scripts
+    && apt-get -y install --no-install-recommends jq gettext-base moreutils
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
-# RUN mkdir -p /config
-# COPY ./config.yml /config
-
-# Install simple web server for now (use nginx perhaps later)
-# RUN yarn global add http-server
-
 # Expose port for http
 EXPOSE 3000
 
+COPY /scripts /scripts
+
 # Default command sleeps forever so docker doesn't kill it
 CMD ["/scripts/start-serving-app.sh"]

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp

@@ -1,4 +1,8 @@
 FROM cerc/nextjs-base:local
+
+ARG CERC_NEXT_VERSION=latest
+ARG CERC_BUILD_TOOL
+
 WORKDIR /app
 COPY . .
 RUN rm -rf node_modules build .next*

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -8,6 +8,15 @@ WORK_DIR="${1:-./}"
 SRC_DIR="${2:-.next}"
 TRG_DIR="${3:-.next-r}"
 
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ]; then
+    CERC_BUILD_TOOL=npm
+  else
+    CERC_BUILD_TOOL=yarn
+  fi
+fi
+
 cd "${WORK_DIR}" || exit 1
 
 rm -rf "$TRG_DIR"
@@ -25,12 +34,12 @@ if [ -f ".env" ]; then
 fi
 
 for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
-  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o '^"CERC_RUNTIME_ENV[^\"]+"$'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)
     esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
-    echo "$cur_name=$cur_val"
+    echo "$f: $cur_name=$cur_val"
     sed -i "s/$orig_name/$esc_val/g" $f
   done
 done

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -4,27 +4,41 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-^14.0.2}"
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
-cp next.config.js next.config.dist
+if [ ! -f "next.config.dist" ]; then
+  cp next.config.js next.config.dist
+fi
+
+which js-beautify >/dev/null
+if [ $? -ne 0 ]; then
+  npm i -g js-beautify
+fi
 
-npm i -g js-beautify
 js-beautify next.config.dist > next.config.js
+echo "" >> next.config.js
 
-npm install
-
-CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
-MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1)
-
-head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1
-
-cat > next.config.js.2 <<EOF
-const webpack = require('webpack');
+WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
+if [ -z "$WEBPACK_REQ_LINE" ]; then
+  cat > next.config.js.0 <<EOF
+    const webpack = require('webpack');
+EOF
+fi
 
+cat > next.config.js.1 <<EOF
 let envMap;
 try {
   // .env-list.json provides us a list of identifiers which should be replaced at runtime.
@@ -43,21 +57,55 @@ try {
 }
 EOF
 
-grep 'module.exports' next.config.js > next.config.js.3
+CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }')
+ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1)
+WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1)
+NEXT_SECTION_ADJUSTMENT=0
 
-cat > next.config.js.4 <<EOF
-  webpack: (config) => {
-    config.plugins.push(new webpack.DefinePlugin(envMap));
-    return config;
-  },
+if [ -n "$WEBPACK_CONF_LINE" ]; then
+  WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2)
+  head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2
+  cat > next.config.js.3 <<EOF
+      $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
 EOF
+  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE))
+elif [ -n "$ENV_LINE" ]; then
+  head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
+  cat > next.config.js.3 <<EOF
+    webpack: (config) => {
+      config.plugins.push(new webpack.DefinePlugin(envMap));
+      return config;
+    },
+EOF
+  NEXT_SECTION_ADJUSTMENT=1
+  NEXT_SECTION_LINE=$ENV_LINE
+else
+  echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
+  rm -f next.config.js.*
+  NEXT_SECTION_LINE=0
+fi
 
-tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5
+tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5
 
-cat next.config.js.* | js-beautify > next.config.js
+cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js
 rm next.config.js.*
 
 "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json
 
-npm run build
-rm .env-list.json
+if [ ! -f "package.dist" ]; then
+  cp package.json package.dist
+fi
+
+cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
+
+CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
+
+if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
+  echo "Changing 'next' version from $CUR_NEXT_VERSION to $CERC_NEXT_VERSION (set with --build-arg CERC_NEXT_VERSION)"
+  cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json
+fi
+
+$CERC_BUILD_TOOL install || exit 1
+$CERC_BUILD_TOOL run cerc_compile || exit 1
+
+exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh

@@ -20,5 +20,10 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' -
   done
 done
 
+NEXT_CONF="next.config.js next.config.dist"
+for f in $NEXT_CONF; do
+    cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF
+done
+
 cat $TMPF | sort -u | jq  --raw-input .  | jq --slurp .
 rm -f $TMPF

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -5,9 +5,39 @@ fi
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
+
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
 
-rm -rf .next-r
 "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r
-npm start .next-r -p ${CERC_LISTEN_PORT:-3000}
+mv .next .next.old
+mv .next-r/.next .
+
+if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
+  jq -e '.scripts.cerc_generate' package.json >/dev/null
+  if [ $? -eq 0 ]; then
+    npm run cerc_generate > gen.out 2>&1 &
+    tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
+    count=0
+    while [ $count -lt 10 ]; do
+      sleep 1
+      ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
+      if [ $? -ne 0 ]; then 
+        break
+      else
+        count=$((count + 1))
+      fi
+    done
+    kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+  fi
+fi
+
+$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}

tests/webapp-test/run-webapp-test.sh

@@ -24,7 +24,7 @@ git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BAS
 # Test webapp command execution
 $TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app
 
-UUID=`uuidgen`
+CHECK="SPECIAL_01234567890_TEST_STRING"
 
 set +e
 
@@ -34,7 +34,7 @@ wget -O test.before -m http://localhost:3000
 
 docker remove -f $CONTAINER_ID
 
-CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -d cerc/test-progressive-web-app:local)
 sleep 3
 wget -O test.after -m http://localhost:3000
 
@@ -43,7 +43,7 @@ docker remove -f $CONTAINER_ID
 echo "###########################################################################"
 echo ""
 
-grep "$UUID" test.before > /dev/null
+grep "$CHECK" test.before > /dev/null
 if [ $? -ne 1 ]; then
   echo "BEFORE: FAILED"
   exit 1
@@ -51,7 +51,7 @@ else
   echo "BEFORE: PASSED"
 fi
 
-grep "$UUID" test.after > /dev/null
+grep "$CHECK" test.after > /dev/null
 if [ $? -ne 0 ]; then
   echo "AFTER: FAILED"
   exit 1