Add --dry-run option (#686 )

Add deployment update and deploy-webapp-from-registry commands. (#676 )
Expose metrics endpoints for sushiswap and merkl sushiswap watchers (#685 )
2023-12-13 22:56:40 -06:00 · 2023-12-13 21:02:34 -06:00 · 2023-12-13 14:58:26 +05:30 · 2023-12-12 13:54:00 +05:30 · 2023-12-11 14:10:54 +05:30 · 2023-12-08 09:35:00 +05:30
46 changed files with 1356 additions and 378 deletions
@@ -0,0 +1,36 @@
+version: '3.7'
+
+services:
+  # Runs an Urbit fake ship and attempts an app installation using given data
+  # Uploads the app glob to given IPFS endpoint
+  # From urbit_app_builds volume:
+  # - takes app build from ${CERC_URBIT_APP}/build (waits for it to appear)
+  # - takes additional mark files from ${CERC_URBIT_APP}/mar
+  # - takes the docket file from ${CERC_URBIT_APP}/desk.docket-0
+  urbit-fake-ship:
+    restart: unless-stopped
+    image: tloncorp/vere
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_URBIT_APP: ${CERC_URBIT_APP}
+      CERC_ENABLE_APP_INSTALL: ${CERC_ENABLE_APP_INSTALL:-true}
+      CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs:5001}
+      CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs:8080}
+    entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-app.sh && tail -f /dev/null"]
+    volumes:
+      - urbit_data:/urbit
+      - urbit_app_builds:/app-builds
+      - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh
+      - ../config/urbit/deploy-app.sh:/urbit/deploy-app.sh
+    ports:
+      - "80"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+volumes:
+  urbit_data:
+  urbit_app_builds:
@@ -1,13 +1,24 @@
 version: "3.2"
+
 # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
 services:
  ipfs:
-    image: ipfs/kubo:master-2023-02-20-714a968
+    image: ipfs/kubo:v0.24.0
    restart: always
    volumes:
-      - ./ipfs/import:/import
-      - ./ipfs/data:/data/ipfs
+      - ipfs-import:/import
+      - ipfs-data:/data/ipfs
    ports:
-      - "0.0.0.0:8080:8080"
-      - "0.0.0.0:4001:4001"
+      - "4001"
+      - "8080"
      - "0.0.0.0:5001:5001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+volumes:
+  ipfs-import:
+  ipfs-data:
@@ -0,0 +1,34 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end:local
+    restart: on-failure
+    environment:
+      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+    working_dir: /app/packages/web
+    command: ["./build-app.sh"]
+    volumes:
+      - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh
+      - app_builds:/app-builds
+
+  nginx:
+    image: nginx:1.23-alpine
+    restart: always
+    depends_on:
+      osmosis-front-end:
+        condition: service_completed_successfully
+    volumes:
+      - ../config/osmosis/nginx:/etc/nginx/conf.d
+      - app_builds:/usr/share/nginx
+    ports:
+      - "80"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  app_builds:
@@ -6,7 +6,7 @@ services:
    restart: on-failure
    working_dir: /app/packages/cli
    environment:
-      ENABLE_PROXY: ${ENABLE_PROXY:-true}
+      ENABLE_PROXY: ${CERC_ENABLE_PROXY:-true}
      PROXY_UPSTREAM: ${CERC_PROXY_UPSTREAM}
      PROXY_ORIGIN_HEADER: ${CERC_PROXY_ORIGIN_HEADER}
    command: ["sh", "-c", "./run.sh"]
@@ -9,9 +9,10 @@ services:
      - REACT_APP_AWS_API_ENDPOINT=${CERC_UNISWAP_GQL}
    command: ["./build-app.sh"]
    volumes:
-      - app_builds:/app-builds
      - ../config/uniswap-interface/build-app.sh:/app/build-app.sh
+      - urbit_app_builds:/app-builds
+      - ../config/uniswap-interface/urbit-files/mar:/app/mar
+      - ../config/uniswap-interface/urbit-files/desk.docket-0:/app/desk.docket-0

 volumes:
-  app_builds:
-  app_globs:
+  urbit_app_builds:
@@ -1,46 +0,0 @@
-version: '3.7'
-
-services:
-  urbit-fake-ship:
-    restart: unless-stopped
-    image: tloncorp/vere
-    environment:
-      CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs-glob-host:5001}
-      CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs-glob-host:8080}
-    entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-uniswap-app.sh && tail -f /dev/null"]
-    volumes:
-      - urbit_data:/urbit
-      - app_builds:/app-builds
-      - app_globs:/app-globs
-      - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh
-      - ../config/uniswap-interface/deploy-uniswap-app.sh:/urbit/deploy-uniswap-app.sh
-    ports:
-      - "80"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "80"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
-  ipfs-glob-host:
-    image: ipfs/kubo:master-2023-02-20-714a968
-    volumes:
-      - ipfs-import:/import
-      - ipfs-data:/data/ipfs
-    ports:
-      - "8080"
-      - "5001"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "5001"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
-volumes:
-  urbit_data:
-  app_builds:
-  app_globs:
-  ipfs-import:
-  ipfs-data:
@@ -35,7 +35,7 @@ services:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
    ports:
-      - "9000"
+      - "9002:9000"
    healthcheck:
      test: ["CMD", "nc", "-v", "localhost", "9000"]
      interval: 20s
@@ -62,7 +62,7 @@ services:
      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
    ports:
      - "127.0.0.1:3007:3008"
-      - "9001"
+      - "9003:9001"
    healthcheck:
      test: ["CMD", "nc", "-v", "localhost", "3008"]
      interval: 20s
@@ -35,7 +35,7 @@ services:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
    ports:
-      - "9000"
+      - "9000:9000"
    healthcheck:
      test: ["CMD", "nc", "-v", "localhost", "9000"]
      interval: 20s
@@ -62,7 +62,7 @@ services:
      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
    ports:
      - "127.0.0.1:3008:3008"
-      - "9001"
+      - "9001:9001"
    healthcheck:
      test: ["CMD", "nc", "-v", "localhost", "3008"]
      interval: 20s
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/osmosis/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build:static
+
+# Move build to app-builds
+mkdir -p /app-builds/osmosis
+cp -r ./out /app-builds/osmosis/build
@@ -0,0 +1,16 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  _;
+
+    error_page   500 502 503 504  /50x.html;
+
+    location / {
+        root   /usr/share/nginx/osmosis/build;
+        index index.html index.htm index.nginx-debian.html;
+
+        # First attempt to serve request as file, then as html,
+		# then as directory, then fall back to displaying a 404.
+		try_files $uri $uri.html $uri/ /index.html =404;
+    }
+}
@@ -13,6 +13,9 @@ fi

 yarn build

-# Move build to app-builds so urbit can deploy it
-mkdir /app-builds/uniswap
+# Copy over build and other files to app-builds for urbit deployment
+mkdir -p /app-builds/uniswap
 cp -r ./build /app-builds/uniswap/
+
+cp -r mar /app-builds/uniswap/
+cp desk.docket-0 /app-builds/uniswap/
@@ -1,149 +0,0 @@
-#!/bin/bash
-
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting globs"
-echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading glob files"
-ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT}
-ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT}
-
-uniswap_app_build='/app-builds/uniswap/build'
-uniswap_desk_dir='/urbit/zod/uniswap'
-
-if [ -d ${uniswap_desk_dir} ]; then
-  echo "Uniswap desk dir already exists, skipping deployment..."
-  exit 0
-fi
-
-# Fire curl requests to perform operations on the ship
-dojo () {
-  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
-}
-
-hood () {
-  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
-}
-
-# Create/mount a uniswap desk
-hood "merge %uniswap our %landscape"
-hood "mount %uniswap"
-
-# Loop until the uniswap build appears
-while [ ! -d ${uniswap_app_build} ]; do
-  echo "Uniswap app build not found, retrying in 5s..."
-  sleep 5
-done
-echo "Build found..."
-
-# Copy over build to desk data dir
-cp -r ${uniswap_app_build} ${uniswap_desk_dir}
-
-# Create a mark file for .map file type
-cat << EOF > "${uniswap_desk_dir}/mar/map.hoon"
-::
-::::  /hoon/map/mar
-  ::  Mark for js source maps
-/?    310
-::
-=,  eyre
-|_  mud=@
-++  grow
-  |%
-  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
-  --
-++  grab
-  |%                                                    ::  convert from
-  ++  mime  |=([p=mite q=octs] (@t q.q))
-  ++  noun  cord                                        ::  clam from %noun
-  --
-++  grad  %mime
--
-EOF
-
-# Create a mark file for .woff file type
-cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/woff dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
--
-EOF
-
-# Create a mark file for .ttf file type
-cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/ttf dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
--
-EOF
-
-rm "${uniswap_desk_dir}/desk.bill"
-rm "${uniswap_desk_dir}/desk.ship"
-
-# Commit changes and create a glob
-hood "commit %uniswap"
-dojo "-landscape!make-glob %uniswap /build"
-
-glob_file=$(ls -1 -c zod/.urb/put | head -1)
-echo "Created glob file: ${glob_file}"
-
-upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add)
-glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//')
-
-echo "Glob file uploaded to IFPS:"
-echo "{ cid: ${glob_cid}, filename: ${glob_file} }"
-
-# Curl and wait for the glob to be hosted
-glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}"
-
-echo "Checking if glob file hosted at ${glob_url}"
-while true; do
-  response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url")
-
-  if [ $response -eq 200 ]; then
-    echo "File found at $glob_url"
-    break  # Exit the loop if the file is found
-  else
-    echo "File not found. Retrying in a few seconds..."
-    sleep 5
-  fi
-done
-
-glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/")
-
-# Update the docket file
-cat << EOF > "${uniswap_desk_dir}/desk.docket-0"
-:~  title+'Uniswap'
-    info+'Self-hosted uniswap frontend.'
-    color+0xcd.75df
-    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
-    base+'uniswap'
-    glob-http+['${glob_url}' ${glob_hash}]
-    version+[0 0 1]
-    website+'https://uniswap.org/'
-    license+'MIT'
-==
-EOF
-
-# Commit changes and install the app
-hood "commit %uniswap"
-hood "install our %uniswap"
-
-echo "Uniswap app installed"
@@ -1,110 +0,0 @@
-#!/bin/bash
-
-# $1: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
-# $2: Glob file hash (eg. 0vabcd)
-# $3: Urbit ship's pier dir (default: ./zod)
-
-if [ "$#" -lt 2 ]; then
-  echo "Insufficient arguments"
-  exit 0
-fi
-
-glob_url=$1
-glob_hash=$2
-echo "Using glob file from ${glob_url} with hash ${glob_hash}"
-
-# Default pier dir: ./zod
-# Default desk dir: ./zod/uniswap
-pier_dir="${3:-./zod}"
-uniswap_desk_dir="${pier_dir}/uniswap"
-echo "Using ${uniswap_desk_dir} as the Uniswap desk dir path"
-
-# Fire curl requests to perform operations on the ship
-dojo () {
-  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
-}
-
-hood () {
-  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
-}
-
-# Create/mount a uniswap desk
-hood "merge %uniswap our %landscape"
-hood "mount %uniswap"
-
-# Create a mark file for .map file type
-cat << EOF > "${uniswap_desk_dir}/mar/map.hoon"
-::
-::::  /hoon/map/mar
-  ::  Mark for js source maps
-/?    310
-::
-=,  eyre
-|_  mud=@
-++  grow
-  |%
-  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
-  --
-++  grab
-  |%                                                    ::  convert from
-  ++  mime  |=([p=mite q=octs] (@t q.q))
-  ++  noun  cord                                        ::  clam from %noun
-  --
-++  grad  %mime
--
-EOF
-
-# Create a mark file for .woff file type
-cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/woff dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
--
-EOF
-
-# Create a mark file for .ttf file type
-cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/ttf dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
--
-EOF
-
-rm "${uniswap_desk_dir}/desk.bill"
-rm "${uniswap_desk_dir}/desk.ship"
-
-# Update the docket file
-cat << EOF > "${uniswap_desk_dir}/desk.docket-0"
-:~  title+'Uniswap'
-    info+'Self-hosted uniswap frontend.'
-    color+0xcd.75df
-    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
-    base+'uniswap'
-    glob-http+['${glob_url}' ${glob_hash}]
-    version+[0 0 1]
-    website+'https://uniswap.org/'
-    license+'MIT'
-==
-EOF
-
-# Commit changes and install the app
-hood "commit %uniswap"
-hood "install our %uniswap"
-
-echo "Uniswap app installed"
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# $1: Remote user host
-# $2: Remote Urbit ship's pier dir path (eg. /home/user/zod)
-# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
-# $4: Glob file hash (eg. 0vabcd)
-
-if [ "$#" -ne 4 ]; then
-  echo "Incorrect number of arguments"
-  echo "Usage: $0 <username@remote_host> </path/to/remote/pier/folder> <glob_url> <glob_hash>"
-  exit 1
-fi
-
-remote_user_host="$1"
-remote_pier_folder="$2"
-glob_url="$3"
-glob_hash="$4"
-
-installation_script="./install-uniswap-app.sh"
-
-ssh "$remote_user_host" "bash -s $glob_url $glob_hash $remote_pier_folder" < "$installation_script"
@@ -0,0 +1,10 @@
+:~  title+'Uniswap'
+    info+'Self-hosted uniswap frontend.'
+    color+0xcd.75df
+    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
+    base+'uniswap'
+    glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
+    version+[0 0 1]
+    website+'https://uniswap.org/'
+    license+'MIT'
+==
@@ -0,0 +1,18 @@
+::
+::::  /hoon/map/mar
+  ::  Mark for js source maps
+/?    310
+::
+=,  eyre
+|_  mud=@
++  grow
+  |%
+  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
+  --
++  grab
+  |%                                                    ::  convert from
+  ++  mime  |=([p=mite q=octs] (@t q.q))
+  ++  noun  cord                                        ::  clam from %noun
+  --
++  grad  %mime
+--
@@ -0,0 +1,12 @@
+|_  dat=octs
++  grow
+  |%
+  ++  mime  [/font/ttf dat]
+  --
++  grab
+  |%
+  ++  mime  |=([=mite =octs] octs)
+  ++  noun  octs
+  --
++  grad  %mime
+--
@@ -0,0 +1,12 @@
+|_  dat=octs
++  grow
+  |%
+  ++  mime  [/font/woff dat]
+  --
++  grab
+  |%
+  ++  mime  |=([=mite =octs] octs)
+  ++  noun  octs
+  --
++  grad  %mime
+--
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# $1: Remote user host
+# $2: App name (eg. uniswap)
+# $3: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files)
+# $4: Remote Urbit ship's pier dir path (eg. /home/user/zod)
+# $5: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $6: Glob file hash (eg. 0vabcd)
+
+if [ "$#" -ne 6 ]; then
+  echo "Incorrect number of arguments"
+  echo "Usage: $0 <username@remote_host> <app_name> </path/to/app/assets/folder> </path/to/remote/pier/folder> <glob_url> <glob_hash>"
+  exit 1
+fi
+
+remote_user_host="$1"
+app_name=$2
+app_assets_folder=$3
+remote_pier_folder="$4"
+glob_url="$5"
+glob_hash="$6"
+
+installation_script="./install-urbit-app.sh"
+
+# Copy over the assets to remote machine in a tmp dir
+remote_app_assets_folder=/tmp/urbit-app-assets/$app_name
+ssh "$remote_user_host" "mkdir -p $remote_app_assets_folder"
+scp -r $app_assets_folder/* $remote_user_host:$remote_app_assets_folder
+
+# Run the installation script
+ssh "$remote_user_host" "bash -s $app_name $remote_app_assets_folder '${glob_url}' $glob_hash $remote_pier_folder" < "$installation_script"
+
+# Remove the tmp assets dir
+ssh "$remote_user_host" "rm -rf $remote_app_assets_folder"
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+if [ -z "$CERC_URBIT_APP" ]; then
+  echo "CERC_URBIT_APP not set, exiting"
+  exit 0
+fi
+
+echo "Creating Urbit application for ${CERC_URBIT_APP}"
+
+app_desk_dir=/urbit/zod/${CERC_URBIT_APP}
+if [ -d ${app_desk_dir} ]; then
+  echo "Desk dir already exists for ${CERC_URBIT_APP}, skipping deployment..."
+  exit 0
+fi
+
+app_build=/app-builds/${CERC_URBIT_APP}/build
+app_mark_files=/app-builds/${CERC_URBIT_APP}/mar
+app_docket_file=/app-builds/${CERC_URBIT_APP}/desk.docket-0
+
+echo "Reading app build from ${app_build}"
+echo "Reading additional mark files from ${app_mark_files}"
+echo "Reading docket file ${app_docket_file}"
+
+# Loop until the app's build appears
+while [ ! -d ${app_build} ]; do
+  echo "${CERC_URBIT_APP} app build not found, retrying in 5s..."
+  sleep 5
+done
+echo "Build found..."
+
+echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting the ${CERC_URBIT_APP} glob"
+echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading ${CERC_URBIT_APP} glob"
+ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT}
+ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT}
+
+# Fire curl requests to perform operations on the ship
+dojo () {
+  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
+}
+
+hood () {
+  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
+}
+
+# Create / mount the app's desk
+hood "merge %${CERC_URBIT_APP} our %landscape"
+hood "mount %${CERC_URBIT_APP}"
+
+# Copy over build to desk data dir
+cp -r ${app_build} ${app_desk_dir}
+
+# Copy over the additional mark files
+cp ${app_mark_files}/* ${app_desk_dir}/mar/
+
+rm "${app_desk_dir}/desk.bill"
+rm "${app_desk_dir}/desk.ship"
+
+# Commit changes and create a glob
+hood "commit %${CERC_URBIT_APP}"
+dojo "-landscape!make-glob %${CERC_URBIT_APP} /build"
+
+glob_file=$(ls -1 -c zod/.urb/put | head -1)
+echo "Created glob file: ${glob_file}"
+
+# Upload the glob file to IPFS
+echo "Uploading glob file to ${ipfs_host_endpoint}"
+upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add)
+glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//')
+
+glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}"
+glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/")
+
+echo "Glob file uploaded to IFPS:"
+echo "{ cid: ${glob_cid}, filename: ${glob_file} }"
+echo "{ url: ${glob_url}, hash: ${glob_hash} }"
+
+# Exit if the installation not required
+if [ "$CERC_ENABLE_APP_INSTALL" = "false" ]; then
+  echo "CERC_ENABLE_APP_INSTALL set to false, skipping app installation"
+  exit 0
+fi
+
+# Curl and wait for the glob to be hosted
+echo "Checking if glob file hosted at ${glob_url}"
+while true; do
+  response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url")
+
+  if [ $response -eq 200 ]; then
+    echo "File found at $glob_url"
+    break  # Exit the loop if the file is found
+  else
+    echo "File not found, retrying in a 5s..."
+    sleep 5
+  fi
+done
+
+# Replace the docket file for app
+# Substitue the glob URL and hash
+cp ${app_docket_file} ${app_desk_dir}/
+sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0
+
+# Commit changes and install the app
+hood "commit %${CERC_URBIT_APP}"
+hood "install our %${CERC_URBIT_APP}"
+
+echo "${CERC_URBIT_APP} app installed"
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# $1: App name (eg. uniswap)
+# $2: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files)
+# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $4: Glob file hash (eg. 0vabcd)
+# $5: Urbit ship's pier dir (default: ./zod)
+
+if [ "$#" -lt 4 ]; then
+  echo "Insufficient arguments"
+  echo "Usage: $0 <app_name> </path/to/app/assets/folder> <glob_url> <glob_hash> [/path/to/remote/pier/folder]"
+  exit 1
+fi
+
+app_name=$1
+app_mark_files=$2/mar
+app_docket_file=$2/desk.docket-0
+echo "Creating Urbit application for ${app_name}"
+echo "Reading additional mark files from ${app_mark_files}"
+echo "Reading docket file ${app_docket_file}"
+
+glob_url=$3
+glob_hash=$4
+echo "Using glob file from ${glob_url} with hash ${glob_hash}"
+
+# Default pier dir: ./zod
+# Default desk dir: ./zod/<app_name>
+pier_dir="${5:-./zod}"
+app_desk_dir="${pier_dir}/${app_name}"
+echo "Using ${app_desk_dir} as the ${app_name} desk dir path"
+
+# Fire curl requests to perform operations on the ship
+dojo () {
+  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
+}
+
+hood () {
+  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
+}
+
+# Create / mount the app's desk
+hood "merge %${app_name} our %landscape"
+hood "mount %${app_name}"
+
+# Copy over the additional mark files
+cp ${app_mark_files}/* ${app_desk_dir}/mar/
+
+rm "${app_desk_dir}/desk.bill"
+rm "${app_desk_dir}/desk.ship"
+
+# Replace the docket file for app
+# Substitue the glob URL and hash
+cp ${app_docket_file} ${app_desk_dir}/
+sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0
+
+# Commit changes and install the app
+hood "commit %${app_name}"
+hood "install our %${app_name}"
+
+echo "${app_name} app installed"
@@ -7,11 +7,13 @@ fi

 pier_dir="/urbit/zod"

+# TODO: Bootstrap fake ship on the first run
+
 # Run urbit ship in daemon mode
 # Check if the directory exists
 if [ -d "$pier_dir" ]; then
  echo "Pier directory already exists, rebooting..."
-  urbit -d zod
+  /urbit/zod/.run -d
 else
  echo "Creating a new fake ship..."
  urbit -d -F zod
@@ -41,7 +41,7 @@
    timeTravelMaxAge = 86400 # 1 day

 [metrics]
-  host = "127.0.0.1"
+  host = "0.0.0.0"
  port = 9000
  [metrics.gql]
    port = 9001
@@ -41,7 +41,7 @@
    timeTravelMaxAge = 86400 # 1 day

 [metrics]
-  host = "127.0.0.1"
+  host = "0.0.0.0"
  port = 9000
  [metrics.gql]
    port = 9001
@@ -0,0 +1,14 @@
+record:
+  type: ApplicationDeploymentRecord
+  version: 1.2.3
+  name: name
+  description: description
+  application: application
+  url: url
+  dns: dns
+  request: request
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b 
@@ -0,0 +1,12 @@
+record:
+  type: GeneralRecord
+  version: 1.2.3
+  name: name
+  description: description
+  category: category
+  value: value
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b 
@@ -0,0 +1,17 @@
+record:
+  type: ApplicationArtifact
+  version: 1.2.3
+  name: name
+  description: description
+  application: appidgoeshere
+  content_type: content_type
+  os: os
+  cpu: cpu
+  uri:
+    - uri://a
+    - uri://b
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b
@@ -0,0 +1,11 @@
+record:
+  type: DnsRecord
+  version: 0.0.1
+  name: "foo"
+  resource_type: "A"
+  value: "bar"
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b
@@ -0,0 +1,18 @@
+record:
+  type: ApplicationRecord
+  version: 0.0.1
+  name: my-demo-app
+  description: "Description of my app"
+  homepage: http://my.demo.app
+  license: license
+  author: author
+  repository:
+    - "https://my.demo.repo"
+  repository_ref: "v0.1.0"
+  app_version: "0.1.0"
+  app_type: "webapp"
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b
@@ -0,0 +1,17 @@
+record:
+  type: ApplicationDeploymentRequest
+  version: 1.2.3
+  application: application
+  dns: dns
+  config:
+    env:
+      ENV_VAR_A: A
+      ENV_VAR_B: B
+  crn:
+    - crn://foo.bar
+    - crn://bar.baz
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the osmosis front end image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend
@@ -0,0 +1,151 @@
+# self-hosted osmosis
+
+Instructions to build and deploy:
+- self-hosted gitea
+- an ipfs node
+- the osmosis front end
+- a laconicd chain
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack osmosis setup-repositories
+
+# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command
+```
+
+Build the container images:
+
+```bash
+# TODO: support image for the gitea package registry
+# laconic-so --stack build-support build-containers
+
+laconic-so --stack osmosis build-containers
+```
+
+## Create a deployment
+
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+```bash
+laconic-so --stack osmosis deploy init --output osmosis-spec.yml
+```
+
+### Ports
+
+Edit `network` in spec file to map container ports to same ports in host
+
+```
+...
+network:
+  ports:
+    proxy-server:
+      - '4000:4000'
+    nginx:
+      - '3000:80'
+```
+
+### Data volumes
+
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+
+Once you've made any needed changes to the spec file, create a deployment from it:
+```bash
+laconic-so --stack osmosis deploy create --spec-file osmosis-spec.yml --deployment-dir osmosis-deployment
+```
+
+## Set env variables
+
+Inside the deployment directory, open the file `config.env` and set the following env variables:
+
+  (Note: Following config can be used as is if the stack is being run locally)
+
+  ```bash
+  # Osmosis API base URL
+  # Set this to proxy server endpoint for osmosis app
+  # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled)
+  # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint)
+  CERC_WEB_API_BASE_URL=http://localhost:4000
+
+  # Optional
+
+  # Whether to run the proxy server
+  # (Disable only if proxy not required to be run) (Default: true)
+  CERC_ENABLE_PROXY=
+
+  # Proxy server configuration
+  # Used only if proxy is enabled
+
+  # Upstream API URL
+  CERC_PROXY_UPSTREAM=https://app.osmosis.zone
+  ```
+
+## Start the stack
+
+Start the deployment:
+
+```bash
+laconic-so deployment --dir osmosis-deployment start
+```
+
+* The `osmosis-front-end` container will take some time to run to completion as it builds the front-end app with given configuration; same can be tracked in the logs:
+
+  ```bash
+  laconic-so deployment --dir osmosis-deployment logs -f osmosis-front-end
+  ```
+
+* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
+
+* The web app can be accessed at http://localhost:3000
+
+## Laconic registry
+
+Setup a test chain:
+
+```bash
+export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
+
+laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+
+laconic-so --stack fixturenet-laconic-loaded build-containers
+
+export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
+
+laconic-so --stack fixturenet-laconic-loaded deploy up
+```
+
+then `docker exec` into the `laconicd` container and either export the private key or create a new one and send funds to it. Use that private key for `LACONIC_HOTWALLET_KEY`.
+
+## Clean up
+
+To stop all osmosis services running in the background, while preserving data:
+
+```bash
+# Only stop the docker containers
+laconic-so deployment --dir osmosis-deployment stop
+
+# Run 'start' to restart the deployment
+```
+
+To stop all osmosis services and also delete data:
+
+```bash
+# Stop the docker containers
+laconic-so deployment --dir osmosis-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r osmosis-deployment
+```
+
+To stop stack running Laconic registry
+
+```bash
+laconic-so --stack fixturenet-laconic-loaded deploy down
+
+# Delete volumes
+laconic-so --stack fixturenet-laconic-loaded deploy down --delete-volumes
+```
@@ -0,0 +1,30 @@
+version: "0.1"
+name: osmosis
+repos:
+  # For gitea
+  # - git.vdb.to/cerc-io/hosting@names-for-so
+  # - gitea.com/gitea/act_runner
+  # For osmosis frontend
+  - github.com/cerc-io/watcher-ts@v0.2.78
+  - github.com/cerc-io/osmosis-frontend@laconic # TODO: Use release
+containers:
+  # - cerc/act-runner
+  # - cerc/act-runner-task-executor
+  - cerc/watcher-ts
+  - cerc/osmosis-front-end
+pods:
+  - kubo
+  - proxy-server
+  - osmosis-front-end
+  # TODO: Fix pod config for deployment command
+  # TODO: mirroring all of osmosis repos: https://git.vdb.to/cerc-io/hosting/pulls/42
+  # - name: gitea
+  #   repository: cerc-io/hosting
+  #   path: gitea
+  #   pre_start_command: "run-this-first.sh"
+  #   post_start_command: "initialize-gitea.sh"
+  # - name: act-runner
+  #   repository: cerc-io/hosting
+  #   path: act-runner
+  #   pre_start_command: "pre_start.sh"
+  #   post_start_command: "post_start.sh"
@@ -47,7 +47,7 @@ laconic-so --stack proxy-server build-containers

  ```bash
  # Whether to run the proxy server (Optional) (Default: true)
-  ENABLE_PROXY=
+  CERC_ENABLE_PROXY=

  # Upstream endpoint
  # (Eg. https://api.example.org)
@@ -33,7 +33,7 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec

 ### Ports

-Edit `network` in spec file to map container ports to same ports in host
+Edit `network` in spec file to map container ports to same ports in host:

 ```
 ...
@@ -43,12 +43,14 @@ network:
      - '8080:80'
    proxy-server:
      - '4000:4000'
-    ipfs-glob-host:
+    ipfs:
      - '8081:8080'
      - '5001:5001'
 ...
 ```

+Note: Skip the `ipfs` ports if need to use an externally running IPFS node
+
 ### Data volumes

 Container data volumes are bind-mounted to specified paths in the host filesystem.
@@ -67,21 +69,28 @@ laconic-so --stack uniswap-urbit-app deploy create --spec-file uniswap-urbit-app
 Inside the deployment directory, open the file `config.env` and set the following env variables:

  ```bash
+  # App to be installed (Do not change)
+  CERC_URBIT_APP=uniswap
+
  # External RPC endpoints
  # https://docs.infura.io/getting-started#2-create-an-api-key
  CERC_INFURA_KEY=

  # Uniswap API GQL Endpoint
  # Set this to GQL proxy server endpoint for uniswap app
-  # (Eg. http://localhost:4000/v1/graphql)
-  # (Eg. https://abc.xyz.com/v1/graphql)
-  CERC_UNISWAP_GQL=
+  # (Eg. http://localhost:4000/v1/graphql - in case stack is being run locally with proxy enabled)
+  # (Eg. https://abc.xyz.com/v1/graphql - in case https://abc.xyz.com is pointed to the proxy endpoint)
+  CERC_UNISWAP_GQL=http://localhost:4000/v1/graphql

  # Optional

+  # Whether to enable app installation on Urbit
+  # (just builds and uploads the glob file if disabled) (Default: true)
+  CERC_ENABLE_APP_INSTALL=
+
  # Whether to run the proxy GQL server
-  # (Disable only if proxy not required to be run) (Default: true)
-  ENABLE_PROXY=
+  # (disable only if proxy not required to be run) (Default: true)
+  CERC_ENABLE_PROXY=

  # Proxy server configuration
  # Used only if proxy is enabled
@@ -97,11 +106,11 @@ Inside the deployment directory, open the file `config.env` and set the followin
  # IPFS configuration

  # IFPS endpoint to host the glob file on
-  # (Default: http://ipfs-glob-host:5001 pointing to in-stack IPFS node)
+  # (Default: http://ipfs:5001 pointing to in-stack IPFS node)
  CERC_IPFS_GLOB_HOST_ENDPOINT=

  # IFPS endpoint to fetch the glob file from
-  # (Default: http://ipfs-glob-host:8080 pointing to in-stack IPFS node)
+  # (Default: http://ipfs:8080 pointing to in-stack IPFS node)
  CERC_IPFS_SERVER_ENDPOINT=
  ```

@@ -141,11 +150,18 @@ laconic-so deployment --dir uniswap-urbit-app-deployment start
 To stop all uniswap-urbit-app services running in the background, while preserving data:

 ```bash
+# Only stop the docker containers
 laconic-so deployment --dir uniswap-urbit-app-deployment stop
+
+# Run 'start' to restart the deployment
 ```

 To stop all uniswap-urbit-app services and also delete data:

 ```bash
+# Stop the docker containers
 laconic-so deployment --dir uniswap-urbit-app-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r uniswap-urbit-app-deployment
 ```
@@ -10,4 +10,5 @@ containers:
 pods:
  - uniswap-interface
  - proxy-server
-  - uniswap-urbit
+  - fixturenet-urbit
+  - kubo
@@ -40,6 +40,12 @@ class DockerDeployer(Deployer):
        except DockerException as e:
            raise DeployerException(e)

+    def update(self):
+        try:
+            return self.docker.compose.restart()
+        except DockerException as e:
+            raise DeployerException(e)
+
    def status(self):
        try:
            for p in self.docker.compose.ps():
@@ -120,6 +120,14 @@ def status_operation(ctx):
        ctx.obj.deployer.status()


+def update_operation(ctx):
+    global_context = ctx.parent.parent.obj
+    if not global_context.dry_run:
+        if global_context.verbose:
+            print("Running compose update")
+        ctx.obj.deployer.update()
+
+
 def ps_operation(ctx):
    global_context = ctx.parent.parent.obj
    if not global_context.dry_run:
@@ -27,6 +27,10 @@ class Deployer(ABC):
    def down(self, timeout, volumes):
        pass

+    @abstractmethod
+    def update(self):
+        pass
+
    @abstractmethod
    def ps(self):
        pass
@@ -19,7 +19,7 @@ import sys
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.images import push_images_operation
 from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation
-from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context
+from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context, update_operation
 from stack_orchestrator.deploy.deploy_types import DeployCommandContext
 from stack_orchestrator.deploy.deployment_context import DeploymentContext

@@ -149,3 +149,10 @@ def logs(ctx, tail, follow, extra_args):
 def status(ctx):
    ctx.obj = make_deploy_context(ctx)
    status_operation(ctx)
+
+
+@command.command()
+@click.pass_context
+def update(ctx):
+    ctx.obj = make_deploy_context(ctx)
+    update_operation(ctx)
@@ -189,6 +189,7 @@ class ClusterInfo:
                container = client.V1Container(
                    name=container_name,
                    image=image_to_use,
+                    image_pull_policy="Always",
                    env=envs_from_environment_variables_map(self.environment_variables.map),
                    ports=[client.V1ContainerPort(container_port=port)],
                    volume_mounts=volume_mounts,
@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

+from datetime import datetime, timezone
+
 from pathlib import Path
 from kubernetes import client, config

@@ -304,6 +306,33 @@ class K8sDeployer(Deployer):
        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
        return log_stream_from_string(log_data)

+    def update(self):
+        self.connect_api()
+        ref_deployment = self.cluster_info.get_deployment()
+
+        deployment = self.apps_api.read_namespaced_deployment(
+            name=ref_deployment.metadata.name,
+            namespace=self.k8s_namespace
+        )
+
+        new_env = ref_deployment.spec.template.spec.containers[0].env
+        for container in deployment.spec.template.spec.containers:
+            old_env = container.env
+            if old_env != new_env:
+                container.env = new_env
+
+        deployment.spec.template.metadata.annotations = {
+            "kubectl.kubernetes.io/restartedAt": datetime.utcnow()
+            .replace(tzinfo=timezone.utc)
+            .isoformat()
+        }
+
+        self.apps_api.patch_namespaced_deployment(
+            name=ref_deployment.metadata.name,
+            namespace=self.k8s_namespace,
+            body=deployment
+        )
+
    def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False):
        # We need to figure out how to do this -- check why we're being called first
        pass
@@ -54,26 +54,7 @@ def _fixup_url_spec(spec_file_name: str, url: str):
        wfile.write(contents)


-@click.group()
-@click.pass_context
-def command(ctx):
-    '''manage a webapp deployment'''
-
-    # Check that --stack wasn't supplied
-    if ctx.parent.obj.stack:
-        error_exit("--stack can't be supplied with the deploy-webapp command")
-
-
-@command.command()
-@click.option("--kube-config", help="Provide a config file for a k8s deployment")
-@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
-@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
-@click.option("--image", help="image to deploy", required=True)
-@click.option("--url", help="url to serve", required=True)
-@click.option("--env-file", help="environment file for webapp")
-@click.pass_context
-def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
-    '''create a deployment for the specified webapp container'''
+def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
    # Do the equivalent of:
    # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
    #   --config (eqivalent of the contents of my-config.env)
@@ -92,7 +73,7 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil
    # TODO: support env file
    deploy_command_context: DeployCommandContext = create_deploy_context(
        global_options2(ctx), None, stack, None, None, None, env_file, "k8s"
-        )
+    )
    init_operation(
        deploy_command_context,
        stack,
@@ -116,3 +97,27 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil
    # Fix up the container tag inside the deployment compose file
    _fixup_container_tag(deployment_dir, image)
    os.remove(spec_file_name)
+
+
+@click.group()
+@click.pass_context
+def command(ctx):
+    '''manage a webapp deployment'''
+
+    # Check that --stack wasn't supplied
+    if ctx.parent.obj.stack:
+        error_exit("--stack can't be supplied with the deploy-webapp command")
+
+
+@command.command()
+@click.option("--kube-config", help="Provide a config file for a k8s deployment")
+@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
+@click.option("--deployment-dir", help="Create deployment files in this directory", required=True)
+@click.option("--image", help="image to deploy", required=True)
+@click.option("--url", help="url to serve", required=True)
+@click.option("--env-file", help="environment file for webapp")
+@click.pass_context
+def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
+    '''create a deployment for the specified webapp container'''
+
+    return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file)
@@ -0,0 +1,270 @@
+# Copyright ©2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import json
+import os
+import shlex
+import shutil
+import sys
+import tempfile
+
+import click
+
+from stack_orchestrator.deploy.webapp import deploy_webapp
+from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
+                                                   build_container_image, push_container_image,
+                                                   file_hash, deploy_to_k8s, publish_deployment,
+                                                   hostname_for_deployment_request, generate_hostname_for_app)
+
+
+def process_app_deployment_request(
+    ctx,
+    laconic: LaconicRegistryClient,
+    app_deployment_request,
+    deployment_record_namespace,
+    dns_record_namespace,
+    dns_suffix,
+    deployment_parent_dir,
+    kube_config,
+    image_registry
+):
+    # 1. look up application
+    app = laconic.get_record(app_deployment_request.attributes.application, require=True)
+
+    # 2. determine dns
+    requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
+
+    # HACK
+    if "." in requested_name:
+        raise Exception("Only unqualified hostnames allowed at this time.")
+
+    fqdn = f"{requested_name}.{dns_suffix}"
+    container_tag = "%s:local" % app.attributes.name.replace("@", "")
+
+    # 3. check ownership of existing dnsrecord vs this request
+    # TODO: Support foreign DNS
+    dns_crn = f"{dns_record_namespace}/{fqdn}"
+    dns_record = laconic.get_record(dns_crn)
+    if dns_record:
+        dns_record_owners = dns_record.owners
+        dns_request_owners = []
+        if dns_record.request:
+            prev_request = laconic.get_record(dns_record.request, require=True)
+            dns_request_owners = prev_request.owners
+
+        owner_match = None
+
+        for owner in app_deployment_request.owners:
+            if owner in dns_request_owners or owner in dns_record_owners:
+                owner_match = owner
+        if owner_match:
+            print("Matched DnsRecord ownership to", owner)
+        else:
+            raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
+                            (dns_record.id, app_deployment_request.id))
+
+    # 4. get build and runtime config from request
+    env_filename = None
+    if app_deployment_request.attributes.config and "env" in app_deployment_request.attributes.config:
+        env_filename = tempfile.mktemp()
+        with open(env_filename, 'w') as file:
+            for k, v in app_deployment_request.attributes.config["env"].items():
+                file.write("%s=%s\n" % (k, shlex.quote(str(v))))
+
+    # 5. determine new or existing deployment
+    #   a. check for deployment crn
+    app_deployment_crn = f"{deployment_record_namespace}/{fqdn}"
+    if app_deployment_request.attributes.deployment:
+        app_deployment_crn = app_deployment_request.attributes.deployment
+    if not app_deployment_crn.startswith(deployment_record_namespace):
+        raise Exception("Deployment CRN %s is not in a supported namespace" % app_deployment_request.attributes.deployment)
+
+    deployment_record = laconic.get_record(app_deployment_crn)
+    deployment_dir = os.path.join(deployment_parent_dir, fqdn)
+    deployment_config_file = os.path.join(deployment_dir, "config.env")
+    #   b. check for deployment directory (create if necessary)
+    if not os.path.exists(deployment_dir):
+        if deployment_record:
+            raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." %
+                   (app_deployment_crn, deployment_dir))
+        print("deploy_webapp", deployment_dir)
+        deploy_webapp.create_deployment(ctx, deployment_dir, container_tag,
+                                        f"https://{fqdn}", kube_config, image_registry, env_filename)
+    elif env_filename:
+        shutil.copyfile(env_filename, deployment_config_file)
+
+    needs_k8s_deploy = False
+    # 6. build container (if needed)
+    if not deployment_record or deployment_record.attributes.application != app.id:
+        build_container_image(app, container_tag)
+        push_container_image(deployment_dir)
+        needs_k8s_deploy = True
+
+    # 7. update config (if needed)
+    if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
+        needs_k8s_deploy = True
+
+    # 8. update k8s deployment
+    if needs_k8s_deploy:
+        print("Deploying to k8s")
+        deploy_to_k8s(
+            deployment_record,
+            deployment_dir,
+        )
+
+    publish_deployment(
+        laconic,
+        app,
+        deployment_record,
+        app_deployment_crn,
+        dns_record,
+        dns_crn,
+        deployment_dir,
+        app_deployment_request
+    )
+
+
+def load_known_requests(filename):
+    if filename and os.path.exists(filename):
+        return json.load(open(filename, "r"))
+    return {}
+
+
+def dump_known_requests(filename, requests):
+    if not filename:
+        return
+    known_requests = load_known_requests(filename)
+    for r in requests:
+        known_requests[r.id] = r.createTime
+    json.dump(known_requests, open(filename, "w"))
+
+
+@click.command()
+@click.option("--kube-config", help="Provide a config file for a k8s deployment")
+@click.option("--kube-config", help="Provide a config file for a k8s deployment")
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
+@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
+@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
+@click.option("--request-id", help="The ApplicationDeploymentRequest to process")
+@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRequests", is_flag=True, default=False)
+@click.option("--state-file", help="File to store state about previously seen requests.")
+@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
+@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world")
+@click.option("--record-namespace-dns", help="eg, crn://laconic/dns")
+@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments")
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
+@click.pass_context
+def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir,
+            request_id, discover, state_file, only_update_state,
+            dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run):
+    if request_id and discover:
+        print("Cannot specify both --request-id and --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if not request_id and not discover:
+        print("Must specify either --request-id or --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if only_update_state and not state_file:
+        print("--only-update-state requires --state-file", file=sys.stderr)
+        sys.exit(2)
+
+    if not only_update_state:
+        if not record_namespace_dns or not record_namespace_deployments or not dns_suffix:
+            print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
+            sys.exit(2)
+
+    laconic = LaconicRegistryClient(laconic_config)
+
+    # Find deployment requests.
+    # single request
+    if request_id:
+        requests = [laconic.get_record(request_id, require=True)]
+    # all requests
+    elif discover:
+        requests = laconic.app_deployment_requests()
+
+    if only_update_state:
+        if not dry_run:
+            dump_known_requests(state_file, requests)
+        return
+
+    previous_requests = load_known_requests(state_file)
+
+    # Collapse related requests.
+    requests.sort(key=lambda r: r.createTime)
+    requests.reverse()
+    requests_by_name = {}
+    for r in requests:
+        app = laconic.get_record(r.attributes.application)
+        if not app:
+            print("Skipping request %s, cannot locate app." % r.id)
+            continue
+
+        requested_name = r.attributes.dns
+        if not requested_name:
+            requested_name = generate_hostname_for_app(app)
+            print("Generating name %s for request %s." % (requested_name, r.id))
+
+        if requested_name not in requests_by_name:
+            print(
+                "Found request %s to run application %s on %s."
+                % (r.id, r.attributes.application, requested_name)
+            )
+            requests_by_name[requested_name] = r
+        else:
+            print(
+                "Ignoring request %s, it is superseded by %s."
+                % (r.id, requests_by_name[requested_name].id)
+            )
+
+    # Find deployments.
+    deployments = laconic.app_deployments()
+    deployments_by_request = {}
+    for d in deployments:
+        if d.attributes.request:
+            deployments_by_request[d.attributes.request] = d
+
+    requests_to_execute = []
+    for r in requests_by_name.values():
+        if r.id not in deployments_by_request:
+            if r.id not in previous_requests:
+                print(f"Request {r.id} needs to processed.")
+                requests_to_execute.append(r)
+            else:
+                print(
+                    f"Skipping unsatisfied request {r.id} because we have seen it before."
+                )
+        else:
+            print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")
+
+    print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
+
+    if not dry_run:
+        for r in requests_to_execute:
+            try:
+                process_app_deployment_request(
+                    ctx,
+                    laconic,
+                    r,
+                    record_namespace_deployments,
+                    record_namespace_dns,
+                    dns_suffix,
+                    deployment_parent_dir,
+                    kube_config,
+                    image_registry
+                )
+            finally:
+                dump_known_requests(state_file, [r])
@@ -0,0 +1,305 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import hashlib
+import json
+import os
+import random
+import subprocess
+import sys
+import tempfile
+import uuid
+
+import yaml
+
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+    def __getattribute__(self, attr):
+        __dict__ = super(AttrDict, self).__getattribute__("__dict__")
+        if attr in __dict__:
+            v = super(AttrDict, self).__getattribute__(attr)
+            if isinstance(v, dict):
+                return AttrDict(v)
+            return v
+
+
+def cmd(*vargs):
+    try:
+        result = subprocess.run(vargs, capture_output=True)
+        result.check_returncode()
+        return result.stdout.decode()
+    except Exception as err:
+        print(result.stderr.decode())
+        raise err
+
+
+class LaconicRegistryClient:
+    def __init__(self, config_file):
+        self.config_file = config_file
+        self.cache = AttrDict(
+            {
+                "name_or_id": {},
+            }
+        )
+
+    def list_records(self, criteria={}, all=False):
+        args = ["laconic", "-c", self.config_file, "cns", "record", "list"]
+
+        if all:
+            args.append("--all")
+
+        if criteria:
+            for k, v in criteria.items():
+                args.append("--%s" % k)
+                args.append(str(v))
+
+        results = [AttrDict(r) for r in json.loads(cmd(*args))]
+
+        # Most recent records first
+        results.sort(key=lambda r: r.createTime)
+        results.reverse()
+
+        return results
+
+    def is_crn(self, name_or_id: str):
+        if name_or_id:
+            return str(name_or_id).startswith("crn://")
+        return False
+
+    def is_id(self, name_or_id: str):
+        return not self.is_crn(name_or_id)
+
+    def _add_to_cache(self, records):
+        if not records:
+            return
+
+        for p in records:
+            self.cache["name_or_id"][p.id] = p
+            if p.names:
+                for crn in p.names:
+                    self.cache["name_or_id"][crn] = p
+            if p.attributes.type not in self.cache:
+                self.cache[p.attributes.type] = []
+            self.cache[p.attributes.type].append(p)
+
+    def resolve(self, name):
+        if not name:
+            return None
+
+        if name in self.cache.name_or_id:
+            return self.cache.name_or_id[name]
+
+        args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name]
+
+        parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
+        if parsed:
+            self._add_to_cache(parsed)
+            return parsed[0]
+
+        return None
+
+    def get_record(self, name_or_id, require=False):
+        if not name_or_id:
+            if require:
+                raise Exception("Cannot locate record:", name_or_id)
+            return None
+
+        if name_or_id in self.cache.name_or_id:
+            return self.cache.name_or_id[name_or_id]
+
+        if self.is_crn(name_or_id):
+            return self.resolve(name_or_id)
+
+        args = [
+            "laconic",
+            "-c",
+            self.config_file,
+            "cns",
+            "record",
+            "get",
+            "--id",
+            name_or_id,
+        ]
+
+        parsed = [AttrDict(r) for r in json.loads(cmd(*args))]
+        if len(parsed):
+            self._add_to_cache(parsed)
+            return parsed[0]
+
+        if require:
+            raise Exception("Cannot locate record:", name_or_id)
+        return None
+
+    def app_deployment_requests(self):
+        return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
+
+    def app_deployments(self):
+        return self.list_records({"type": "ApplicationDeploymentRecord"})
+
+    def publish(self, record, names=[]):
+        tmpdir = tempfile.mkdtemp()
+        try:
+            record_fname = os.path.join(tmpdir, "record.yml")
+            record_file = open(record_fname, 'w')
+            yaml.dump(record, record_file)
+            record_file.close()
+            print(open(record_fname, 'r').read())
+
+            new_record_id = json.loads(
+                cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
+            )["id"]
+            for name in names:
+                cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
+            return new_record_id
+        finally:
+            cmd("rm", "-rf", tmpdir)
+
+
+def file_hash(filename):
+    return hashlib.sha1(open(filename).read().encode()).hexdigest()
+
+
+def build_container_image(app_record, tag, extra_build_args=[]):
+    tmpdir = tempfile.mkdtemp()
+
+    try:
+        record_id = app_record["id"]
+        ref = app_record.attributes.repository_ref
+        repo = random.choice(app_record.attributes.repository)
+        clone_dir = os.path.join(tmpdir, record_id)
+
+        print(f"Cloning repository {repo} to {clone_dir} ...")
+        if ref:
+            # TODO: Determing branch or hash, and use depth 1 if we can.
+            result = subprocess.run(["git", "clone", repo, clone_dir])
+            result.check_returncode()
+            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
+        else:
+            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
+            result.check_returncode()
+
+        print("Building webapp ...")
+        build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag]
+        if extra_build_args:
+            build_command.append("--extra-build-args")
+            build_command.append(" ".join(extra_build_args))
+
+        result = subprocess.run(build_command)
+        result.check_returncode()
+    finally:
+        cmd("rm", "-rf", tmpdir)
+
+
+def push_container_image(deployment_dir):
+    print("Pushing image ...")
+    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"])
+    result.check_returncode()
+
+
+def deploy_to_k8s(deploy_record, deployment_dir):
+    if not deploy_record:
+        command = "up"
+    else:
+        command = "update"
+
+    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command])
+    result.check_returncode()
+
+
+def publish_deployment(laconic: LaconicRegistryClient,
+                       app_record,
+                       deploy_record,
+                       deployment_crn,
+                       dns_record,
+                       dns_crn,
+                       deployment_dir,
+                       app_deployment_request=None):
+    if not deploy_record:
+        deploy_ver = "0.0.1"
+    else:
+        deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1)
+
+    if not dns_record:
+        dns_ver = "0.0.1"
+    else:
+        dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1)
+
+    spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml")))
+    fqdn = spec["network"]["http-proxy"][0]["host-name"]
+
+    uniq = uuid.uuid4()
+
+    new_dns_record = {
+        "record": {
+            "type": "DnsRecord",
+            "version": dns_ver,
+            "name": fqdn,
+            "resource_type": "A",
+            "meta": {
+                "so": uniq.hex
+            },
+        }
+    }
+    if app_deployment_request:
+        new_dns_record["record"]["request"] = app_deployment_request.id
+
+    dns_id = laconic.publish(new_dns_record, [dns_crn])
+
+    new_deployment_record = {
+        "record": {
+            "type": "ApplicationDeploymentRecord",
+            "version": deploy_ver,
+            "url": f"https://{fqdn}",
+            "name": app_record.attributes.name,
+            "application": app_record.id,
+            "dns": dns_id,
+            "meta": {
+                "config": file_hash(os.path.join(deployment_dir, "config.env")),
+                "so": uniq.hex
+            },
+        }
+    }
+    if app_deployment_request:
+        new_deployment_record["record"]["request"] = app_deployment_request.id
+
+    deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
+    return {"dns": dns_id, "deployment": deployment_id}
+
+
+def hostname_for_deployment_request(app_deployment_request, laconic):
+    dns_name = app_deployment_request.attributes.dns
+    if not dns_name:
+        app = laconic.get_record(app_deployment_request.attributes.application, require=True)
+        dns_name = generate_hostname_for_app(app)
+    elif dns_name.startswith("crn://"):
+        record = laconic.get_record(dns_name, require=True)
+        dns_name = record.attributes.name
+    return dns_name
+
+
+def generate_hostname_for_app(app):
+    last_part = app.attributes.name.split("/")[-1]
+    m = hashlib.sha256()
+    m.update(app.attributes.name.encode())
+    m.update(b"|")
+    if isinstance(app.attributes.repository, list):
+        m.update(app.attributes.repository[0].encode())
+    else:
+        m.update(app.attributes.repository.encode())
+    return "%s-%s" % (last_part, m.hexdigest()[0:10])
@@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
-from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp
+from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -53,6 +53,7 @@ cli.add_command(build_npms.command, "build-npms")
 cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")
 cli.add_command(deploy_webapp.command, "deploy-webapp")
+cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")
Author	SHA1	Message	Date
telackey	1f4eb57069	Add --dry-run option (#686 )	2023-12-13 22:56:40 -06:00
telackey	88f66a3626	Add `deployment update` and `deploy-webapp-from-registry` commands. (#676 )	2023-12-13 21:02:34 -06:00
prathamesh0	1ef0b316c6	Expose metrics endpoints for sushiswap and merkl sushiswap watchers (#685 )	2023-12-13 14:58:26 +05:30
prathamesh0	232d5618cb	Update instructions in osmosis stack (#684 )	2023-12-12 13:54:00 +05:30
nabarun zramsay prathamesh	fa6b570f4a	Add stack for running osmosis frontend app (#673 ) * osmosis FE stack * chmod * dont use 3000 * fix for neww stack format * updates * update osmosis readme * Update stack.yml * Update osmosis frontend stack to serve app * Host osmosis app static build using python server * Fix mapped ports in deployment for containers * Update instructions * Use nginx server to host files and handle page reloads * Fix typo --------- Co-authored-by: zramsay <zach@bluecollarcoding.ca> Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2023-12-11 14:10:54 +05:30
prathamesh0	f9eb5a4ba8	Refactor to make Urbit setup generic (#682 ) * Refactor to make Urbit app deployment script generic * Rename urbit pod and update instructions * Add a flag to allow skipping app installation on Urbit * Make remote Urbit app deployment scripts generic * Move remote deployment scripts to urbit fixturenet * Update and use existing kubo pod for Urbit glob hosting	2023-12-08 09:35:00 +05:30