Use IPFS for hosting glob files for Urbit (#677)

* Use IPFS for hosting glob files for Urbit

* Add env configuration for IPFS endpoints to instructions

* Make ship pier dir configurable in remote deployment script

* Update remote deployment script to accept glob hash arg

stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml

@@ -12,25 +12,6 @@ services:
       - app_builds:/app-builds
       - ../config/uniswap-interface/build-app.sh:/app/build-app.sh
 
-  uniswap-glob-host:
-    image: cerc/urbit-globs-host:local
-    restart: unless-stopped
-    depends_on:
-      uniswap-interface:
-        condition: service_completed_successfully
-    command: ["./host-uniswap-glob.sh"]
-    volumes:
-      - app_globs:/app-globs
-      - ../config/uniswap-interface/host-uniswap-glob.sh:/app/host-uniswap-glob.sh
-    ports:
-      - "3000"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3000"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
   uniswap-gql-proxy:
     image: cerc/uniswap-interface:local
     restart: on-failure

stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml

@@ -4,6 +4,9 @@ services:
   urbit-fake-ship:
     restart: unless-stopped
     image: tloncorp/vere
+    environment:
+      CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs-glob-host:5001}
+      CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs-glob-host:8080}
     entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-uniswap-app.sh && tail -f /dev/null"]
     volumes:
       - urbit_data:/urbit
@@ -20,7 +23,24 @@ services:
       retries: 15
       start_period: 10s
 
+  ipfs-glob-host:
+    image: ipfs/kubo:master-2023-02-20-714a968
+    volumes:
+      - ipfs-import:/import
+      - ipfs-data:/data/ipfs
+    ports:
+      - "8080"
+      - "5001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
 volumes:
   urbit_data:
   app_builds:
   app_globs:
+  ipfs-import:
+  ipfs-data:

stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh

@@ -4,6 +4,11 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
+echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting globs"
+echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading glob files"
+ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT}
+ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT}
+
 uniswap_app_build='/app-builds/uniswap/build'
 uniswap_desk_dir='/urbit/zod/uniswap'
 
@@ -96,15 +101,17 @@ rm "${uniswap_desk_dir}/desk.ship"
 hood "commit %uniswap"
 dojo "-landscape!make-glob %uniswap /build"
 
-echo "Copying over glob file to mounted volume"
-mkdir -p /app-globs/uniswap
-cp /urbit/zod/.urb/put/* /app-globs/uniswap/
-
 glob_file=$(ls -1 -c zod/.urb/put | head -1)
-echo "Glob filename: ${glob_file}"
+echo "Created glob file: ${glob_file}"
+
+upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add)
+glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//')
+
+echo "Glob file uploaded to IFPS:"
+echo "{ cid: ${glob_cid}, filename: ${glob_file} }"
 
 # Curl and wait for the glob to be hosted
-glob_url="http://uniswap-glob-host:3000/${glob_file}"
+glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}"
 
 echo "Checking if glob file hosted at ${glob_url}"
 while true; do
@@ -128,7 +135,7 @@ cat << EOF > "${uniswap_desk_dir}/desk.docket-0"
     color+0xcd.75df
     image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
     base+'uniswap'
-    glob-http+['http://uniswap-glob-host:3000/${glob_file}' ${glob_hash}]
+    glob-http+['${glob_url}' ${glob_hash}]
     version+[0 0 1]
     website+'https://uniswap.org/'
     license+'MIT'

stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-# Use config from mounted volume (when running web-app along with watcher stack)
-echo "Waiting for uniswap app glob"
-while [ ! -d /app-globs/uniswap ]; do
-  echo "Glob directory not found, retrying in 5 seconds..."
-  sleep 5
-done
-
-
-# Copy to a new globs directory
-mkdir -p globs
-cp -r /app-globs/uniswap/* ./globs
-
-# Serve the glob file
-cd globs
-echo "Hosting glob file at port 3000"
-python3 -m http.server 3000 --bind 0.0.0.0

stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh

@@ -1,21 +1,22 @@
 #!/bin/bash
 
-# $1: Glob file URL (eg. https://xyz.com/glob-abcd.glob)
+# $1: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
-# $2: Uniswap desk dir (default: ./zod/uniswap)
+# $2: Glob file hash (eg. 0vabcd)
+# $3: Urbit ship's pier dir (default: ./zod)
 
-if [ -z "$1" ]; then
+if [ "$#" -lt 2 ]; then
-  echo "Glob file URL arg not provided"
+  echo "Insufficient arguments"
   exit 0
 fi
 
 glob_url=$1
-glob_file=$(basename "$glob_url")
+glob_hash=$2
-glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/")
+echo "Using glob file from ${glob_url} with hash ${glob_hash}"
-echo "Using glob file ${glob_file}"
 
+# Default pier dir: ./zod
 # Default desk dir: ./zod/uniswap
-uniswap_desk_dir="${2:-./zod/uniswap}"
+pier_dir="${3:-./zod}"
-
+uniswap_desk_dir="${pier_dir}/uniswap"
 echo "Using ${uniswap_desk_dir} as the Uniswap desk dir path"
 
 # Fire curl requests to perform operations on the ship

stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh

@@ -1,18 +1,21 @@
 #!/bin/bash
 
 # $1: Remote user host
-# $2: Path to run the app installation in (where urbit ship dir is located)
+# $2: Remote Urbit ship's pier dir path (eg. /home/user/zod)
-# $3: Glob file URL (eg. https://xyz.com/glob-abcd.glob)
+# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $4: Glob file hash (eg. 0vabcd)
 
-if [ "$#" -ne 3 ]; then
+if [ "$#" -ne 4 ]; then
-  echo "Usage: $0 <username@remote_host> </path/to/remote/folder> <glob_url>"
+  echo "Incorrect number of arguments"
+  echo "Usage: $0 <username@remote_host> </path/to/remote/pier/folder> <glob_url> <glob_hash>"
   exit 1
 fi
 
 remote_user_host="$1"
-remote_folder="$2"
+remote_pier_folder="$2"
 glob_url="$3"
+glob_hash="$4"
 
 installation_script="./install-uniswap-app.sh"
 
-ssh "$remote_user_host" "cd $remote_folder && bash -s $glob_url" < "$installation_script"
+ssh "$remote_user_host" "bash -s $glob_url $glob_hash $remote_pier_folder" < "$installation_script"

stack_orchestrator/data/stacks/uniswap-urbit-app/README.md

@@ -41,10 +41,11 @@ network:
   ports:
     urbit-fake-ship:
       - '8080:80'
-    uniswap-glob-host:
-      - '3000:3000'
     uniswap-gql-proxy:
       - '4000:4000'
+    ipfs-glob-host:
+      - '8081:8080'
+      - '5001:5001'
 ...
 ```
 
@@ -74,6 +75,16 @@ Inside the deployment directory, open the file `config.env` and add variable for
   # Set this to GQL proxy server endpoint for uniswap app
   # (Eg. http://localhost:4000/graphql)
   CERC_UNISWAP_GQL=
+
+  # Optional IPFS endpoints:
+
+  # IFPS endpoint to host the glob file on
+  # (Default: http://ipfs-glob-host:5001 pointing to in-stack IPFS node)
+  CERC_IPFS_GLOB_HOST_ENDPOINT=
+
+  # IFPS endpoint to fetch the glob file from
+  # (Default: http://ipfs-glob-host:8080 pointing to in-stack IPFS node)
+  CERC_IPFS_SERVER_ENDPOINT=
   ```
 
 ## Start the stack