Merge pull request #7819 from filecoin-project/digitalocean-use-snap

feat: ci: packer snap

.circleci/config.yml

@@ -715,6 +715,17 @@ jobs:
       - packer/build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
+    executor:
+      name: packer/default
+      packer-version: 1.6.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: "."
+      - packer/build:
+          template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
     description: publish to dockerhub
     machine:
@@ -1087,3 +1098,13 @@ workflows:
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap

.circleci/template.yml

@@ -715,6 +715,17 @@ jobs:
       - packer/build:
           template: tools/packer/lotus.pkr.hcl
           args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG"
+  publish-packer-snap:
+    description: build packer image with snap. mainnet only.
+    executor:
+      name: packer/default
+      packer-version: 1.6.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: "."
+      - packer/build:
+          template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
     description: publish to dockerhub
     machine:
@@ -917,3 +928,13 @@ workflows:
       - publish-dockerhub:
           name: publish-dockerhub-nightly
           tag: nightly
+  monthly:
+    triggers:
+      - schedule:
+          cron: "0 0 1 * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - publish-packer-snap

.gitignore

@@ -40,6 +40,7 @@ build/paramfetch.sh
 /bundle
 /darwin
 /linux
+*.snap
 
 *-fuzz.zip
 /chain/types/work_msg/

scripts/snap-lotus-entrypoint.sh

@@ -0,0 +1,10 @@
+LOTUS_IMPORT_SNAPSHOT="https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car"
+LOTUS_BINARY=$(dirname "$0")/lotus
+GATE="$LOTUS_PATH"/date_initialized
+if [ ! -f "$GATE" ]; then
+	echo importing minimal snapshot
+	$LOTUS_BINARY daemon --import-snapshot "$LOTUS_IMPORT_SNAPSHOT" --halt-after-import
+	# Block future inits
+	date > "$GATE"
+fi
+$LOTUS_BINARY daemon $ARGS

snap/snapcraft.yaml

@@ -38,6 +38,15 @@ parts:
     override-build: |
       LDFLAGS="" make lotus lotus-miner lotus-worker
       cp lotus lotus-miner lotus-worker $SNAPCRAFT_PART_INSTALL
+      cp scripts/snap-lotus-entrypoint.sh $SNAPCRAFT_PART_INSTALL
+
+layout:
+  /var/lib/lotus:
+    symlink: $SNAP_COMMON/lotus
+  /var/lib/lotus-miner:
+    symlink: $SNAP_COMMON/lotus-miner
+  /var/lib/lotus-worker:
+    symlink: $SNAP_COMMON/lotus-worker
 
 apps:
   lotus:
@@ -48,9 +57,9 @@ apps:
       - home
     environment:
       FIL_PROOFS_PARAMETER_CACHE: $SNAP_USER_COMMON/filecoin-proof-parameters
-      LOTUS_PATH: $SNAP_USER_COMMON/lotus
+      LOTUS_PATH: $SNAP_COMMON/lotus
-      LOTUS_MINER_PATH: $SNAP_USER_COMMON/lotus-miner
+      LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner
-      LOTUS_WORKER_PATH: $SNAP_USER_COMMON/lotus-worker
+      LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker
   lotus-miner:
     command: lotus-miner
     plugs:
@@ -59,9 +68,9 @@ apps:
       - opengl
     environment:
       FIL_PROOFS_PARAMETER_CACHE: $SNAP_USER_COMMON/filecoin-proof-parameters
-      LOTUS_PATH: $SNAP_USER_COMMON/lotus
+      LOTUS_PATH: $SNAP_COMMON/lotus
-      LOTUS_MINER_PATH: $SNAP_USER_COMMON/lotus-miner
+      LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner
-      LOTUS_WORKER_PATH: $SNAP_USER_COMMON/lotus-worker
+      LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker
   lotus-worker:
     command: lotus-worker
     plugs:
@@ -70,6 +79,18 @@ apps:
       - opengl
     environment:
       FIL_PROOFS_PARAMETER_CACHE: $SNAP_USER_COMMON/filecoin-proof-parameters
-      LOTUS_PATH: $SNAP_USER_COMMON/lotus
+      LOTUS_PATH: $SNAP_COMMON/lotus
-      LOTUS_MINER_PATH: $SNAP_USER_COMMON/lotus-miner
+      LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner
-      LOTUS_WORKER_PATH: $SNAP_USER_COMMON/lotus-worker
+      LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker
+  lotus-daemon:
+    command: snap-lotus-entrypoint.sh
+    daemon: simple
+    install-mode: enable
+    plugs:
+      - network
+      - network-bind
+    environment:
+      FIL_PROOFS_PARAMETER_CACHE: $SNAP_COMMON/filecoin-proof-parameters
+      LOTUS_PATH: $SNAP_COMMON/lotus
+      LOTUS_MINER_PATH: $SNAP_COMMON/lotus-miner
+      LOTUS_WORKER_PATH: $SNAP_COMMON/lotus-worker

tools/packer/lotus-snap.pkr.hcl

@@ -0,0 +1,84 @@
+variable "ci_workspace_bins" {
+  type = string
+  default = "./linux"
+}
+
+variable "lotus_network" {
+  type = string
+  default = "mainnet"
+}
+
+locals {
+  timestamp = regex_replace(timestamp(), "[- TZ:]", "")
+}
+
+source "amazon-ebs" "lotus" {
+  ami_name      = "lotus-${var.lotus_network}-snap-${local.timestamp}"
+  ami_regions = [
+    "ap-east-1",
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "ap-northeast-3",
+    "ap-south-1",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ca-central-1",
+    "eu-central-1",
+    "eu-north-1",
+    "eu-west-1",
+    "eu-west-2",
+    "eu-west-3",
+    "sa-east-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-1",
+    "us-west-2",
+  ]
+  ami_groups = [
+    # This causes the ami to be publicly-accessable.
+    "all",
+  ]
+  ami_description = "Lotus Filecoin AMI"
+  launch_block_device_mappings {
+    device_name = "/dev/sda1"
+    volume_size = 100
+    delete_on_termination = true
+  }
+
+  instance_type = "t2.micro"
+  source_ami_filter {
+    filters = {
+      name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*"
+      root-device-type = "ebs"
+      virtualization-type = "hvm"
+    }
+    most_recent = true
+    owners = ["099720109477"]
+  }
+  ssh_username = "ubuntu"
+}
+
+source "digitalocean" "lotus" {
+  droplet_name = "lotus-snap"
+  size = "s-1vcpu-1gb"
+  region = "nyc3"
+  image = "ubuntu-20-04-x64"
+  snapshot_name = "lotus-${var.lotus_network}-snap-${local.timestamp}"
+  ssh_username = "root"
+}
+
+build {
+  sources = [
+    "source.amazon-ebs.lotus",
+    "source.digitalocean.lotus",
+  ]
+
+  provisioner "file" {
+    source = "./tools/packer/etc/motd"
+    destination = "motd"
+  }
+  # build it.
+  provisioner "shell" {
+    script = "./tools/packer/setup-snap.sh"
+  }
+}

tools/packer/lotus.pkr.hcl

@@ -63,19 +63,9 @@ source "amazon-ebs" "lotus" {
   ssh_username = "ubuntu"
 }
 
-source "digitalocean" "lotus" {
-  droplet_name = "lotus-${var.lotus_network}"
-  size = "s-1vcpu-1gb"
-  region = "nyc3"
-  image = "ubuntu-20-04-x64"
-  snapshot_name = "lotus-${var.lotus_network}-${var.git_tag}-${local.timestamp}"
-  ssh_username = "root"
-}
-
 build {
   sources = [
     "source.amazon-ebs.lotus",
-    "source.digitalocean.lotus",
   ]
 
   # Lotus software (from CI workspace)

tools/packer/setup-snap.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# This script is executed by packer to setup the image.
+# When this script is run, packer will have already copied binaries into the home directory of
+# whichever user it has access too. This script is executed from within the home directory of that
+# user. Bear in mind that different cloud providers, and different images on the same cloud
+# provider will have a different initial user account.
+
+set -x
+
+# Become root, if we aren't already.
+# Docker images will already be root. AMIs will have an SSH user account.
+if [ x$UID != x0 ]
+then
+	printf -v cmd_str '%q ' "$0" "$@"
+	exec sudo su -c "$cmd_str"
+fi
+
+set -e
+
+MANAGED_FILES=(
+	/etc/motd
+)
+
+# this is required on digitalocean, which does not have snap seeded correctly at this phase.
+apt update
+apt reinstall snapd
+
+snap install lotus-filecoin
+
+snap alias lotus-filecoin.lotus lotus
+snap alias lotus-file.con.lotus-daemon lotus-daemon
+snap alias lotus-filecoin.lotus-miner lotus-miner
+snap alias lotus-filecoin.lotus-worker lotus-worker
+
+snap stop lotus-filecoin.lotus-daemon
+
+# Setup firewall
+yes | ufw enable
+ufw default deny incoming
+ufw default allow outgoing
+ufw allow ssh
+
+set +e
+
+curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash

tools/packer/setup.sh

@@ -65,3 +65,5 @@ ufw default deny incoming
 ufw default allow outgoing
 ufw allow ssh
 ufw allow 5678   #libp2p
+
+curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash