Merge pull request #9751 from filecoin-project/id/remove-packer

_build_: Remove support for AMIs and DO Droplets

.circleci/config.yml

@@ -12,16 +12,6 @@ executors:
   ubuntu:
     docker:
       - image: ubuntu:20.04
-  packer:
-    description: |
-      The HashiCorp provided Packer container
-    parameters:
-      packer-version:
-        type: string
-        default: "1.8"
-    docker:
-      - image: hashicorp/packer:<< parameters.packer-version >>
-
 
 commands:
   prepare:
@@ -109,25 +99,6 @@ commands:
           name: fetch all tags
           command: |
             git fetch --all
-  packer_build:
-    description: "Run a packer build"
-    parameters:
-      template:
-        description: |
-          The name of the packer template file
-        type: string
-        default: packer.json
-      args:
-        description: |
-          Arguments to pass to the packer build command
-        type: string
-        default: ""
-
-    steps:
-      - run:
-          name: "Run a packer build"
-          command: packer build << parameters.args >> << parameters.template >>
-          no_output_timeout: 1h
 
 jobs:
   mod-tidy-check:
@@ -640,15 +611,6 @@ jobs:
                       image: filecoin/<<parameters.image>>
                       extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
 
-  publish-packer-snap:
-    description: build packer image with snap. mainnet only.
-    executor:
-      name: packer
-    steps:
-      - checkout
-      - packer_build:
-          template: tools/packer/lotus-snap.pkr.hcl
-
 workflows:
   version: 2.1
   ci:
@@ -1389,13 +1351,3 @@ workflows:
           channel: nightly
           network: debug
           push: true
-  biweekly:
-    triggers:
-      - schedule:
-          cron: "0 0 1,15 * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - publish-packer-snap

.circleci/template.yml

@@ -12,16 +12,6 @@ executors:
   ubuntu:
     docker:
       - image: ubuntu:20.04
-  packer:
-    description: |
-      The HashiCorp provided Packer container
-    parameters:
-      packer-version:
-        type: string
-        default: "1.8"
-    docker:
-      - image: hashicorp/packer:<< parameters.packer-version >>
-
 
 commands:
   prepare:
@@ -109,25 +99,6 @@ commands:
           name: fetch all tags
           command: |
             git fetch --all
-  packer_build:
-    description: "Run a packer build"
-    parameters:
-      template:
-        description: |
-          The name of the packer template file
-        type: string
-        default: packer.json
-      args:
-        description: |
-          Arguments to pass to the packer build command
-        type: string
-        default: ""
-
-    steps:
-      - run:
-          name: "Run a packer build"
-          command: packer build << parameters.args >> << parameters.template >>
-          no_output_timeout: 1h
 
 jobs:
   mod-tidy-check:
@@ -640,15 +611,6 @@ jobs:
                       image: filecoin/<<parameters.image>>
                       extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
 
-  publish-packer-snap:
-    description: build packer image with snap. mainnet only.
-    executor:
-      name: packer
-    steps:
-      - checkout
-      - packer_build:
-          template: tools/packer/lotus-snap.pkr.hcl
-
 workflows:
   version: 2.1
   ci:
@@ -903,13 +865,3 @@ workflows:
           network: [[.]]
           push: true
       [[- end]]
-  biweekly:
-    triggers:
-      - schedule:
-          cron: "0 0 1,15 * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - publish-packer-snap

tools/packer/etc/motd

@@ -1,64 +0,0 @@
-Your lotus node is up and running!
-
-This image contains the two most important pieces of the lotus filecoin suite, the
-daemon and the miner. The daemon is is configured to download a snapshot and start
-running. In fact, by the time you read this, the daemon may already be in sync.
-Go ahead and make sure everything is working correctly with the following commands.
-
-
-
-To check if the daemon is running:
-
-     systemctl status lotus-daemon
-
-
-
-To check if the daemon is in sync:
-
-    lotus sync status
-
-    **note: When starting lotus for the first time, it will download a chain snapshot.
-            This is a large download and will take several minutes to complete. During
-            this time, the lotus API will not be up yet. Give it time! You can see
-	    progress by looking at the systemd journal.
-
-
-To check if the daemon is connecting to other lotus nodes:
-
-    lotus net peers
-
-
-
-No wallets are crated by default. You can view, create, and delete wallets with
-the lotus command. On this image, lotus is running as the user `fc`.
-Be careful, now. Don't delete a wallet with funds!
-
-    sudo -E -u fc lotus wallet list
-    sudo -E -u fc lotus wallet new bls
-
-
-
-The lotus miner is also installed, but it's not running by default. If you have no
-special disk or worker requirements, you can initialize the lotus-miner repo like this:
-
-    sudo -E -u fc lotus-miner init -o <wallet_you_created_before>
-
-
-
-You only need to do this once, after which, you can enable and start the miner.
-
-    sudo systemctl enable lotus-miner
-    sudo systemctl start lotus-miner
-
-
-
-Do you want to access your lotus daemon remotely? Learn how to setup token authentication
-and use client libraries from lotus docs.
-
-https://lotus.filecoin.io/reference/basics/api-access/
-
-
-
-For more information, see             https://lotus.filecoin.io/
-Found a bug? let us know!             https://github.com/filecoin-project/lotus
-Chat with us on slack!                https://filecoinproject.slack.com/archives/CEGN061C5

tools/packer/homedir/bashrc

@@ -1,12 +0,0 @@
-PS1="[\h \w] ⨎ "
-
-export PROMT_DIRTRIM=1
-
-# Where to find the lotus repo
-export LOTUS_PATH=/var/lib/lotus
-
-# The miner is not running in this image by default.
-# export LOTUS_MINER_PATH=/var/lib/lotus-miner
-
-# To access the lotus node remotely, the following environment variable may be used.
-# export FULLNODE_API_INFO=<token>:/ip4/<public-ip>/tcp/1234/http

tools/packer/lotus-snap.pkr.hcl

@@ -1,90 +0,0 @@
-variable "ci_workspace_bins" {
-  type = string
-  default = "./linux"
-}
-
-variable "lotus_network" {
-  type = string
-  default = "mainnet"
-}
-
-locals {
-  timestamp = regex_replace(timestamp(), "[- TZ:]", "")
-}
-
-source "amazon-ebs" "lotus" {
-  ami_name      = "lotus-${var.lotus_network}-snap-${local.timestamp}"
-  ami_regions = [
-    "ap-east-1",
-    "ap-northeast-1",
-    "ap-northeast-2",
-    "ap-northeast-3",
-    "ap-south-1",
-    "ap-southeast-1",
-    "ap-southeast-2",
-    "ca-central-1",
-    "eu-central-1",
-    "eu-north-1",
-    "eu-west-1",
-    "eu-west-2",
-    "eu-west-3",
-    "sa-east-1",
-    "us-east-1",
-    "us-east-2",
-    "us-west-1",
-    "us-west-2",
-  ]
-  ami_groups = [
-    # This causes the ami to be publicly-accessable.
-    "all",
-  ]
-  ami_description = "Lotus Filecoin AMI"
-  launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 100
-    delete_on_termination = true
-  }
-
-  instance_type = "t2.micro"
-  source_ami_filter {
-    filters = {
-      name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*"
-      root-device-type = "ebs"
-      virtualization-type = "hvm"
-    }
-    most_recent = true
-    owners = ["099720109477"]
-  }
-  ssh_username = "ubuntu"
-
-  aws_polling {
-    delay_seconds = 60
-    max_attempts  = 60
-  }
-}
-
-source "digitalocean" "lotus" {
-  droplet_name = "lotus-snap"
-  size = "s-1vcpu-1gb"
-  region = "nyc3"
-  image = "ubuntu-20-04-x64"
-  snapshot_name = "lotus-${var.lotus_network}-snap-${local.timestamp}"
-  ssh_username = "root"
-}
-
-build {
-  sources = [
-    "source.amazon-ebs.lotus",
-    "source.digitalocean.lotus",
-  ]
-
-  provisioner "file" {
-    source = "./tools/packer/etc/motd"
-    destination = "motd"
-  }
-  # build it.
-  provisioner "shell" {
-    script = "./tools/packer/setup-snap.sh"
-  }
-}
-

tools/packer/lotus.pkr.hcl

@@ -1,110 +0,0 @@
-variable "ci_workspace_bins" {
-  type = string
-  default = "./linux"
-}
-
-variable "lotus_network" {
-  type = string
-  default = "mainnet"
-}
-
-variable "git_tag" {
-  type = string
-  default = ""
-}
-
-locals {
-  timestamp = regex_replace(timestamp(), "[- TZ:]", "")
-} 
-
-source "amazon-ebs" "lotus" {
-  ami_name      = "lotus-${var.lotus_network}-${var.git_tag}-${local.timestamp}"
-  ami_regions = [
-    "ap-east-1",
-    "ap-northeast-1",
-    "ap-northeast-2",
-    "ap-northeast-3",
-    "ap-south-1",
-    "ap-southeast-1",
-    "ap-southeast-2",
-    "ca-central-1",
-    "eu-central-1",
-    "eu-north-1",
-    "eu-west-1",
-    "eu-west-2",
-    "eu-west-3",
-    "sa-east-1",
-    "us-east-1",
-    "us-east-2",
-    "us-west-1",
-    "us-west-2",
-  ]
-  ami_groups = [
-    # This causes the ami to be publicly-accessable.
-    "all",
-  ]
-  ami_description = "Lotus Filecoin AMI"
-  launch_block_device_mappings {
-    device_name = "/dev/sda1"
-    volume_size = 100
-    delete_on_termination = true
-  }
-
-  instance_type = "t2.micro"
-  source_ami_filter {
-    filters = {
-      name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*"
-      root-device-type = "ebs"
-      virtualization-type = "hvm"
-    }
-    most_recent = true
-    owners = ["099720109477"]
-  }
-  ssh_username = "ubuntu"
-}
-
-build {
-  sources = [
-    "source.amazon-ebs.lotus",
-  ]
-
-  # Lotus software (from CI workspace)
-  provisioner "file" {
-    source = "${var.ci_workspace_bins}/lotus"
-    destination = "lotus"
-  }
-  provisioner "file" {
-    source = "${var.ci_workspace_bins}/lotus-miner"
-    destination = "lotus-miner"
-  }
-  # First run script
-  provisioner "file" {
-    source = "./tools/packer/scripts/${var.lotus_network}/lotus-init.sh"
-    destination = "lotus-init.sh"
-  }
-  # Systemd service units.
-  provisioner "file" {
-    source = "./tools/packer/systemd/lotus-daemon.service"
-    destination = "lotus-daemon.service"
-  }
-  provisioner "file" {
-    source = "./tools/packer/systemd/lotus-miner.service"
-    destination = "lotus-miner.service"
-  }
-  provisioner "file" {
-    source = "./tools/packer/repo/config.toml"
-    destination = "config.toml"
-  }
-  provisioner "file" {
-    source = "./tools/packer/etc/motd"
-    destination = "motd"
-  }
-  provisioner "file" {
-    source = "./tools/packer/homedir/bashrc"
-    destination = ".bashrc"
-  }
-  # build it.
-  provisioner "shell" {
-    script = "./tools/packer/setup.sh"
-  }
-}

tools/packer/repo/config.toml

@@ -1,38 +0,0 @@
-[API]
-ListenAddress = "/ip4/0.0.0.0/tcp/1234/http"
-#  RemoteListenAddress = ""
-#  Timeout = "30s"
-#
-[Libp2p]
-ListenAddresses = ["/ip4/0.0.0.0/tcp/5678", "/ip6/::/tcp/5678"]
-#  AnnounceAddresses = []
-#  NoAnnounceAddresses = []
-#  ConnMgrLow = 150
-#  ConnMgrHigh = 180
-#  ConnMgrGrace = "20s"
-#
-[Pubsub]
-#  Bootstrapper = false
-#  RemoteTracer = ""
-#
-[Client]
-#  UseIpfs = false
-#  IpfsOnlineMode = false
-#  IpfsMAddr = ""
-#  IpfsUseForRetrieval = false
-#  SimultaneousTransfersForStorage = 20
-#  SimultaneousTransfersForStoragePerClient = 0
-#  SimultaneousTransfersForRetrieval = 20
-#
-[Metrics]
-#  Nickname = ""
-#  HeadNotifs = false
-#
-[Wallet]
-#  RemoteBackend = ""
-#  EnableLedger = false
-#  DisableLocal = false
-#
-[Fees]
-#  DefaultMaxFee = "0.007 FIL"
-#

tools/packer/scripts/butterflynet/lotus-init.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# This script sets up an initial configuraiton for the lotus daemon and miner
-# It will only run once.
-
-GATE="$LOTUS_PATH"/date_initialized
-
-# Don't init if already initialized.
-if [ -f "$GATE" ]; then
-	echo lotus already initialized.
-	exit 0
-fi
-
-# Not importing snapshot on butterflynet
-#
-# echo importing minimal snapshot
-# lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import
-
-# Block future inits
-date > "$GATE"

tools/packer/scripts/calibrationnet/lotus-init.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# This script sets up an initial configuraiton for the lotus daemon and miner
-# It will only run once.
-
-GATE="$LOTUS_PATH"/date_initialized
-
-# Don't init if already initialized.
-if [ -f "$GATE" ]; then
-	echo lotus already initialized.
-	exit 0
-fi
-
-# Not importing snapshot on calibrationnet.
-#
-# echo importing minimal snapshot
-# lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import
-
-# Block future inits
-date > "$GATE"

tools/packer/scripts/mainnet/lotus-init.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-# This script sets up an initial configuraiton for the lotus daemon and miner
-# It will only run once.
-
-GATE="$LOTUS_PATH"/date_initialized
-
-# Don't init if already initialized.
-if [ -f "$GATE" ]; then
-	echo lotus already initialized.
-	exit 0
-fi
-
-echo importing minimal snapshot
-lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import
-
-# Block future inits
-date > "$GATE"

tools/packer/setup-snap.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is executed by packer to setup the image.
-# When this script is run, packer will have already copied binaries into the home directory of
-# whichever user it has access too. This script is executed from within the home directory of that
-# user. Bear in mind that different cloud providers, and different images on the same cloud
-# provider will have a different initial user account.
-
-set -x
-
-# Become root, if we aren't already.
-# Docker images will already be root. AMIs will have an SSH user account.
-if [ x$UID != x0 ]
-then
-	printf -v cmd_str '%q ' "$0" "$@"
-	exec sudo su -c "$cmd_str"
-fi
-
-set -e
-
-MANAGED_FILES=(
-	/etc/motd
-)
-
-# this is required on digitalocean, which does not have snap seeded correctly at this phase.
-apt-get -y -o DPkg::Lock::Timeout=3 update \
-  && apt-get -y -o DPkg::Lock::Timeout=3 reinstall snapd
-
-snap install lotus
-
-snap alias lotus.lotus-miner lotus-miner
-snap alias lotus.lotus-worker lotus-worker
-
-# Setup firewall
-yes | ufw enable
-ufw default deny incoming
-ufw default allow outgoing
-ufw allow ssh
-
-set +e
-
-curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash

tools/packer/setup.sh

@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-
-# This script is executed by packer to setup the image.
-# When this script is run, packer will have already copied binaries into the home directory of
-# whichever user it has access too. This script is executed from within the home directory of that
-# user. Bear in mind that different cloud providers, and different images on the same cloud
-# provider will have a different initial user account.
-
-set -x
-
-# Become root, if we aren't already.
-# Docker images will already be root. AMIs will have an SSH user account.
-UID=$(id -u)
-if [ x$UID != x0 ]
-then
-	printf -v cmd_str '%q ' "$0" "$@"
-	exec sudo su -c "$cmd_str"
-fi
-
-MANAGED_BINS=( lotus lotus-miner lotus-init.sh )
-MANAGED_FILES=(
-  /lib/systemd/system/lotus-daemon.service
-	/lib/systemd/system/lotus-miner.service
-	/etc/motd
-	/var/lib/lotus/config.toml
-)
-
-# install libs.
-export DEBIAN_FRONTEND=noninteractive
-apt-get update
-apt-get -y install libhwloc15 ocl-icd-libopencl1 ufw
-apt-get -y upgrade -q -y -u -o Dpkg::Options::="--force-confold"
-ln -s /usr/lib/x86_64-linux-gnu/libhwloc.so.15 /usr/lib/x86_64-linux-gnu/libhwloc.so.5
-
-# Create lotus user
-useradd -c "lotus system account" -r fc
-install -o fc -g fc -d /var/lib/lotus
-install -o fc -g fc -d /var/lib/lotus-miner
-
-# Install software
-for i in "${MANAGED_BINS[@]}"
-do
-	install -o root -g root -m 755 -t /usr/local/bin $i
-	rm $i
-done
-
-# Install systemd and other files.
-# Because packer doesn't copy files with root permisison,
-# files are in the home directory of the ssh user. Copy
-# these files into the right position.
-for i in "${MANAGED_FILES[@]}"
-do
-	fn=$(basename $i)
-	install -o root -g root -m 644 $fn $i
-	rm $fn
-done
-
-# Enable services
-systemctl daemon-reload
-systemctl enable lotus-daemon
-
-# Setup firewall
-yes | ufw enable
-ufw default deny incoming
-ufw default allow outgoing
-ufw allow ssh
-ufw allow 5678   #libp2p
-
-curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash

tools/packer/systemd/lotus-daemon.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=Lotus Daemon
-After=network.target
-
-[Service]
-User=fc
-Group=fc
-ExecStartPre=/usr/local/bin/lotus-init.sh
-ExecStart=/usr/local/bin/lotus daemon
-ExecStop=/usr/local/bin/lotus daemon stop
-Environment=LOTUS_PATH=/var/lib/lotus
-Restart=always
-RestartSec=30
-TimeoutSec=infinity
-
-[Install]
-WantedBy=multi-user.target

tools/packer/systemd/lotus-miner.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Lotus Miner
-After=network.target
-
-[Service]
-User=fc
-Group=fc
-ExecStart=/usr/local/bin/lotus-miner run
-Environment=LOTUS_PATH=/var/lib/lotus
-Environment=LOTUS_MINER_PATH=/var/lib/lotus-miner
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target