From 76baeb321415c445a3bff13873a1325142281799 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Tue, 29 Nov 2022 17:21:28 +0000 Subject: [PATCH] Remove support for AMIs and DO Droplets This discontinues packer builds and stops publishing (or attempting to publish) them to AWS / DO. These have been broken for more than 6 months, and no one seems to have complained or even noticed, which seems to indicate they are not being used much. For cloud platfrom support, we can optionally write docker / terraform automation that folks could use to more easily launch instances to cloud environments instead. --- .circleci/config.yml | 48 -------- .circleci/template.yml | 48 -------- tools/packer/etc/motd | 64 ---------- tools/packer/homedir/bashrc | 12 -- tools/packer/lotus-snap.pkr.hcl | 90 -------------- tools/packer/lotus.pkr.hcl | 110 ------------------ tools/packer/repo/config.toml | 38 ------ .../packer/scripts/butterflynet/lotus-init.sh | 20 ---- .../scripts/calibrationnet/lotus-init.sh | 20 ---- tools/packer/scripts/mainnet/lotus-init.sh | 18 --- tools/packer/setup-snap.sh | 42 ------- tools/packer/setup.sh | 69 ----------- tools/packer/systemd/lotus-daemon.service | 17 --- tools/packer/systemd/lotus-miner.service | 15 --- 14 files changed, 611 deletions(-) delete mode 100644 tools/packer/etc/motd delete mode 100644 tools/packer/homedir/bashrc delete mode 100644 tools/packer/lotus-snap.pkr.hcl delete mode 100644 tools/packer/lotus.pkr.hcl delete mode 100644 tools/packer/repo/config.toml delete mode 100755 tools/packer/scripts/butterflynet/lotus-init.sh delete mode 100755 tools/packer/scripts/calibrationnet/lotus-init.sh delete mode 100755 tools/packer/scripts/mainnet/lotus-init.sh delete mode 100644 tools/packer/setup-snap.sh delete mode 100644 tools/packer/setup.sh delete mode 100644 tools/packer/systemd/lotus-daemon.service delete mode 100644 tools/packer/systemd/lotus-miner.service diff --git a/.circleci/config.yml b/.circleci/config.yml index 0df595bfc..d6494c451 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,16 +12,6 @@ executors: ubuntu: docker: - image: ubuntu:20.04 - packer: - description: | - The HashiCorp provided Packer container - parameters: - packer-version: - type: string - default: "1.8" - docker: - - image: hashicorp/packer:<< parameters.packer-version >> - commands: prepare: @@ -105,25 +95,6 @@ commands: name: fetch all tags command: | git fetch --all - packer_build: - description: "Run a packer build" - parameters: - template: - description: | - The name of the packer template file - type: string - default: packer.json - args: - description: | - Arguments to pass to the packer build command - type: string - default: "" - - steps: - - run: - name: "Run a packer build" - command: packer build << parameters.args >> << parameters.template >> - no_output_timeout: 1h jobs: mod-tidy-check: @@ -671,15 +642,6 @@ jobs: image: filecoin/<> extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> - publish-packer-snap: - description: build packer image with snap. mainnet only. - executor: - name: packer - steps: - - checkout - - packer_build: - template: tools/packer/lotus-snap.pkr.hcl - workflows: version: 2.1 ci: @@ -1342,13 +1304,3 @@ workflows: channel: nightly network: debug push: true - biweekly: - triggers: - - schedule: - cron: "0 0 1,15 * *" - filters: - branches: - only: - - master - jobs: - - publish-packer-snap diff --git a/.circleci/template.yml b/.circleci/template.yml index 8cd3e34dd..353c5a023 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -12,16 +12,6 @@ executors: ubuntu: docker: - image: ubuntu:20.04 - packer: - description: | - The HashiCorp provided Packer container - parameters: - packer-version: - type: string - default: "1.8" - docker: - - image: hashicorp/packer:<< parameters.packer-version >> - commands: prepare: @@ -105,25 +95,6 @@ commands: name: fetch all tags command: | git fetch --all - packer_build: - description: "Run a packer build" - parameters: - template: - description: | - The name of the packer template file - type: string - default: packer.json - args: - description: | - Arguments to pass to the packer build command - type: string - default: "" - - steps: - - run: - name: "Run a packer build" - command: packer build << parameters.args >> << parameters.template >> - no_output_timeout: 1h jobs: mod-tidy-check: @@ -671,15 +642,6 @@ jobs: image: filecoin/<> extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> - publish-packer-snap: - description: build packer image with snap. mainnet only. - executor: - name: packer - steps: - - checkout - - packer_build: - template: tools/packer/lotus-snap.pkr.hcl - workflows: version: 2.1 ci: @@ -886,13 +848,3 @@ workflows: network: [[.]] push: true [[- end]] - biweekly: - triggers: - - schedule: - cron: "0 0 1,15 * *" - filters: - branches: - only: - - master - jobs: - - publish-packer-snap diff --git a/tools/packer/etc/motd b/tools/packer/etc/motd deleted file mode 100644 index 892b156e1..000000000 --- a/tools/packer/etc/motd +++ /dev/null @@ -1,64 +0,0 @@ -Your lotus node is up and running! - -This image contains the two most important pieces of the lotus filecoin suite, the -daemon and the miner. The daemon is is configured to download a snapshot and start -running. In fact, by the time you read this, the daemon may already be in sync. -Go ahead and make sure everything is working correctly with the following commands. - - - -To check if the daemon is running: - - systemctl status lotus-daemon - - - -To check if the daemon is in sync: - - lotus sync status - - **note: When starting lotus for the first time, it will download a chain snapshot. - This is a large download and will take several minutes to complete. During - this time, the lotus API will not be up yet. Give it time! You can see - progress by looking at the systemd journal. - - -To check if the daemon is connecting to other lotus nodes: - - lotus net peers - - - -No wallets are crated by default. You can view, create, and delete wallets with -the lotus command. On this image, lotus is running as the user `fc`. -Be careful, now. Don't delete a wallet with funds! - - sudo -E -u fc lotus wallet list - sudo -E -u fc lotus wallet new bls - - - -The lotus miner is also installed, but it's not running by default. If you have no -special disk or worker requirements, you can initialize the lotus-miner repo like this: - - sudo -E -u fc lotus-miner init -o - - - -You only need to do this once, after which, you can enable and start the miner. - - sudo systemctl enable lotus-miner - sudo systemctl start lotus-miner - - - -Do you want to access your lotus daemon remotely? Learn how to setup token authentication -and use client libraries from lotus docs. - -https://lotus.filecoin.io/reference/basics/api-access/ - - - -For more information, see https://lotus.filecoin.io/ -Found a bug? let us know! https://github.com/filecoin-project/lotus -Chat with us on slack! https://filecoinproject.slack.com/archives/CEGN061C5 diff --git a/tools/packer/homedir/bashrc b/tools/packer/homedir/bashrc deleted file mode 100644 index db4dbd0b6..000000000 --- a/tools/packer/homedir/bashrc +++ /dev/null @@ -1,12 +0,0 @@ -PS1="[\h \w] ⨎ " - -export PROMT_DIRTRIM=1 - -# Where to find the lotus repo -export LOTUS_PATH=/var/lib/lotus - -# The miner is not running in this image by default. -# export LOTUS_MINER_PATH=/var/lib/lotus-miner - -# To access the lotus node remotely, the following environment variable may be used. -# export FULLNODE_API_INFO=:/ip4//tcp/1234/http diff --git a/tools/packer/lotus-snap.pkr.hcl b/tools/packer/lotus-snap.pkr.hcl deleted file mode 100644 index b42272c20..000000000 --- a/tools/packer/lotus-snap.pkr.hcl +++ /dev/null @@ -1,90 +0,0 @@ -variable "ci_workspace_bins" { - type = string - default = "./linux" -} - -variable "lotus_network" { - type = string - default = "mainnet" -} - -locals { - timestamp = regex_replace(timestamp(), "[- TZ:]", "") -} - -source "amazon-ebs" "lotus" { - ami_name = "lotus-${var.lotus_network}-snap-${local.timestamp}" - ami_regions = [ - "ap-east-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-south-1", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "eu-central-1", - "eu-north-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - ] - ami_groups = [ - # This causes the ami to be publicly-accessable. - "all", - ] - ami_description = "Lotus Filecoin AMI" - launch_block_device_mappings { - device_name = "/dev/sda1" - volume_size = 100 - delete_on_termination = true - } - - instance_type = "t2.micro" - source_ami_filter { - filters = { - name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["099720109477"] - } - ssh_username = "ubuntu" - - aws_polling { - delay_seconds = 60 - max_attempts = 60 - } -} - -source "digitalocean" "lotus" { - droplet_name = "lotus-snap" - size = "s-1vcpu-1gb" - region = "nyc3" - image = "ubuntu-20-04-x64" - snapshot_name = "lotus-${var.lotus_network}-snap-${local.timestamp}" - ssh_username = "root" -} - -build { - sources = [ - "source.amazon-ebs.lotus", - "source.digitalocean.lotus", - ] - - provisioner "file" { - source = "./tools/packer/etc/motd" - destination = "motd" - } - # build it. - provisioner "shell" { - script = "./tools/packer/setup-snap.sh" - } -} - diff --git a/tools/packer/lotus.pkr.hcl b/tools/packer/lotus.pkr.hcl deleted file mode 100644 index cfaca8398..000000000 --- a/tools/packer/lotus.pkr.hcl +++ /dev/null @@ -1,110 +0,0 @@ -variable "ci_workspace_bins" { - type = string - default = "./linux" -} - -variable "lotus_network" { - type = string - default = "mainnet" -} - -variable "git_tag" { - type = string - default = "" -} - -locals { - timestamp = regex_replace(timestamp(), "[- TZ:]", "") -} - -source "amazon-ebs" "lotus" { - ami_name = "lotus-${var.lotus_network}-${var.git_tag}-${local.timestamp}" - ami_regions = [ - "ap-east-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-south-1", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "eu-central-1", - "eu-north-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "sa-east-1", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - ] - ami_groups = [ - # This causes the ami to be publicly-accessable. - "all", - ] - ami_description = "Lotus Filecoin AMI" - launch_block_device_mappings { - device_name = "/dev/sda1" - volume_size = 100 - delete_on_termination = true - } - - instance_type = "t2.micro" - source_ami_filter { - filters = { - name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["099720109477"] - } - ssh_username = "ubuntu" -} - -build { - sources = [ - "source.amazon-ebs.lotus", - ] - - # Lotus software (from CI workspace) - provisioner "file" { - source = "${var.ci_workspace_bins}/lotus" - destination = "lotus" - } - provisioner "file" { - source = "${var.ci_workspace_bins}/lotus-miner" - destination = "lotus-miner" - } - # First run script - provisioner "file" { - source = "./tools/packer/scripts/${var.lotus_network}/lotus-init.sh" - destination = "lotus-init.sh" - } - # Systemd service units. - provisioner "file" { - source = "./tools/packer/systemd/lotus-daemon.service" - destination = "lotus-daemon.service" - } - provisioner "file" { - source = "./tools/packer/systemd/lotus-miner.service" - destination = "lotus-miner.service" - } - provisioner "file" { - source = "./tools/packer/repo/config.toml" - destination = "config.toml" - } - provisioner "file" { - source = "./tools/packer/etc/motd" - destination = "motd" - } - provisioner "file" { - source = "./tools/packer/homedir/bashrc" - destination = ".bashrc" - } - # build it. - provisioner "shell" { - script = "./tools/packer/setup.sh" - } -} diff --git a/tools/packer/repo/config.toml b/tools/packer/repo/config.toml deleted file mode 100644 index 380d5a28f..000000000 --- a/tools/packer/repo/config.toml +++ /dev/null @@ -1,38 +0,0 @@ -[API] -ListenAddress = "/ip4/0.0.0.0/tcp/1234/http" -# RemoteListenAddress = "" -# Timeout = "30s" -# -[Libp2p] -ListenAddresses = ["/ip4/0.0.0.0/tcp/5678", "/ip6/::/tcp/5678"] -# AnnounceAddresses = [] -# NoAnnounceAddresses = [] -# ConnMgrLow = 150 -# ConnMgrHigh = 180 -# ConnMgrGrace = "20s" -# -[Pubsub] -# Bootstrapper = false -# RemoteTracer = "" -# -[Client] -# UseIpfs = false -# IpfsOnlineMode = false -# IpfsMAddr = "" -# IpfsUseForRetrieval = false -# SimultaneousTransfersForStorage = 20 -# SimultaneousTransfersForStoragePerClient = 0 -# SimultaneousTransfersForRetrieval = 20 -# -[Metrics] -# Nickname = "" -# HeadNotifs = false -# -[Wallet] -# RemoteBackend = "" -# EnableLedger = false -# DisableLocal = false -# -[Fees] -# DefaultMaxFee = "0.007 FIL" -# diff --git a/tools/packer/scripts/butterflynet/lotus-init.sh b/tools/packer/scripts/butterflynet/lotus-init.sh deleted file mode 100755 index cfbf93f78..000000000 --- a/tools/packer/scripts/butterflynet/lotus-init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# This script sets up an initial configuraiton for the lotus daemon and miner -# It will only run once. - -GATE="$LOTUS_PATH"/date_initialized - -# Don't init if already initialized. -if [ -f "$GATE" ]; then - echo lotus already initialized. - exit 0 -fi - -# Not importing snapshot on butterflynet -# -# echo importing minimal snapshot -# lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import - -# Block future inits -date > "$GATE" diff --git a/tools/packer/scripts/calibrationnet/lotus-init.sh b/tools/packer/scripts/calibrationnet/lotus-init.sh deleted file mode 100755 index 77260fa29..000000000 --- a/tools/packer/scripts/calibrationnet/lotus-init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# This script sets up an initial configuraiton for the lotus daemon and miner -# It will only run once. - -GATE="$LOTUS_PATH"/date_initialized - -# Don't init if already initialized. -if [ -f "$GATE" ]; then - echo lotus already initialized. - exit 0 -fi - -# Not importing snapshot on calibrationnet. -# -# echo importing minimal snapshot -# lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import - -# Block future inits -date > "$GATE" diff --git a/tools/packer/scripts/mainnet/lotus-init.sh b/tools/packer/scripts/mainnet/lotus-init.sh deleted file mode 100755 index b22853365..000000000 --- a/tools/packer/scripts/mainnet/lotus-init.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# This script sets up an initial configuraiton for the lotus daemon and miner -# It will only run once. - -GATE="$LOTUS_PATH"/date_initialized - -# Don't init if already initialized. -if [ -f "$GATE" ]; then - echo lotus already initialized. - exit 0 -fi - -echo importing minimal snapshot -lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import - -# Block future inits -date > "$GATE" diff --git a/tools/packer/setup-snap.sh b/tools/packer/setup-snap.sh deleted file mode 100644 index 615877f83..000000000 --- a/tools/packer/setup-snap.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# This script is executed by packer to setup the image. -# When this script is run, packer will have already copied binaries into the home directory of -# whichever user it has access too. This script is executed from within the home directory of that -# user. Bear in mind that different cloud providers, and different images on the same cloud -# provider will have a different initial user account. - -set -x - -# Become root, if we aren't already. -# Docker images will already be root. AMIs will have an SSH user account. -if [ x$UID != x0 ] -then - printf -v cmd_str '%q ' "$0" "$@" - exec sudo su -c "$cmd_str" -fi - -set -e - -MANAGED_FILES=( - /etc/motd -) - -# this is required on digitalocean, which does not have snap seeded correctly at this phase. -apt-get -y -o DPkg::Lock::Timeout=3 update \ - && apt-get -y -o DPkg::Lock::Timeout=3 reinstall snapd - -snap install lotus - -snap alias lotus.lotus-miner lotus-miner -snap alias lotus.lotus-worker lotus-worker - -# Setup firewall -yes | ufw enable -ufw default deny incoming -ufw default allow outgoing -ufw allow ssh - -set +e - -curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash diff --git a/tools/packer/setup.sh b/tools/packer/setup.sh deleted file mode 100644 index 2b190045e..000000000 --- a/tools/packer/setup.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -# This script is executed by packer to setup the image. -# When this script is run, packer will have already copied binaries into the home directory of -# whichever user it has access too. This script is executed from within the home directory of that -# user. Bear in mind that different cloud providers, and different images on the same cloud -# provider will have a different initial user account. - -set -x - -# Become root, if we aren't already. -# Docker images will already be root. AMIs will have an SSH user account. -UID=$(id -u) -if [ x$UID != x0 ] -then - printf -v cmd_str '%q ' "$0" "$@" - exec sudo su -c "$cmd_str" -fi - -MANAGED_BINS=( lotus lotus-miner lotus-init.sh ) -MANAGED_FILES=( - /lib/systemd/system/lotus-daemon.service - /lib/systemd/system/lotus-miner.service - /etc/motd - /var/lib/lotus/config.toml -) - -# install libs. -export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get -y install libhwloc15 ocl-icd-libopencl1 ufw -apt-get -y upgrade -q -y -u -o Dpkg::Options::="--force-confold" -ln -s /usr/lib/x86_64-linux-gnu/libhwloc.so.15 /usr/lib/x86_64-linux-gnu/libhwloc.so.5 - -# Create lotus user -useradd -c "lotus system account" -r fc -install -o fc -g fc -d /var/lib/lotus -install -o fc -g fc -d /var/lib/lotus-miner - -# Install software -for i in "${MANAGED_BINS[@]}" -do - install -o root -g root -m 755 -t /usr/local/bin $i - rm $i -done - -# Install systemd and other files. -# Because packer doesn't copy files with root permisison, -# files are in the home directory of the ssh user. Copy -# these files into the right position. -for i in "${MANAGED_FILES[@]}" -do - fn=$(basename $i) - install -o root -g root -m 644 $fn $i - rm $fn -done - -# Enable services -systemctl daemon-reload -systemctl enable lotus-daemon - -# Setup firewall -yes | ufw enable -ufw default deny incoming -ufw default allow outgoing -ufw allow ssh -ufw allow 5678 #libp2p - -curl -L https://raw.githubusercontent.com/digitalocean/marketplace-partners/master/scripts/90-cleanup.sh | bash diff --git a/tools/packer/systemd/lotus-daemon.service b/tools/packer/systemd/lotus-daemon.service deleted file mode 100644 index edbc91151..000000000 --- a/tools/packer/systemd/lotus-daemon.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Lotus Daemon -After=network.target - -[Service] -User=fc -Group=fc -ExecStartPre=/usr/local/bin/lotus-init.sh -ExecStart=/usr/local/bin/lotus daemon -ExecStop=/usr/local/bin/lotus daemon stop -Environment=LOTUS_PATH=/var/lib/lotus -Restart=always -RestartSec=30 -TimeoutSec=infinity - -[Install] -WantedBy=multi-user.target diff --git a/tools/packer/systemd/lotus-miner.service b/tools/packer/systemd/lotus-miner.service deleted file mode 100644 index d7289c888..000000000 --- a/tools/packer/systemd/lotus-miner.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Lotus Miner -After=network.target - -[Service] -User=fc -Group=fc -ExecStart=/usr/local/bin/lotus-miner run -Environment=LOTUS_PATH=/var/lib/lotus -Environment=LOTUS_MINER_PATH=/var/lib/lotus-miner -Restart=always -RestartSec=30 - -[Install] -WantedBy=multi-user.target