From cbda4ff7bf99a4453096d5e5fb4eeb699b6198ea Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 14:09:01 -0700 Subject: [PATCH 1/8] other network images --- .circleci/config.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 85b3fc73e..7b479fe7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -283,6 +283,36 @@ jobs: root: "." paths: - linux-calibnet + build-ntwk-butterfly: + description: | + Compile lotus binaries for the butterfly network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make butterflynet + - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet + - persist_to_workspace: + root: "." + paths: + - linux-butterflynet + build-ntwk-nerpa: + description: | + Compile lotus binaries for the nerpa network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make nerpanet + - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet + - persist_to_workspace: + root: "." + paths: + - linux-nerpanet build-lotus-soup: description: | Compile `lotus-soup` Testground test plan @@ -604,6 +634,12 @@ jobs: - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" workflows: version: 2.1 @@ -709,6 +745,8 @@ workflows: requires: - build-all - build-ntwk-calibration + - build-ntwk-butterfly + - build-ntwk-nerpa filters: branches: ignore: From ed47c3654b89678fa08a0723b857cea47afb3895 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 14:10:55 -0700 Subject: [PATCH 2/8] temporary: remove packer build filters --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b479fe7c..ac1e277e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -747,11 +747,11 @@ workflows: - build-ntwk-calibration - build-ntwk-butterfly - build-ntwk-nerpa - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ + # filters: + # branches: + # ignore: + # - /.*/ + # tags: + # only: + # - /^v\d+\.\d+\.\d+$/ From de586237f4c903d989eb41738319e5d541bef814 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 14:12:58 -0700 Subject: [PATCH 3/8] add other networks to build list --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac1e277e3..295708e14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -701,6 +701,8 @@ workflows: packages: "./conformance" vectors-branch: master - build-ntwk-calibration + - build-ntwk-butterfly + - build-ntwk-nerpa - build-lotus-soup - trigger-testplans: filters: From c17e8b1a03d34b3426935a04adec6c14176535e5 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 14:41:44 -0700 Subject: [PATCH 4/8] network init scripts --- .../packer/scripts/butterflynet/lotus-init.sh | 20 +++++++++++++++++++ tools/packer/scripts/nerpanet/lotus-init.sh | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100755 tools/packer/scripts/butterflynet/lotus-init.sh create mode 100755 tools/packer/scripts/nerpanet/lotus-init.sh diff --git a/tools/packer/scripts/butterflynet/lotus-init.sh b/tools/packer/scripts/butterflynet/lotus-init.sh new file mode 100755 index 000000000..f7afd4dfa --- /dev/null +++ b/tools/packer/scripts/butterflynet/lotus-init.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# This script sets up an initial configuraiton for the lotus daemon and miner +# It will only run once. + +GATE="$LOTUS_PATH"/date_initialized + +# Don't init if already initialized. +if [ -f "GATE" ]; then + echo lotus already initialized. + exit 0 +fi + +# Not importing snapshot on butterflynet +# +# echo importing minimal snapshot +# lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import + +# Block future inits +date > "$GATE" diff --git a/tools/packer/scripts/nerpanet/lotus-init.sh b/tools/packer/scripts/nerpanet/lotus-init.sh new file mode 100755 index 000000000..968ae395c --- /dev/null +++ b/tools/packer/scripts/nerpanet/lotus-init.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# This script sets up an initial configuraiton for the lotus daemon and miner +# It will only run once. + +GATE="$LOTUS_PATH"/date_initialized + +# Don't init if already initialized. +if [ -f "GATE" ]; then + echo lotus already initialized. + exit 0 +fi + +# Not importing snapshot on nerpanet +# +# echo importing minimal snapshot +# lotus daemon --import-snapshot https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car --halt-after-import + +# Block future inits +date > "$GATE" From 49ca9bef414dabfd86c11cd0ac7dbc9473a793bd Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 14:56:12 -0700 Subject: [PATCH 5/8] temporary: build-all without waiting --- .circleci/config.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 295708e14..147f44eb6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -710,13 +710,14 @@ workflows: only: - master - build-debug - - build-all: - requires: - - test-short - filters: - tags: - only: - - /^v\d+\.\d+\.\d+$/ + - build-all + # - build-all: + # requires: + # - test-short + # filters: + # tags: + # only: + # - /^v\d+\.\d+\.\d+$/ - build-macos: requires: - test-short From 99d2b31551525948da020ea2fcf42a1b1f829a42 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 15:07:55 -0700 Subject: [PATCH 6/8] separate different networks --- .circleci/config.yml | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 147f44eb6..8e213df4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -278,11 +278,11 @@ jobs: - install-deps - prepare - run: make calibnet - - run: mkdir linux-calibnet && mv lotus lotus-miner lotus-worker linux-calibnet + - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet - persist_to_workspace: root: "." paths: - - linux-calibnet + - linux-calibrationnet build-ntwk-butterfly: description: | Compile lotus binaries for the butterfly network @@ -619,7 +619,7 @@ jobs: docker push $<>/<>:${tag} done - publish-packer: + publish-packer-mainnet: description: build and push AWS IAM and DigitalOcean droplet. executor: name: packer/default @@ -631,12 +631,39 @@ jobs: - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" + publish-packer-calibrationnet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." - packer/build: template: tools/packer/lotus.pkr.hcl - args: "-var ci_workspace_bins=./linux-calibnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" + args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" + publish-packer-butterflynet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" + publish-packer-nerpanet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" @@ -744,11 +771,17 @@ workflows: path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' - - publish-packer: + - publish-packer-mainnet: requires: - build-all + - publish-packer-calibrationnet: + requires: - build-ntwk-calibration + - publish-packer-butterflynet: + requires: - build-ntwk-butterfly + - publish-packer-nerpanet: + requires: - build-ntwk-nerpa # filters: # branches: From b9ce5254a3b08da97dff90089f48e2463cace730 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 31 Mar 2021 16:12:33 -0700 Subject: [PATCH 7/8] restore filters --- .circleci/config.yml | 77 ++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e213df4e..69b308e19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -727,24 +727,41 @@ workflows: test-suite-name: conformance-bleeding-edge packages: "./conformance" vectors-branch: master - - build-ntwk-calibration - - build-ntwk-butterfly - - build-ntwk-nerpa - - build-lotus-soup - trigger-testplans: filters: branches: only: - master - build-debug - - build-all - # - build-all: - # requires: - # - test-short - # filters: - # tags: - # only: - # - /^v\d+\.\d+\.\d+$/ + - build-all: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-ntwk-calibration: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-ntwk-butterfly: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-ntwk-nerpa: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-lotus-soup - build-macos: requires: - test-short @@ -774,20 +791,40 @@ workflows: - publish-packer-mainnet: requires: - build-all + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ - publish-packer-calibrationnet: requires: - build-ntwk-calibration + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ - publish-packer-butterflynet: requires: - build-ntwk-butterfly + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ - publish-packer-nerpanet: requires: - build-ntwk-nerpa - # filters: - # branches: - # ignore: - # - /.*/ - # tags: - # only: - # - /^v\d+\.\d+\.\d+$/ - + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ From 38a3aaf93659c1d51ab3b4ff920e5f85912649c7 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Thu, 1 Apr 2021 11:20:11 -0700 Subject: [PATCH 8/8] answer yes to ufw --- tools/packer/setup.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/packer/setup.sh b/tools/packer/setup.sh index d7d21664a..6c0742254 100644 --- a/tools/packer/setup.sh +++ b/tools/packer/setup.sh @@ -22,11 +22,14 @@ MANAGED_FILES=( /lib/systemd/system/lotus-daemon.service /lib/systemd/system/lotus-miner.service /etc/motd + /var/lib/lotus/config.toml ) # install libs. -apt update -apt -y install libhwloc15 ocl-icd-libopencl1 +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get -y install libhwloc15 ocl-icd-libopencl1 ufw +apt-get -y upgrade -q -y -u -o Dpkg::Options::="--force-confold" ln -s /usr/lib/x86_64-linux-gnu/libhwloc.so.15 /usr/lib/x86_64-linux-gnu/libhwloc.so.5 # Create lotus user @@ -55,3 +58,10 @@ done # Enable services systemctl daemon-reload systemctl enable lotus-daemon + +# Setup firewall +yes | ufw enable +ufw default deny incoming +ufw default allow outgoing +ufw allow ssh +ufw allow 5678 #libp2p