Merge pull request #9600 from filecoin-project/refactor-release-pipeline

_ci_ Refactor release pipeline to better support m1 builds

.circleci/config.yml

@@ -23,11 +23,6 @@ executors:
 
 
 commands:
-  install-deps:
-    steps:
-      - run: |
-          sudo apt update
-          sudo apt install python-is-python3
   prepare:
     parameters:
       linux:
@@ -41,9 +36,8 @@ commands:
     steps:
       - checkout
       - git_fetch_all_tags
-      - checkout
       - when:
-          condition: << parameters.linux >>
+          condition: <<parameters.linux>>
           steps:
             - run:
                 name: Check Go Version
@@ -56,6 +50,8 @@ commands:
                   fi
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+            - run: sudo apt-get install python-is-python3
+
       - when:
           condition: <<parameters.darwin>>
           steps:
@@ -70,21 +66,12 @@ commands:
                   echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
             - run: go version
             - run:
-                name: Install pkg-config, goreleaser, and sha512sum
+                name: Install dependencies with Homebrew
-                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config coreutils jq hwloc
             - run:
                 name: Install Rust
                 command: |
                   curl https://sh.rustup.rs -sSf | sh -s -- -y
-            - run:
-                name: Install hwloc
-                command: |
-                  mkdir ~/hwloc
-                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-                  cd ~/hwloc
-                  tar -xvzpf hwloc-2.4.1.tar.gz
-                  cd hwloc-2.4.1
-                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -104,28 +91,12 @@ commands:
   install_ipfs:
     steps:
       - run: |
-          apt update
+          curl -O https://dist.ipfs.tech/kubo/v0.16.0/kubo_v0.16.0_linux-amd64.tar.gz
-          apt install -y wget
+          tar -xvzf kubo_v0.16.0_linux-amd64.tar.gz
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512
-          if [ "$(sha512sum go-ipfs_v0.12.2_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512)" ]
-          then
-            echo "ipfs failed checksum check"
-            exit 1
-          fi
-          tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
-          mv go-ipfs/ipfs /usr/local/bin/ipfs
-          chmod +x /usr/local/bin/ipfs
-  install_ipfs_macos:
-    steps:
-      - run: |
-          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
-          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
           pushd kubo
           sudo bash install.sh
           popd
-          rm -rf kubo/
+          rm -rf kubo
-          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -156,7 +127,6 @@ jobs:
   mod-tidy-check:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run: go mod tidy -v
       - run:
@@ -164,34 +134,9 @@ jobs:
           command: |
             git --no-pager diff go.mod go.sum
             git --no-pager diff --quiet go.mod go.sum
-  build-linux:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: sudo apt-get update
-      - run: sudo apt-get install npm
-      - run:
-          command: make buildall
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
-      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux
-
   build-debug:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run:
           command: make debug
@@ -224,7 +169,6 @@ jobs:
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           command: make deps lotus
@@ -268,7 +212,6 @@ jobs:
           submodule is used.
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           command: make deps lotus
@@ -311,36 +254,6 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibrationnet
-  build-ntwk-butterfly:
-    description: |
-      Compile lotus binaries for the butterfly network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make butterflynet
-      - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-butterflynet
   build-lotus-soup:
     description: |
       Compile `lotus-soup` Testground test plan
@@ -348,7 +261,6 @@ jobs:
       <<: *test-params
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run: cd extern/filecoin-ffi && make
       - run:
@@ -364,7 +276,6 @@ jobs:
       <<: *test-params
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           name: "download testground"
@@ -384,48 +295,104 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-  build-macos:
+
+  build-linux-amd64:
+    executor: golang
+    steps:
+      - prepare
+      - run: make lotus lotus-miner lotus-worker
+      - run:
+          name: check tag and version output match
+          command: ./scripts/version-check.sh ./lotus
+      - run: |
+          mkdir -p /tmp/workspace/linux_amd64_v1 && \
+          mv lotus lotus-miner lotus-worker /tmp/workspace/linux_amd64_v1/
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - linux_amd64_v1
+
+  build-darwin-amd64:
     description: build darwin lotus binary
-    parameters:
+    working_directory: ~/go/src/github.com/filecoin-project/lotus
-      publish:
-        default: false
-        description: publish github release and homebrew?
-        type: boolean
     macos:
       xcode: "13.4.1"
-    working_directory: ~/go/src/github.com/filecoin-project/lotus
     steps:
       - prepare:
           linux: false
           darwin: true
-      - install_ipfs_macos
+      - run: make lotus lotus-miner lotus-worker
-      - restore_cache:
+      - run:
-          name: restore cargo cache
+          name: check tag and version output match
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
+          command: ./scripts/version-check.sh ./lotus
+      - run: |
+          mkdir -p /tmp/workspace/darwin_amd64_v1 && \
+          mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_amd64_v1/
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - darwin_amd64_v1
+
+  build-darwin-arm64:
+    description: self-hosted m1 runner
+    working_directory: ~/go/src/github.com/filecoin-project/lotus
+    machine: true
+    resource_class: filecoin-project/self-hosted-m1
+    steps:
+      - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV"
+      - prepare:
+          linux: false
+          darwin: true
+      - run: |
+          export CPATH=$(brew --prefix)/include
+          export LIBRARY_PATH=$(brew --prefix)/lib
+          make lotus lotus-miner lotus-worker
+      - run:
+          name: check tag and version output match
+          command: ./scripts/version-check.sh ./lotus
+      - run: |
+          mkdir -p /tmp/workspace/darwin_arm64 && \
+          mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_arm64/
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - darwin_arm64
+      - run:
+          command: make clean
+          when: always
+      - run:
+          name: cleanup homebrew
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall pkg-config coreutils jq hwloc
+          when: always
+
+  release:
+    executor: golang
+    parameters:
+      dry-run:
+        default: false
+        description: should this release actually publish it's artifacts?
+        type: boolean
+    steps:
+      - checkout
+      - run: |
+          echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list
+          sudo apt update
+          sudo apt install goreleaser-pro
+      - install_ipfs
+      - attach_workspace:
+          at: /tmp/workspace
       - when:
-          condition: << parameters.publish >>
+          condition: << parameters.dry-run >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
+      - when:
+          condition:
+            not: << parameters.dry-run >>
           steps:
             - run: goreleaser release --rm-dist
             - run: ./scripts/generate-checksums.sh
             - run: ./scripts/publish-checksums.sh
-      - when:
-          condition:
-            not: << parameters.publish >>
-          steps:
-            - run: goreleaser release --rm-dist --snapshot
-            - run: ./scripts/generate-checksums.sh
-      - store_artifacts:
-          path: dist
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - dist
-      - save_cache:
-          name: save cargo cache
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-          paths:
-            - "~/.rustup"
-            - "~/.cargo"
 
   build-appimage:
     machine:
@@ -433,7 +400,7 @@ jobs:
     steps:
       - checkout
       - attach_workspace:
-          at: "."
+          at: /tmp/workspace
       - run:
           name: Update Go
           command: |
@@ -468,13 +435,11 @@ jobs:
           command: |
             sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
             make appimage
-      - run:
+      - run: |
-          name: prepare workspace
+            mkdir -p /tmp/workspace/appimage && \
-          command: |
+            mv Lotus-*.AppImage /tmp/workspace/appimage/
-            mkdir appimage
-            mv Lotus-*.AppImage appimage
       - persist_to_workspace:
-          root: "."
+          root: /tmp/workspace
           paths:
             - appimage
 
@@ -482,7 +447,6 @@ jobs:
   gofmt:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run:
           command: "! go fmt ./... 2>&1 | read"
@@ -490,7 +454,6 @@ jobs:
   gen-check:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run: make deps
       - run: go install golang.org/x/tools/cmd/goimports
@@ -505,7 +468,6 @@ jobs:
   docs-check:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run: go install golang.org/x/tools/cmd/goimports
       - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
@@ -542,7 +504,6 @@ jobs:
           Arguments to pass to golangci-lint
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           command: make deps
@@ -570,13 +531,13 @@ jobs:
     steps:
       - run:
           name: Install git jq curl
-          command: apt update && apt install -y git jq curl
+          command: apt update && apt install -y git jq curl sudo
       - checkout
       - git_fetch_all_tags
       - checkout
       - install_ipfs
       - attach_workspace:
-          at: "."
+          at: /tmp/workspace
       - when:
           condition: << parameters.linux >>
           steps:
@@ -751,8 +712,6 @@ jobs:
       name: packer
     steps:
       - checkout
-      - attach_workspace:
-          at: "."
       - packer_build:
           template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
@@ -1141,41 +1100,12 @@ workflows:
               only:
                 - master
       - build-debug
-      - build-linux:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-calibration:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-butterfly:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
-      - build-macos:
+
-          name: publish-macos
+  release:
-          publish: true
+    jobs:
-          filters:
+      - build-linux-amd64:
-            branches:
+          name: "Build ( linux / amd64 )"
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-      - build-macos:
-          filters:
-            branches:
-              only:
-                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
-      - build-appimage:
           filters:
             branches:
               only:
@@ -1183,11 +1113,30 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
+      - build-darwin-amd64:
-          name: publish-linux
+          name: "Build ( darwin / amd64 )"
-          linux: true
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-darwin-arm64:
+          name: "Build ( darwin / arm64 )"
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - release:
+          name: "Release"
           requires:
-            - build-linux
+            - "Build ( darwin / amd64 )"
+            - "Build ( linux / amd64 )"
+            - "Build ( darwin / arm64 )"
           filters:
             branches:
               ignore:
@@ -1195,11 +1144,31 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - release:
+          name: "Release (dry-run)"
+          dry-run: true
+          requires:
+            - "Build ( darwin / amd64 )"
+            - "Build ( linux / amd64 )"
+            - "Build ( darwin / arm64 )"
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-appimage:
+          name: "Build AppImage"
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
-          name: publish-appimage
+          name: "Publish AppImage"
           appimage: true
           requires:
-            - build-appimage
+            - "Build AppImage"
           filters:
             branches:
               ignore:
@@ -1208,14 +1177,14 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
-          name: build-and-push/lotus-all-in-one
+          name: "Publish ECR (lotus-all-in-one)"
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
-          name: build-and-push/lotus-test
+          name: "Publish ECR (lotus-test)"
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test
@@ -1266,7 +1235,7 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-dockerhub:
-          name: publish-dockerhub
+          name: "Publish Dockerhub (stable)"
           tag: stable
           filters:
             branches:
@@ -1274,7 +1243,17 @@ workflows:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
+      - publish-dockerhub:
+          name: "Publish Dockerhub (candidate)"
+          tag: candidate
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
 
   nightly:
     triggers:

.circleci/template.yml

@@ -23,11 +23,6 @@ executors:
 
 
 commands:
-  install-deps:
-    steps:
-      - run: |
-          sudo apt update
-          sudo apt install python-is-python3
   prepare:
     parameters:
       linux:
@@ -41,9 +36,8 @@ commands:
     steps:
       - checkout
       - git_fetch_all_tags
-      - checkout
       - when:
-          condition: << parameters.linux >>
+          condition: <<parameters.linux>>
           steps:
             - run:
                 name: Check Go Version
@@ -56,6 +50,8 @@ commands:
                   fi
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+            - run: sudo apt-get install python-is-python3
+
       - when:
           condition: <<parameters.darwin>>
           steps:
@@ -70,21 +66,12 @@ commands:
                   echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
             - run: go version
             - run:
-                name: Install pkg-config, goreleaser, and sha512sum
+                name: Install dependencies with Homebrew
-                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config coreutils jq hwloc
             - run:
                 name: Install Rust
                 command: |
                   curl https://sh.rustup.rs -sSf | sh -s -- -y
-            - run:
-                name: Install hwloc
-                command: |
-                  mkdir ~/hwloc
-                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-                  cd ~/hwloc
-                  tar -xvzpf hwloc-2.4.1.tar.gz
-                  cd hwloc-2.4.1
-                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -104,28 +91,12 @@ commands:
   install_ipfs:
     steps:
       - run: |
-          apt update
+          curl -O https://dist.ipfs.tech/kubo/v0.16.0/kubo_v0.16.0_linux-amd64.tar.gz
-          apt install -y wget
+          tar -xvzf kubo_v0.16.0_linux-amd64.tar.gz
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz
-          wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512
-          if [ "$(sha512sum go-ipfs_v0.12.2_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512)" ]
-          then
-            echo "ipfs failed checksum check"
-            exit 1
-          fi
-          tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
-          mv go-ipfs/ipfs /usr/local/bin/ipfs
-          chmod +x /usr/local/bin/ipfs
-  install_ipfs_macos:
-    steps:
-      - run: |
-          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
-          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
           pushd kubo
           sudo bash install.sh
           popd
-          rm -rf kubo/
+          rm -rf kubo
-          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -156,7 +127,6 @@ jobs:
   mod-tidy-check:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run: go mod tidy -v
       - run:
@@ -164,34 +134,9 @@ jobs:
           command: |
             git --no-pager diff go.mod go.sum
             git --no-pager diff --quiet go.mod go.sum
-  build-linux:
-    executor: golang
-    steps:
-      - install-deps
-      - prepare
-      - run: sudo apt-get update
-      - run: sudo apt-get install npm
-      - run:
-          command: make buildall
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
-      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux
-
   build-debug:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run:
           command: make debug
@@ -224,7 +169,6 @@ jobs:
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           command: make deps lotus
@@ -268,7 +212,6 @@ jobs:
           submodule is used.
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           command: make deps lotus
@@ -311,36 +254,6 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/conformance-coverage.html
-  build-ntwk-calibration:
-    description: |
-      Compile lotus binaries for the calibration network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make calibnet
-      - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-calibrationnet
-  build-ntwk-butterfly:
-    description: |
-      Compile lotus binaries for the butterfly network
-    parameters:
-      <<: *test-params
-    executor: << parameters.executor >>
-    steps:
-      - install-deps
-      - prepare
-      - run: make butterflynet
-      - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - linux-butterflynet
   build-lotus-soup:
     description: |
       Compile `lotus-soup` Testground test plan
@@ -348,7 +261,6 @@ jobs:
       <<: *test-params
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run: cd extern/filecoin-ffi && make
       - run:
@@ -364,7 +276,6 @@ jobs:
       <<: *test-params
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           name: "download testground"
@@ -384,48 +295,104 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-  build-macos:
+
+  build-linux-amd64:
+    executor: golang
+    steps:
+      - prepare
+      - run: make lotus lotus-miner lotus-worker
+      - run:
+          name: check tag and version output match
+          command: ./scripts/version-check.sh ./lotus
+      - run: |
+          mkdir -p /tmp/workspace/linux_amd64_v1 && \
+          mv lotus lotus-miner lotus-worker /tmp/workspace/linux_amd64_v1/
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - linux_amd64_v1
+
+  build-darwin-amd64:
     description: build darwin lotus binary
-    parameters:
+    working_directory: ~/go/src/github.com/filecoin-project/lotus
-      publish:
-        default: false
-        description: publish github release and homebrew?
-        type: boolean
     macos:
       xcode: "13.4.1"
-    working_directory: ~/go/src/github.com/filecoin-project/lotus
     steps:
       - prepare:
           linux: false
           darwin: true
-      - install_ipfs_macos
+      - run: make lotus lotus-miner lotus-worker
-      - restore_cache:
+      - run:
-          name: restore cargo cache
+          name: check tag and version output match
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
+          command: ./scripts/version-check.sh ./lotus
+      - run: |
+          mkdir -p /tmp/workspace/darwin_amd64_v1 && \
+          mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_amd64_v1/
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - darwin_amd64_v1
+
+  build-darwin-arm64:
+    description: self-hosted m1 runner
+    working_directory: ~/go/src/github.com/filecoin-project/lotus
+    machine: true
+    resource_class: filecoin-project/self-hosted-m1
+    steps:
+      - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV"
+      - prepare:
+          linux: false
+          darwin: true
+      - run: |
+          export CPATH=$(brew --prefix)/include
+          export LIBRARY_PATH=$(brew --prefix)/lib
+          make lotus lotus-miner lotus-worker
+      - run:
+          name: check tag and version output match
+          command: ./scripts/version-check.sh ./lotus
+      - run: |
+          mkdir -p /tmp/workspace/darwin_arm64 && \
+          mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_arm64/
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - darwin_arm64
+      - run:
+          command: make clean
+          when: always
+      - run:
+          name: cleanup homebrew
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall pkg-config coreutils jq hwloc
+          when: always
+
+  release:
+    executor: golang
+    parameters:
+      dry-run:
+        default: false
+        description: should this release actually publish it's artifacts?
+        type: boolean
+    steps:
+      - checkout
+      - run: |
+          echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list
+          sudo apt update
+          sudo apt install goreleaser-pro
+      - install_ipfs
+      - attach_workspace:
+          at: /tmp/workspace
       - when:
-          condition: << parameters.publish >>
+          condition: << parameters.dry-run >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
+      - when:
+          condition:
+            not: << parameters.dry-run >>
           steps:
             - run: goreleaser release --rm-dist
             - run: ./scripts/generate-checksums.sh
             - run: ./scripts/publish-checksums.sh
-      - when:
-          condition:
-            not: << parameters.publish >>
-          steps:
-            - run: goreleaser release --rm-dist --snapshot
-            - run: ./scripts/generate-checksums.sh
-      - store_artifacts:
-          path: dist
-      - persist_to_workspace:
-          root: "."
-          paths:
-            - dist
-      - save_cache:
-          name: save cargo cache
-          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-          paths:
-            - "~/.rustup"
-            - "~/.cargo"
 
   build-appimage:
     machine:
@@ -433,7 +400,7 @@ jobs:
     steps:
       - checkout
       - attach_workspace:
-          at: "."
+          at: /tmp/workspace
       - run:
           name: Update Go
           command: |
@@ -468,13 +435,11 @@ jobs:
           command: |
             sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
             make appimage
-      - run:
+      - run: |
-          name: prepare workspace
+            mkdir -p /tmp/workspace/appimage && \
-          command: |
+            mv Lotus-*.AppImage /tmp/workspace/appimage/
-            mkdir appimage
-            mv Lotus-*.AppImage appimage
       - persist_to_workspace:
-          root: "."
+          root: /tmp/workspace
           paths:
             - appimage
 
@@ -482,7 +447,6 @@ jobs:
   gofmt:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run:
           command: "! go fmt ./... 2>&1 | read"
@@ -490,7 +454,6 @@ jobs:
   gen-check:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run: make deps
       - run: go install golang.org/x/tools/cmd/goimports
@@ -505,7 +468,6 @@ jobs:
   docs-check:
     executor: golang
     steps:
-      - install-deps
       - prepare
       - run: go install golang.org/x/tools/cmd/goimports
       - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
@@ -542,7 +504,6 @@ jobs:
           Arguments to pass to golangci-lint
     executor: << parameters.executor >>
     steps:
-      - install-deps
       - prepare
       - run:
           command: make deps
@@ -570,13 +531,13 @@ jobs:
     steps:
       - run:
           name: Install git jq curl
-          command: apt update && apt install -y git jq curl
+          command: apt update && apt install -y git jq curl sudo
       - checkout
       - git_fetch_all_tags
       - checkout
       - install_ipfs
       - attach_workspace:
-          at: "."
+          at: /tmp/workspace
       - when:
           condition: << parameters.linux >>
           steps:
@@ -751,8 +712,6 @@ jobs:
       name: packer
     steps:
       - checkout
-      - attach_workspace:
-          at: "."
       - packer_build:
           template: tools/packer/lotus-snap.pkr.hcl
   publish-dockerhub:
@@ -846,41 +805,12 @@ workflows:
               only:
                 - master
       - build-debug
-      - build-linux:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-calibration:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-ntwk-butterfly:
-          filters:
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
-      - build-macos:
+
-          name: publish-macos
+  release:
-          publish: true
+    jobs:
-          filters:
+      - build-linux-amd64:
-            branches:
+          name: "Build ( linux / amd64 )"
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-      - build-macos:
-          filters:
-            branches:
-              only:
-                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
-      - build-appimage:
           filters:
             branches:
               only:
@@ -888,11 +818,30 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
+      - build-darwin-amd64:
-          name: publish-linux
+          name: "Build ( darwin / amd64 )"
-          linux: true
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-darwin-arm64:
+          name: "Build ( darwin / arm64 )"
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - release:
+          name: "Release"
           requires:
-            - build-linux
+            - "Build ( darwin / amd64 )"
+            - "Build ( linux / amd64 )"
+            - "Build ( darwin / arm64 )"
           filters:
             branches:
               ignore:
@@ -900,11 +849,31 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - release:
+          name: "Release (dry-run)"
+          dry-run: true
+          requires:
+            - "Build ( darwin / amd64 )"
+            - "Build ( linux / amd64 )"
+            - "Build ( darwin / arm64 )"
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-appimage:
+          name: "Build AppImage"
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
-          name: publish-appimage
+          name: "Publish AppImage"
           appimage: true
           requires:
-            - build-appimage
+            - "Build AppImage"
           filters:
             branches:
               ignore:
@@ -913,14 +882,14 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-and-push-image:
-          name: build-and-push/lotus-all-in-one
+          name: "Publish ECR (lotus-all-in-one)"
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-dev
           tag: '${CIRCLE_SHA1:0:8}'
           target: lotus-all-in-one
       - build-and-push-image:
-          name: build-and-push/lotus-test
+          name: "Publish ECR (lotus-test)"
           dockerfile: Dockerfile.lotus
           path: .
           repo: lotus-test
@@ -971,7 +940,7 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-dockerhub:
-          name: publish-dockerhub
+          name: "Publish Dockerhub (stable)"
           tag: stable
           filters:
             branches:
@@ -979,7 +948,17 @@ workflows:
                 - /.*/
             tags:
               only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+                - /^v\d+\.\d+\.\d+$/
+      - publish-dockerhub:
+          name: "Publish Dockerhub (candidate)"
+          tag: candidate
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
 
   nightly:
     triggers:

.goreleaser.yaml

@@ -1,119 +1,50 @@
 project_name: lotus
-before:
-  hooks:
-    - go mod tidy
-    - make deps
 
 universal_binaries:
   - id: lotus
     replace: true
     name_template: lotus
-    ids:
-      - lotus_darwin_amd64
-      - lotus_darwin_arm64
   - id: lotus-miner
     replace: true
     name_template: lotus-miner
-    ids:
-      - lotus-miner_darwin_amd64
-      - lotus-miner_darwin_arm64
   - id: lotus-worker
     replace: true
     name_template: lotus-worker
-    ids:
-      - lotus-worker_darwin_amd64
-      - lotus-worker_darwin_arm64
 
 builds:
-  - id: lotus_darwin_amd64
+  - id: lotus
-    main: ./cmd/lotus
+    builder: prebuilt
-    binary: lotus
     goos:
       - darwin
     goarch:
       - amd64
-    env:
+      - arm64
-      - CGO_ENABLED=1
+    goamd64:
-      - FFI_BUILD_FROM_SOURCE=1
+      - v1
-    ldflags:
+    prebuilt:
-      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+      path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus
-  - id: lotus-miner_darwin_amd64
+  - id: lotus-miner
-    main: ./cmd/lotus-miner
+    builder: prebuilt
-    binary: lotus-miner
     goos:
       - darwin
     goarch:
       - amd64
-    env:
+      - arm64
-      - CGO_ENABLED=1
+    goamd64:
-      - FFI_BUILD_FROM_SOURCE=1
+      - v1
-    ldflags:
+    prebuilt:
-      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+      path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner
-  - id: lotus-worker_darwin_amd64
+  - id: lotus-worker
-    main: ./cmd/lotus-worker
+    builder: prebuilt
-    binary: lotus-worker
     goos:
       - darwin
     goarch:
       - amd64
-    env:
-      - CGO_ENABLED=1
-      - FFI_BUILD_FROM_SOURCE=1
-    ldflags:
-      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
-  - id: lotus_darwin_arm64
-    main: ./cmd/lotus
-    binary: lotus
-    goos:
-      - darwin
-    goarch:
       - arm64
-    env:
+    goamd64:
-      - CGO_ENABLED=1
+      - v1
-      - FFI_BUILD_FROM_SOURCE=1
+    prebuilt:
-      - CPATH=/opt/homebrew/include
+      path: /tmp/workspace/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker
-      - LIBRARY_PATH=/opt/homebrew/lib
-    ldflags:
-      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
-  - id: lotus-miner_darwin_arm64
-    main: ./cmd/lotus-miner
-    binary: lotus-miner
-    goos:
-      - darwin
-    goarch:
-      - arm64
-    env:
-      - CGO_ENABLED=1
-      - FFI_BUILD_FROM_SOURCE=1
-      - CPATH=/opt/homebrew/include
-      - LIBRARY_PATH=/opt/homebrew/lib
-    ldflags:
-      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
-  - id: lotus-worker_darwin_arm64
-    main: ./cmd/lotus-worker
-    binary: lotus-worker
-    goos:
-      - darwin
-    goarch:
-      - arm64
-    env:
-      - CGO_ENABLED=1
-      - FFI_BUILD_FROM_SOURCE=1
-      - CPATH=/opt/homebrew/include
-      - LIBRARY_PATH=/opt/homebrew/lib
-    ldflags:
-      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
-#  - id: linux
-#    main: ./cmd/lotus
-#    binary: lotus
-#    goos:
-#      - linux
-#    goarch:
-#      - amd64
-#    env:
-#      - CGO_ENABLED=1
-#    ldflags:
-#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
 
 archives:
   - id: primary
@@ -131,7 +62,6 @@ release:
   prerelease: auto
   name_template: "Release v{{.Version}}"
 
-
 brews:
   - tap:
       owner: filecoin-project
@@ -151,6 +81,7 @@ brews:
     homepage: "https://filecoin.io"
     description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
     license: MIT
+    skip_upload: auto
     dependencies:
       - name: hwloc