Use goreleaser to build universal darwin binaries

This is a small refactor of our workflow to test out goreleaser, a yaml
based tool for building, packaging, and releasing go binaries on
multiple platforms. It supports building binaries for to most of the platforms we
care about, including linux and  macos, and also supports publishing
those binaries automatically as releases in Github, homebrew, snap, and
even apt / deb.

If this trial goes well, I think we should eventually replace the entire
release workflow with goreleaser. For now, this test is more tightly
scoped to only automated the MacOS release process, since that is the
one we have the most issues with. This PRi / commit:
- Builds darwin-amd64 and darwin-arm64 binaries of lotus, lotus-miner,
  and lotus-worker
- Packages them into a universal darwin binary
- Publishes those to a release in Github based on the current tag
- Uses the binaries in the release to auto-publish and updated homebrew
  configuration to filecoin-project/homebrew-lotus
- Does a `dry-run` build to produce a snapshot on release branches with
  no tag
- Manually generate and upload checksums after goreleaser

.circleci/config.yml

@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,6 +106,16 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -335,9 +374,13 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
       xcode: "12.5.0"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
@@ -345,48 +388,28 @@ jobs:
       - prepare:
           linux: false
           darwin: true
-      - run:
+      - install_ipfs_macos
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
+      - when:
-          command: make build
+          condition: << parameters.publish >>
-          no_output_timeout: 30m
+          steps:
-      - run:
+            - run: goreleaser release --rm-dist
-          name: check tag and version output match
+            - run: ./scripts/generate-checksums.sh
-          command: ./scripts/version-check.sh ./lotus
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
+          path: dist
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -523,10 +546,6 @@ jobs:
         default: false
         description: publish linux binaries?
         type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
       appimage:
         default: false
         description: publish appimage binaries?
@@ -546,11 +565,6 @@ jobs:
           steps:
             - run: ./scripts/build-arch-bundle.sh linux
             - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
       - when:
           condition: << parameters.appimage >>
           steps:
@@ -1057,10 +1071,20 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-appimage:
           filters:
             branches:
@@ -1069,18 +1093,6 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           name: publish-linux
           linux: true

.circleci/template.yml

@@ -46,6 +46,35 @@ commands:
           steps:
             - run: sudo apt-get update
             - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
       - run: git submodule sync
       - run: git submodule update --init
   download-params:
@@ -77,6 +106,16 @@ commands:
           tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
           mv go-ipfs/ipfs /usr/local/bin/ipfs
           chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
   git_fetch_all_tags:
     steps:
       - run:
@@ -335,9 +374,13 @@ jobs:
       - run:
           name: "trigger payment channel stress testplan on taas"
           command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
   build-macos:
     description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
     macos:
       xcode: "12.5.0"
     working_directory: ~/go/src/github.com/filecoin-project/lotus
@@ -345,48 +388,28 @@ jobs:
       - prepare:
           linux: false
           darwin: true
-      - run:
+      - install_ipfs_macos
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
       - restore_cache:
           name: restore cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
+      - when:
-          command: make build
+          condition: << parameters.publish >>
-          no_output_timeout: 30m
+          steps:
-      - run:
+            - run: goreleaser release --rm-dist
-          name: check tag and version output match
+            - run: ./scripts/generate-checksums.sh
-          command: ./scripts/version-check.sh ./lotus
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
       - store_artifacts:
-          path: lotus
+          path: dist
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
       - persist_to_workspace:
           root: "."
           paths:
-            - darwin
+            - dist
       - save_cache:
           name: save cargo cache
           key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@@ -523,10 +546,6 @@ jobs:
         default: false
         description: publish linux binaries?
         type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
       appimage:
         default: false
         description: publish appimage binaries?
@@ -546,11 +565,6 @@ jobs:
           steps:
             - run: ./scripts/build-arch-bundle.sh linux
             - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
       - when:
           condition: << parameters.appimage >>
           steps:
@@ -817,10 +831,20 @@ workflows:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-lotus-soup
       - build-macos:
+          name: publish-macos
+          publish: true
           filters:
+            branches:
+              ignore:
+                - /.*/
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
       - build-appimage:
           filters:
             branches:
@@ -829,18 +853,6 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
       - publish:
           name: publish-linux
           linux: true

.gitignore

@@ -51,3 +51,5 @@ scratchpad
 
 build/builtin-actors/v*
 build/builtin-actors/*.car
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,166 @@
+project_name: lotus
+before:
+  hooks:
+    - go mod tidy
+    - make deps
+
+universal_binaries:
+  - id: lotus
+    replace: true
+    name_template: lotus
+    ids:
+      - lotus_darwin_amd64
+      - lotus_darwin_arm64
+  - id: lotus-miner
+    replace: true
+    name_template: lotus-miner
+    ids:
+      - lotus-miner_darwin_amd64
+      - lotus-miner_darwin_arm64
+  - id: lotus-worker
+    replace: true
+    name_template: lotus-worker
+    ids:
+      - lotus-worker_darwin_amd64
+      - lotus-worker_darwin_arm64
+
+builds:
+  - id: lotus_darwin_amd64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_amd64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_amd64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus_darwin_arm64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_arm64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_arm64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+#  - id: linux
+#    main: ./cmd/lotus
+#    binary: lotus
+#    goos:
+#      - linux
+#    goarch:
+#      - amd64
+#    env:
+#      - CGO_ENABLED=1
+#    ldflags:
+#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+
+archives:
+  - id: primary
+    format: tar.gz
+    wrap_in_directory: true
+    files:
+      # this is a dumb but required hack so it doesn't include the default files
+      # https://github.com/goreleaser/goreleaser/issues/602
+      - _n_o_n_e_*
+
+release:
+  github:
+    owner: ianconsolata
+    name: lotus
+  prerelease: auto
+  mode: append
+  name_template: "Release v{{.Version}}"
+
+
+brews:
+  - tap:
+      owner: ianconsolata
+      name: homebrew-lotus
+      branch: master
+    ids:
+      - primary
+    install: |
+      bin.install "lotus"
+      bin.install "lotus-miner"
+      bin.install "lotus-worker"
+    test: |
+      system "#{bin}/lotus --version"
+      system "#{bin}/lotus-miner --version"
+      system "#{bin}/lotus-worker --version"
+    folder: Formula
+    homepage: "https://filecoin.io"
+    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
+    license: MIT
+    dependencies:
+      - name: pkg-config
+      - name: jq
+      - name: bzr
+      - name: hwloc
+
+# produced manually so we can include cid checksums
+checksum:
+  disable: true
+
+snapshot:
+  name_template: "{{ .Tag }}"

scripts/generate-checksums.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -exo
+
+REQUIRED=(
+    "sha512sum"
+    "ipfs"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+# start ipfs
+export IPFS_PATH=`mktemp -d`
+ipfs init
+ipfs daemon &
+PID="$!"
+trap "kill -9 ${PID}" EXIT
+
+# generate checksums
+for FILE in dist/*.tar.gz
+do
+  sha512sum "${FILE}" > "${FILE}.sha512"
+  until ipfs add -q "${FILE}" > "${FILE}.cid"
+  do
+    echo "Waiting for ipfs daemon to start..."
+    sleep 2
+  done
+done

scripts/publish-checksums.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -exo
+
+pushd dist
+
+# make sure we have a token set, api requests won't work otherwise
+if [ -z "${GITHUB_TOKEN}" ]; then
+  echo "\${GITHUB_TOKEN} not set, publish failed"
+  exit 1
+fi
+
+REQUIRED=(
+    "jq"
+    "curl"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+#see if the release already exists by tag
+RELEASE_RESPONSE=`
+  curl \
+    --fail \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}"
+`
+RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'`
+
+if [ "${RELEASE_ID}" = "null" ]; then
+  echo "creating release"
+
+  COND_CREATE_DISCUSSION=""
+  PRERELEASE=true
+  if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\","
+    PRERELEASE=false
+  fi
+
+  RELEASE_DATA="{
+    \"tag_name\": \"${CIRCLE_TAG}\",
+    \"target_commitish\": \"${CIRCLE_SHA1}\",
+    ${COND_CREATE_DISCUSSION}
+    \"name\": \"${CIRCLE_TAG}\",
+    \"body\": \"\",
+    \"prerelease\": ${PRERELEASE}
+  }"
+
+  # create it if it doesn't exist yet
+  RELEASE_RESPONSE=`
+    curl \
+        --fail \
+        --request POST \
+        --header "Authorization: token ${GITHUB_TOKEN}" \
+        --header "Content-Type: application/json" \
+        --data "${RELEASE_DATA}" \
+        "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases"
+  `
+else
+  echo "release already exists"
+fi
+
+RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1`
+echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}"
+
+for CHECKSUM_FILE in *.{cid,sha512}
+do
+  echo "Uploading ${CHECKSUM_FILE}..."
+  curl \
+    --fail \
+    --request POST \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    --header "Content-Type: application/octet-stream" \
+    --data-binary "@${CHECKSUM_FILE}" \
+    "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")"
+
+  echo "Uploaded ${CHECKSUM_FILE}"
+done
+
+popd