Use goreleaser to build universal darwin binaries

This is a small refactor of our workflow to test out goreleaser, a yaml based tool for building, packaging, and releasing go binaries on multiple platforms. It supports building binaries for to most of the platforms we care about, including linux and macos, and also supports publishing those binaries automatically as releases in Github, homebrew, snap, and even apt / deb. If this trial goes well, I think we should eventually replace the entire release workflow with goreleaser. For now, this test is more tightly scoped to only automated the MacOS release process, since that is the one we have the most issues with. This PRi / commit: - Builds darwin-amd64 and darwin-arm64 binaries of lotus, lotus-miner, and lotus-worker - Packages them into a universal darwin binary - Publishes those to a release in Github based on the current tag - Uses the binaries in the release to auto-publish and updated homebrew configuration to filecoin-project/homebrew-lotus - Does a `dry-run` build to produce a snapshot on release branches with no tag - Manually generate and upload checksums after goreleaser
2022-07-28 19:23:03 -07:00 · 2022-07-28 19:23:03 -07:00 · 7f1b4dc438
commit 7f1b4dc438
parent a843c52e38
6 changed files with 415 additions and 114 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -46,6 +46,35 @@ commands:
          steps:
            - run: sudo apt-get update
            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
      - run: git submodule sync
      - run: git submodule update --init
  download-params:
@ -77,6 +106,16 @@ commands:
          tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
          mv go-ipfs/ipfs /usr/local/bin/ipfs
          chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
  git_fetch_all_tags:
    steps:
      - run:
@ -335,9 +374,13 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
  build-macos:
    description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
    macos:
      xcode: "12.5.0"
    working_directory: ~/go/src/github.com/filecoin-project/lotus
@ -345,48 +388,28 @@ jobs:
      - prepare:
          linux: false
          darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
      - restore_cache:
          name: restore cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
      - persist_to_workspace:
          root: "."
          paths:
-            - darwin
+            - dist
      - save_cache:
          name: save cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@ -523,10 +546,6 @@ jobs:
        default: false
        description: publish linux binaries?
        type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
      appimage:
        default: false
        description: publish appimage binaries?
@ -546,11 +565,6 @@ jobs:
          steps:
            - run: ./scripts/build-arch-bundle.sh linux
            - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
      - when:
          condition: << parameters.appimage >>
          steps:
@ -1057,10 +1071,20 @@ workflows:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-lotus-soup
      - build-macos:
+          name: publish-macos
+          publish: true
          filters:
+            branches:
+              ignore:
+                - /.*/
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-appimage:
          filters:
            branches:
@ -1069,18 +1093,6 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          name: publish-linux
          linux: true
--- a/.circleci/template.yml
+++ b/.circleci/template.yml
@ -46,6 +46,35 @@ commands:
          steps:
            - run: sudo apt-get update
            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
      - run: git submodule sync
      - run: git submodule update --init
  download-params:
@ -77,6 +106,16 @@ commands:
          tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
          mv go-ipfs/ipfs /usr/local/bin/ipfs
          chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
  git_fetch_all_tags:
    steps:
      - run:
@ -335,9 +374,13 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
  build-macos:
    description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
    macos:
      xcode: "12.5.0"
    working_directory: ~/go/src/github.com/filecoin-project/lotus
@ -345,48 +388,28 @@ jobs:
      - prepare:
          linux: false
          darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
      - restore_cache:
          name: restore cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
      - persist_to_workspace:
          root: "."
          paths:
-            - darwin
+            - dist
      - save_cache:
          name: save cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@ -523,10 +546,6 @@ jobs:
        default: false
        description: publish linux binaries?
        type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
      appimage:
        default: false
        description: publish appimage binaries?
@ -546,11 +565,6 @@ jobs:
          steps:
            - run: ./scripts/build-arch-bundle.sh linux
            - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
      - when:
          condition: << parameters.appimage >>
          steps:
@ -817,10 +831,20 @@ workflows:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-lotus-soup
      - build-macos:
+          name: publish-macos
+          publish: true
          filters:
+            branches:
+              ignore:
+                - /.*/
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-appimage:
          filters:
            branches:
@ -829,18 +853,6 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          name: publish-linux
          linux: true
--- a/.gitignore
+++ b/.gitignore
@ -51,3 +51,5 @@ scratchpad

 build/builtin-actors/v*
 build/builtin-actors/*.car
+
+dist/
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@ -0,0 +1,166 @@
+project_name: lotus
+before:
+  hooks:
+    - go mod tidy
+    - make deps
+
+universal_binaries:
+  - id: lotus
+    replace: true
+    name_template: lotus
+    ids:
+      - lotus_darwin_amd64
+      - lotus_darwin_arm64
+  - id: lotus-miner
+    replace: true
+    name_template: lotus-miner
+    ids:
+      - lotus-miner_darwin_amd64
+      - lotus-miner_darwin_arm64
+  - id: lotus-worker
+    replace: true
+    name_template: lotus-worker
+    ids:
+      - lotus-worker_darwin_amd64
+      - lotus-worker_darwin_arm64
+
+builds:
+  - id: lotus_darwin_amd64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_amd64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_amd64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus_darwin_arm64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_arm64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_arm64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+#  - id: linux
+#    main: ./cmd/lotus
+#    binary: lotus
+#    goos:
+#      - linux
+#    goarch:
+#      - amd64
+#    env:
+#      - CGO_ENABLED=1
+#    ldflags:
+#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+
+archives:
+  - id: primary
+    format: tar.gz
+    wrap_in_directory: true
+    files:
+      # this is a dumb but required hack so it doesn't include the default files
+      # https://github.com/goreleaser/goreleaser/issues/602
+      - _n_o_n_e_*
+
+release:
+  github:
+    owner: ianconsolata
+    name: lotus
+  prerelease: auto
+  mode: append
+  name_template: "Release v{{.Version}}"
+
+
+brews:
+  - tap:
+      owner: ianconsolata
+      name: homebrew-lotus
+      branch: master
+    ids:
+      - primary
+    install: |
+      bin.install "lotus"
+      bin.install "lotus-miner"
+      bin.install "lotus-worker"
+    test: |
+      system "#{bin}/lotus --version"
+      system "#{bin}/lotus-miner --version"
+      system "#{bin}/lotus-worker --version"
+    folder: Formula
+    homepage: "https://filecoin.io"
+    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
+    license: MIT
+    dependencies:
+      - name: pkg-config
+      - name: jq
+      - name: bzr
+      - name: hwloc
+
+# produced manually so we can include cid checksums
+checksum:
+  disable: true
+
+snapshot:
+  name_template: "{{ .Tag }}"
--- a/scripts/generate-checksums.sh
+++ b/scripts/generate-checksums.sh
@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -exo
+
+REQUIRED=(
+    "sha512sum"
+    "ipfs"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+# start ipfs
+export IPFS_PATH=`mktemp -d`
+ipfs init
+ipfs daemon &
+PID="$!"
+trap "kill -9 ${PID}" EXIT
+
+# generate checksums
+for FILE in dist/*.tar.gz
+do
+  sha512sum "${FILE}" > "${FILE}.sha512"
+  until ipfs add -q "${FILE}" > "${FILE}.cid"
+  do
+    echo "Waiting for ipfs daemon to start..."
+    sleep 2
+  done
+done
--- a/scripts/publish-checksums.sh
+++ b/scripts/publish-checksums.sh
@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -exo
+
+pushd dist
+
+# make sure we have a token set, api requests won't work otherwise
+if [ -z "${GITHUB_TOKEN}" ]; then
+  echo "\${GITHUB_TOKEN} not set, publish failed"
+  exit 1
+fi
+
+REQUIRED=(
+    "jq"
+    "curl"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+#see if the release already exists by tag
+RELEASE_RESPONSE=`
+  curl \
+    --fail \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}"
+`
+RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'`
+
+if [ "${RELEASE_ID}" = "null" ]; then
+  echo "creating release"
+
+  COND_CREATE_DISCUSSION=""
+  PRERELEASE=true
+  if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\","
+    PRERELEASE=false
+  fi
+
+  RELEASE_DATA="{
+    \"tag_name\": \"${CIRCLE_TAG}\",
+    \"target_commitish\": \"${CIRCLE_SHA1}\",
+    ${COND_CREATE_DISCUSSION}
+    \"name\": \"${CIRCLE_TAG}\",
+    \"body\": \"\",
+    \"prerelease\": ${PRERELEASE}
+  }"
+
+  # create it if it doesn't exist yet
+  RELEASE_RESPONSE=`
+    curl \
+        --fail \
+        --request POST \
+        --header "Authorization: token ${GITHUB_TOKEN}" \
+        --header "Content-Type: application/json" \
+        --data "${RELEASE_DATA}" \
+        "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases"
+  `
+else
+  echo "release already exists"
+fi
+
+RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1`
+echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}"
+
+for CHECKSUM_FILE in *.{cid,sha512}
+do
+  echo "Uploading ${CHECKSUM_FILE}..."
+  curl \
+    --fail \
+    --request POST \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    --header "Content-Type: application/octet-stream" \
+    --data-binary "@${CHECKSUM_FILE}" \
+    "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")"
+
+  echo "Uploaded ${CHECKSUM_FILE}"
+done
+
+popd