Merge pull request #9096 from filecoin-project/goreleaser

_ci_: Use goreleaser to build macos universal binaries (including M1 macs)
2022-08-01 11:36:49 -07:00 · 2022-08-01 11:36:49 -07:00 · b0b660b52d
commit b0b660b52d
parent a843c52e38 52aef0b926
6 changed files with 419 additions and 114 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -46,6 +46,35 @@ commands:
          steps:
            - run: sudo apt-get update
            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
      - run: git submodule sync
      - run: git submodule update --init
  download-params:
@ -77,6 +106,16 @@ commands:
          tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
          mv go-ipfs/ipfs /usr/local/bin/ipfs
          chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
  git_fetch_all_tags:
    steps:
      - run:
@ -335,9 +374,13 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
  build-macos:
    description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
    macos:
      xcode: "12.5.0"
    working_directory: ~/go/src/github.com/filecoin-project/lotus
@ -345,48 +388,28 @@ jobs:
      - prepare:
          linux: false
          darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
      - restore_cache:
          name: restore cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
      - persist_to_workspace:
          root: "."
          paths:
-            - darwin
+            - dist
      - save_cache:
          name: save cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@ -523,10 +546,6 @@ jobs:
        default: false
        description: publish linux binaries?
        type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
      appimage:
        default: false
        description: publish appimage binaries?
@ -546,11 +565,6 @@ jobs:
          steps:
            - run: ./scripts/build-arch-bundle.sh linux
            - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
      - when:
          condition: << parameters.appimage >>
          steps:
@ -1057,10 +1071,20 @@ workflows:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-lotus-soup
      - build-macos:
+          name: publish-macos
+          publish: true
          filters:
+            branches:
+              ignore:
+                - /.*/
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-appimage:
          filters:
            branches:
@ -1069,18 +1093,6 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          name: publish-linux
          linux: true
@ -1106,12 +1118,14 @@ workflows:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
          dockerfile: Dockerfile.lotus
          path: .
          repo: lotus-dev
          tag: '${CIRCLE_SHA1:0:8}'
          target: lotus-all-in-one
      - build-and-push-image:
+          name: build-and-push/lotus-test
          dockerfile: Dockerfile.lotus
          path: .
          repo: lotus-test
--- a/.circleci/template.yml
+++ b/.circleci/template.yml
@ -46,6 +46,35 @@ commands:
          steps:
            - run: sudo apt-get update
            - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
+      - when:
+          condition: <<parameters.darwin>>
+          steps:
+            - run:
+               name: Install Go
+               command: |
+                 curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
+                 sudo installer -pkg /tmp/go.pkg -target /
+            - run:
+                name: Export Go
+                command: |
+                 echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
+            - run: go version
+            - run:
+                name: Install pkg-config, goreleaser, and sha512sum
+                command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
+            - run:
+                name: Install Rust
+                command: |
+                  curl https://sh.rustup.rs -sSf | sh -s -- -y
+            - run:
+                name: Install hwloc
+                command: |
+                  mkdir ~/hwloc
+                  curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
+                  cd ~/hwloc
+                  tar -xvzpf hwloc-2.4.1.tar.gz
+                  cd hwloc-2.4.1
+                  ./configure && make && sudo make install
      - run: git submodule sync
      - run: git submodule update --init
  download-params:
@ -77,6 +106,16 @@ commands:
          tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
          mv go-ipfs/ipfs /usr/local/bin/ipfs
          chmod +x /usr/local/bin/ipfs
+  install_ipfs_macos:
+    steps:
+      - run: |
+          curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
+          tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
+          pushd kubo
+          sudo bash install.sh
+          popd
+          rm -rf kubo/
+          rm kubo_v0.14.0_darwin-amd64.tar.gz
  git_fetch_all_tags:
    steps:
      - run:
@ -335,9 +374,13 @@ jobs:
      - run:
          name: "trigger payment channel stress testplan on taas"
          command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
-
  build-macos:
    description: build darwin lotus binary
+    parameters:
+      publish:
+        default: false
+        description: publish github release and homebrew?
+        type: boolean
    macos:
      xcode: "12.5.0"
    working_directory: ~/go/src/github.com/filecoin-project/lotus
@ -345,48 +388,28 @@ jobs:
      - prepare:
          linux: false
          darwin: true
-      - run:
-          name: Install go
-          command: |
-            curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \
-            sudo installer -pkg go1.17.9.darwin-amd64.pkg -target /
-      - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
-      - run: go version
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run:
-          name: Install hwloc
-          command: |
-            mkdir ~/hwloc
-            curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
-            cd ~/hwloc
-            tar -xvzpf hwloc-2.4.1.tar.gz
-            cd hwloc-2.4.1
-            ./configure && make && sudo make install
+      - install_ipfs_macos
      - restore_cache:
          name: restore cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
-      - run:
-          command: make build
-          no_output_timeout: 30m
-      - run:
-          name: check tag and version output match
-          command: ./scripts/version-check.sh ./lotus
+      - when:
+          condition: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist
+            - run: ./scripts/generate-checksums.sh
+            - run: ./scripts/publish-checksums.sh
+      - when:
+          condition:
+            not: << parameters.publish >>
+          steps:
+            - run: goreleaser release --rm-dist --snapshot
+            - run: ./scripts/generate-checksums.sh
      - store_artifacts:
-          path: lotus
-      - store_artifacts:
-          path: lotus-miner
-      - store_artifacts:
-          path: lotus-worker
-      - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/
+          path: dist
      - persist_to_workspace:
          root: "."
          paths:
-            - darwin
+            - dist
      - save_cache:
          name: save cargo cache
          key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
@ -523,10 +546,6 @@ jobs:
        default: false
        description: publish linux binaries?
        type: boolean
-      darwin:
-        default: false
-        description: publish darwin binaries?
-        type: boolean
      appimage:
        default: false
        description: publish appimage binaries?
@ -546,11 +565,6 @@ jobs:
          steps:
            - run: ./scripts/build-arch-bundle.sh linux
            - run: ./scripts/publish-arch-release.sh linux
-      - when:
-          condition: << parameters.darwin>>
-          steps:
-            - run: ./scripts/build-arch-bundle.sh darwin
-            - run: ./scripts/publish-arch-release.sh darwin
      - when:
          condition: << parameters.appimage >>
          steps:
@ -817,10 +831,20 @@ workflows:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-lotus-soup
      - build-macos:
+          name: publish-macos
+          publish: true
          filters:
+            branches:
+              ignore:
+                - /.*/
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-macos:
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-appimage:
          filters:
            branches:
@ -829,18 +853,6 @@ workflows:
            tags:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish:
-          name: publish-macos
-          darwin: true
-          requires:
-            - build-macos
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - publish:
          name: publish-linux
          linux: true
@ -866,12 +878,14 @@ workflows:
              only:
                - /^v\d+\.\d+\.\d+(-rc\d+)?$/
      - build-and-push-image:
+          name: build-and-push/lotus-all-in-one
          dockerfile: Dockerfile.lotus
          path: .
          repo: lotus-dev
          tag: '${CIRCLE_SHA1:0:8}'
          target: lotus-all-in-one
      - build-and-push-image:
+          name: build-and-push/lotus-test
          dockerfile: Dockerfile.lotus
          path: .
          repo: lotus-test
--- a/.gitignore
+++ b/.gitignore
@ -51,3 +51,5 @@ scratchpad

 build/builtin-actors/v*
 build/builtin-actors/*.car
+
+dist/
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@ -0,0 +1,166 @@
+project_name: lotus
+before:
+  hooks:
+    - go mod tidy
+    - make deps
+
+universal_binaries:
+  - id: lotus
+    replace: true
+    name_template: lotus
+    ids:
+      - lotus_darwin_amd64
+      - lotus_darwin_arm64
+  - id: lotus-miner
+    replace: true
+    name_template: lotus-miner
+    ids:
+      - lotus-miner_darwin_amd64
+      - lotus-miner_darwin_arm64
+  - id: lotus-worker
+    replace: true
+    name_template: lotus-worker
+    ids:
+      - lotus-worker_darwin_amd64
+      - lotus-worker_darwin_arm64
+
+builds:
+  - id: lotus_darwin_amd64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_amd64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_amd64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus_darwin_arm64
+    main: ./cmd/lotus
+    binary: lotus
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-miner_darwin_arm64
+    main: ./cmd/lotus-miner
+    binary: lotus-miner
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+  - id: lotus-worker_darwin_arm64
+    main: ./cmd/lotus-worker
+    binary: lotus-worker
+    goos:
+      - darwin
+    goarch:
+      - arm64
+    env:
+      - CGO_ENABLED=1
+      - FFI_BUILD_FROM_SOURCE=1
+      - CPATH=/opt/homebrew/include
+      - LIBRARY_PATH=/opt/homebrew/lib
+    ldflags:
+      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+#  - id: linux
+#    main: ./cmd/lotus
+#    binary: lotus
+#    goos:
+#      - linux
+#    goarch:
+#      - amd64
+#    env:
+#      - CGO_ENABLED=1
+#    ldflags:
+#      - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}}
+
+archives:
+  - id: primary
+    format: tar.gz
+    wrap_in_directory: true
+    files:
+      # this is a dumb but required hack so it doesn't include the default files
+      # https://github.com/goreleaser/goreleaser/issues/602
+      - _n_o_n_e_*
+
+release:
+  github:
+    owner: filecoin-project
+    name: lotus
+  prerelease: auto
+  mode: append
+  name_template: "Release v{{.Version}}"
+
+
+brews:
+  - tap:
+      owner: filecoin-project
+      name: homebrew-lotus
+      branch: master
+    ids:
+      - primary
+    install: |
+      bin.install "lotus"
+      bin.install "lotus-miner"
+      bin.install "lotus-worker"
+    test: |
+      system "#{bin}/lotus --version"
+      system "#{bin}/lotus-miner --version"
+      system "#{bin}/lotus-worker --version"
+    folder: Formula
+    homepage: "https://filecoin.io"
+    description: "A homebrew cask for installing filecoin-project/lotus on MacOS"
+    license: MIT
+    dependencies:
+      - name: pkg-config
+      - name: jq
+      - name: bzr
+      - name: hwloc
+
+# produced manually so we can include cid checksums
+checksum:
+  disable: true
+
+snapshot:
+  name_template: "{{ .Tag }}"
--- a/scripts/generate-checksums.sh
+++ b/scripts/generate-checksums.sh
@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -exo
+
+REQUIRED=(
+    "sha512sum"
+    "ipfs"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+# start ipfs
+export IPFS_PATH=`mktemp -d`
+ipfs init
+ipfs daemon &
+PID="$!"
+trap "kill -9 ${PID}" EXIT
+
+# generate checksums
+for FILE in dist/*.tar.gz
+do
+  sha512sum "${FILE}" > "${FILE}.sha512"
+  until ipfs add -q "${FILE}" > "${FILE}.cid"
+  do
+    echo "Waiting for ipfs daemon to start..."
+    sleep 2
+  done
+done
--- a/scripts/publish-checksums.sh
+++ b/scripts/publish-checksums.sh
@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -exo
+
+pushd dist
+
+# make sure we have a token set, api requests won't work otherwise
+if [ -z "${GITHUB_TOKEN}" ]; then
+  echo "\${GITHUB_TOKEN} not set, publish failed"
+  exit 1
+fi
+
+REQUIRED=(
+    "jq"
+    "curl"
+)
+for REQUIRE in "${REQUIRED[@]}"
+do
+    command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed"
+done
+
+#see if the release already exists by tag
+RELEASE_RESPONSE=`
+  curl \
+    --fail \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}"
+`
+RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'`
+
+if [ "${RELEASE_ID}" = "null" ]; then
+  echo "creating release"
+
+  COND_CREATE_DISCUSSION=""
+  PRERELEASE=true
+  if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\","
+    PRERELEASE=false
+  fi
+
+  RELEASE_DATA="{
+    \"tag_name\": \"${CIRCLE_TAG}\",
+    \"target_commitish\": \"${CIRCLE_SHA1}\",
+    ${COND_CREATE_DISCUSSION}
+    \"name\": \"${CIRCLE_TAG}\",
+    \"body\": \"\",
+    \"prerelease\": ${PRERELEASE}
+  }"
+
+  # create it if it doesn't exist yet
+  RELEASE_RESPONSE=`
+    curl \
+        --fail \
+        --request POST \
+        --header "Authorization: token ${GITHUB_TOKEN}" \
+        --header "Content-Type: application/json" \
+        --data "${RELEASE_DATA}" \
+        "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases"
+  `
+else
+  echo "release already exists"
+fi
+
+RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1`
+echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}"
+
+for CHECKSUM_FILE in *.{cid,sha512}
+do
+  echo "Uploading ${CHECKSUM_FILE}..."
+  curl \
+    --fail \
+    --request POST \
+    --header "Authorization: token ${GITHUB_TOKEN}" \
+    --header "Content-Type: application/octet-stream" \
+    --data-binary "@${CHECKSUM_FILE}" \
+    "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")"
+
+  echo "Uploaded ${CHECKSUM_FILE}"
+done
+
+popd