From 7f1b4dc438b58b4a0af317b6ec4f8a00168e49ef Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Thu, 28 Jul 2022 19:23:03 -0700 Subject: [PATCH] Use goreleaser to build universal darwin binaries This is a small refactor of our workflow to test out goreleaser, a yaml based tool for building, packaging, and releasing go binaries on multiple platforms. It supports building binaries for to most of the platforms we care about, including linux and macos, and also supports publishing those binaries automatically as releases in Github, homebrew, snap, and even apt / deb. If this trial goes well, I think we should eventually replace the entire release workflow with goreleaser. For now, this test is more tightly scoped to only automated the MacOS release process, since that is the one we have the most issues with. This PRi / commit: - Builds darwin-amd64 and darwin-arm64 binaries of lotus, lotus-miner, and lotus-worker - Packages them into a universal darwin binary - Publishes those to a release in Github based on the current tag - Uses the binaries in the release to auto-publish and updated homebrew configuration to filecoin-project/homebrew-lotus - Does a `dry-run` build to produce a snapshot on release branches with no tag - Manually generate and upload checksums after goreleaser --- .circleci/config.yml | 126 ++++++++++++++------------ .circleci/template.yml | 126 ++++++++++++++------------ .gitignore | 2 + .goreleaser.yaml | 166 ++++++++++++++++++++++++++++++++++ scripts/generate-checksums.sh | 29 ++++++ scripts/publish-checksums.sh | 80 ++++++++++++++++ 6 files changed, 415 insertions(+), 114 deletions(-) create mode 100644 .goreleaser.yaml create mode 100755 scripts/generate-checksums.sh create mode 100755 scripts/publish-checksums.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index e9c43ab4f..f4b05b730 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,35 @@ commands: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - when: + condition: <> + steps: + - run: + name: Install Go + command: | + curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \ + sudo installer -pkg /tmp/go.pkg -target / + - run: + name: Export Go + command: | + echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV + - run: go version + - run: + name: Install pkg-config, goreleaser, and sha512sum + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils + - run: + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + - run: + name: Install hwloc + command: | + mkdir ~/hwloc + curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz + cd ~/hwloc + tar -xvzpf hwloc-2.4.1.tar.gz + cd hwloc-2.4.1 + ./configure && make && sudo make install - run: git submodule sync - run: git submodule update --init download-params: @@ -77,6 +106,16 @@ commands: tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz mv go-ipfs/ipfs /usr/local/bin/ipfs chmod +x /usr/local/bin/ipfs + install_ipfs_macos: + steps: + - run: | + curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz + tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz + pushd kubo + sudo bash install.sh + popd + rm -rf kubo/ + rm kubo_v0.14.0_darwin-amd64.tar.gz git_fetch_all_tags: steps: - run: @@ -335,9 +374,13 @@ jobs: - run: name: "trigger payment channel stress testplan on taas" command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - build-macos: description: build darwin lotus binary + parameters: + publish: + default: false + description: publish github release and homebrew? + type: boolean macos: xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus @@ -345,48 +388,28 @@ jobs: - prepare: linux: false darwin: true - - run: - name: Install go - command: | - curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \ - sudo installer -pkg go1.17.9.darwin-amd64.pkg -target / - - run: - name: Install pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config - - run: go version - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: - name: Install hwloc - command: | - mkdir ~/hwloc - curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz - cd ~/hwloc - tar -xvzpf hwloc-2.4.1.tar.gz - cd hwloc-2.4.1 - ./configure && make && sudo make install + - install_ipfs_macos - restore_cache: name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - - run: - command: make build - no_output_timeout: 30m - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus + - when: + condition: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist + - run: ./scripts/generate-checksums.sh + - run: ./scripts/publish-checksums.sh + - when: + condition: + not: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist --snapshot + - run: ./scripts/generate-checksums.sh - store_artifacts: - path: lotus - - store_artifacts: - path: lotus-miner - - store_artifacts: - path: lotus-worker - - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ + path: dist - persist_to_workspace: root: "." paths: - - darwin + - dist - save_cache: name: save cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} @@ -523,10 +546,6 @@ jobs: default: false description: publish linux binaries? type: boolean - darwin: - default: false - description: publish darwin binaries? - type: boolean appimage: default: false description: publish appimage binaries? @@ -546,11 +565,6 @@ jobs: steps: - run: ./scripts/build-arch-bundle.sh linux - run: ./scripts/publish-arch-release.sh linux - - when: - condition: << parameters.darwin>> - steps: - - run: ./scripts/build-arch-bundle.sh darwin - - run: ./scripts/publish-arch-release.sh darwin - when: condition: << parameters.appimage >> steps: @@ -1057,10 +1071,20 @@ workflows: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - build-macos: + name: publish-macos + publish: true filters: + branches: + ignore: + - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-macos: + filters: + branches: + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - build-appimage: filters: branches: @@ -1069,18 +1093,6 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish: - name: publish-macos - darwin: true - requires: - - build-macos - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish: name: publish-linux linux: true diff --git a/.circleci/template.yml b/.circleci/template.yml index a7f6590e7..820b2fa92 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -46,6 +46,35 @@ commands: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - when: + condition: <> + steps: + - run: + name: Install Go + command: | + curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \ + sudo installer -pkg /tmp/go.pkg -target / + - run: + name: Export Go + command: | + echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV + - run: go version + - run: + name: Install pkg-config, goreleaser, and sha512sum + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils + - run: + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + - run: + name: Install hwloc + command: | + mkdir ~/hwloc + curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz + cd ~/hwloc + tar -xvzpf hwloc-2.4.1.tar.gz + cd hwloc-2.4.1 + ./configure && make && sudo make install - run: git submodule sync - run: git submodule update --init download-params: @@ -77,6 +106,16 @@ commands: tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz mv go-ipfs/ipfs /usr/local/bin/ipfs chmod +x /usr/local/bin/ipfs + install_ipfs_macos: + steps: + - run: | + curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz + tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz + pushd kubo + sudo bash install.sh + popd + rm -rf kubo/ + rm kubo_v0.14.0_darwin-amd64.tar.gz git_fetch_all_tags: steps: - run: @@ -335,9 +374,13 @@ jobs: - run: name: "trigger payment channel stress testplan on taas" command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - build-macos: description: build darwin lotus binary + parameters: + publish: + default: false + description: publish github release and homebrew? + type: boolean macos: xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus @@ -345,48 +388,28 @@ jobs: - prepare: linux: false darwin: true - - run: - name: Install go - command: | - curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \ - sudo installer -pkg go1.17.9.darwin-amd64.pkg -target / - - run: - name: Install pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config - - run: go version - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: - name: Install hwloc - command: | - mkdir ~/hwloc - curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz - cd ~/hwloc - tar -xvzpf hwloc-2.4.1.tar.gz - cd hwloc-2.4.1 - ./configure && make && sudo make install + - install_ipfs_macos - restore_cache: name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - - run: - command: make build - no_output_timeout: 30m - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus + - when: + condition: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist + - run: ./scripts/generate-checksums.sh + - run: ./scripts/publish-checksums.sh + - when: + condition: + not: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist --snapshot + - run: ./scripts/generate-checksums.sh - store_artifacts: - path: lotus - - store_artifacts: - path: lotus-miner - - store_artifacts: - path: lotus-worker - - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ + path: dist - persist_to_workspace: root: "." paths: - - darwin + - dist - save_cache: name: save cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} @@ -523,10 +546,6 @@ jobs: default: false description: publish linux binaries? type: boolean - darwin: - default: false - description: publish darwin binaries? - type: boolean appimage: default: false description: publish appimage binaries? @@ -546,11 +565,6 @@ jobs: steps: - run: ./scripts/build-arch-bundle.sh linux - run: ./scripts/publish-arch-release.sh linux - - when: - condition: << parameters.darwin>> - steps: - - run: ./scripts/build-arch-bundle.sh darwin - - run: ./scripts/publish-arch-release.sh darwin - when: condition: << parameters.appimage >> steps: @@ -817,10 +831,20 @@ workflows: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - build-macos: + name: publish-macos + publish: true filters: + branches: + ignore: + - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-macos: + filters: + branches: + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - build-appimage: filters: branches: @@ -829,18 +853,6 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish: - name: publish-macos - darwin: true - requires: - - build-macos - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish: name: publish-linux linux: true diff --git a/.gitignore b/.gitignore index 1db5e2858..b111195cf 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ scratchpad build/builtin-actors/v* build/builtin-actors/*.car + +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 000000000..218f078fb --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,166 @@ +project_name: lotus +before: + hooks: + - go mod tidy + - make deps + +universal_binaries: + - id: lotus + replace: true + name_template: lotus + ids: + - lotus_darwin_amd64 + - lotus_darwin_arm64 + - id: lotus-miner + replace: true + name_template: lotus-miner + ids: + - lotus-miner_darwin_amd64 + - lotus-miner_darwin_arm64 + - id: lotus-worker + replace: true + name_template: lotus-worker + ids: + - lotus-worker_darwin_amd64 + - lotus-worker_darwin_arm64 + +builds: + - id: lotus_darwin_amd64 + main: ./cmd/lotus + binary: lotus + goos: + - darwin + goarch: + - amd64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-miner_darwin_amd64 + main: ./cmd/lotus-miner + binary: lotus-miner + goos: + - darwin + goarch: + - amd64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-worker_darwin_amd64 + main: ./cmd/lotus-worker + binary: lotus-worker + goos: + - darwin + goarch: + - amd64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus_darwin_arm64 + main: ./cmd/lotus + binary: lotus + goos: + - darwin + goarch: + - arm64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + - CPATH=/opt/homebrew/include + - LIBRARY_PATH=/opt/homebrew/lib + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-miner_darwin_arm64 + main: ./cmd/lotus-miner + binary: lotus-miner + goos: + - darwin + goarch: + - arm64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + - CPATH=/opt/homebrew/include + - LIBRARY_PATH=/opt/homebrew/lib + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-worker_darwin_arm64 + main: ./cmd/lotus-worker + binary: lotus-worker + goos: + - darwin + goarch: + - arm64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + - CPATH=/opt/homebrew/include + - LIBRARY_PATH=/opt/homebrew/lib + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} +# - id: linux +# main: ./cmd/lotus +# binary: lotus +# goos: +# - linux +# goarch: +# - amd64 +# env: +# - CGO_ENABLED=1 +# ldflags: +# - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + +archives: + - id: primary + format: tar.gz + wrap_in_directory: true + files: + # this is a dumb but required hack so it doesn't include the default files + # https://github.com/goreleaser/goreleaser/issues/602 + - _n_o_n_e_* + +release: + github: + owner: ianconsolata + name: lotus + prerelease: auto + mode: append + name_template: "Release v{{.Version}}" + + +brews: + - tap: + owner: ianconsolata + name: homebrew-lotus + branch: master + ids: + - primary + install: | + bin.install "lotus" + bin.install "lotus-miner" + bin.install "lotus-worker" + test: | + system "#{bin}/lotus --version" + system "#{bin}/lotus-miner --version" + system "#{bin}/lotus-worker --version" + folder: Formula + homepage: "https://filecoin.io" + description: "A homebrew cask for installing filecoin-project/lotus on MacOS" + license: MIT + dependencies: + - name: pkg-config + - name: jq + - name: bzr + - name: hwloc + +# produced manually so we can include cid checksums +checksum: + disable: true + +snapshot: + name_template: "{{ .Tag }}" diff --git a/scripts/generate-checksums.sh b/scripts/generate-checksums.sh new file mode 100755 index 000000000..ae0920138 --- /dev/null +++ b/scripts/generate-checksums.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -exo + +REQUIRED=( + "sha512sum" + "ipfs" +) +for REQUIRE in "${REQUIRED[@]}" +do + command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" +done + +# start ipfs +export IPFS_PATH=`mktemp -d` +ipfs init +ipfs daemon & +PID="$!" +trap "kill -9 ${PID}" EXIT + +# generate checksums +for FILE in dist/*.tar.gz +do + sha512sum "${FILE}" > "${FILE}.sha512" + until ipfs add -q "${FILE}" > "${FILE}.cid" + do + echo "Waiting for ipfs daemon to start..." + sleep 2 + done +done diff --git a/scripts/publish-checksums.sh b/scripts/publish-checksums.sh new file mode 100755 index 000000000..346ca9b4c --- /dev/null +++ b/scripts/publish-checksums.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -exo + +pushd dist + +# make sure we have a token set, api requests won't work otherwise +if [ -z "${GITHUB_TOKEN}" ]; then + echo "\${GITHUB_TOKEN} not set, publish failed" + exit 1 +fi + +REQUIRED=( + "jq" + "curl" +) +for REQUIRE in "${REQUIRED[@]}" +do + command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" +done + +#see if the release already exists by tag +RELEASE_RESPONSE=` + curl \ + --fail \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}" +` +RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` + +if [ "${RELEASE_ID}" = "null" ]; then + echo "creating release" + + COND_CREATE_DISCUSSION="" + PRERELEASE=true + if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\"," + PRERELEASE=false + fi + + RELEASE_DATA="{ + \"tag_name\": \"${CIRCLE_TAG}\", + \"target_commitish\": \"${CIRCLE_SHA1}\", + ${COND_CREATE_DISCUSSION} + \"name\": \"${CIRCLE_TAG}\", + \"body\": \"\", + \"prerelease\": ${PRERELEASE} + }" + + # create it if it doesn't exist yet + RELEASE_RESPONSE=` + curl \ + --fail \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "${RELEASE_DATA}" \ + "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases" + ` +else + echo "release already exists" +fi + +RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1` +echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}" + +for CHECKSUM_FILE in *.{cid,sha512} +do + echo "Uploading ${CHECKSUM_FILE}..." + curl \ + --fail \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/octet-stream" \ + --data-binary "@${CHECKSUM_FILE}" \ + "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")" + + echo "Uploaded ${CHECKSUM_FILE}" +done + +popd