diff --git a/.circleci/config.yml b/.circleci/config.yml index 9bcd6e8d8..a606ae7a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,9 @@ executors: docker: - image: circleci/golang:1.13 resource_class: 2xlarge + ubuntu: + docker: + - image: ubuntu:19.10 commands: install-deps: @@ -24,6 +27,8 @@ commands: description: is a darwin build environment? type: boolean steps: + - checkout + - git_fetch_all_tags - checkout - when: condition: << parameters.linux >> @@ -46,7 +51,27 @@ commands: key: 'v20-1k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - + install_ipfs: + steps: + - run: | + apt update + apt install -y wget + wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz + wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512 + if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ] + then + echo "ipfs failed checksum check" + exit 1 + fi + tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz + mv go-ipfs/ipfs /usr/local/bin/ipfs + chmod +x /usr/local/bin/ipfs + git_fetch_all_tags: + steps: + - run: + name: fetch all tags + command: | + git fetch --all jobs: mod-tidy-check: @@ -74,6 +99,13 @@ jobs: path: lotus - store_artifacts: path: lotus-storage-miner + - store_artifacts: + path: lotus-seal-worker + - run: mkdir linux && mv lotus lotus-storage-miner lotus-seal-worker linux/ + - persist_to_workspace: + root: "." + paths: + - linux test: &test description: | @@ -153,7 +185,7 @@ jobs: - "~/go/src/github.com" - "~/go/src/golang.org" - test-short: + test-short: <<: *test build-macos: @@ -196,6 +228,13 @@ jobs: path: lotus - store_artifacts: path: lotus-storage-miner + - store_artifacts: + path: lotus-seal-worker + - run: mkdir darwin && mv lotus lotus-storage-miner lotus-seal-worker darwin/ + - persist_to_workspace: + root: "." + paths: + - darwin - save_cache: name: save cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} @@ -246,6 +285,26 @@ jobs: lint-all: <<: *lint + publish: + description: publish binary artifacts + executor: ubuntu + steps: + - run: + name: Install git jq curl + command: apt update && apt install -y git jq curl + - checkout + - git_fetch_all_tags + - checkout + - install_ipfs + - attach_workspace: + at: "." + - run: + name: Create bundles + command: ./scripts/build-bundle.sh + - run: + name: Publish release + command: ./scripts/publish-release.sh + workflows: version: 2.1 @@ -255,8 +314,38 @@ workflows: args: "--new-from-rev origin/master" - test: codecov-upload: true + - mod-tidy-check - test-short: go-test-flags: "--timeout 10m --short" - - mod-tidy-check - - build-all - - build-macos + filters: + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-all: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-macos: + requires: + - test-short + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - publish: + requires: + - build-all + - build-macos + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ diff --git a/.gitignore b/.gitignore index 9989ac40b..529f938c3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ build/paramfetch.sh /blocks.svg /chainwatch /chainwatch.db +/bundle +/darwin +/linux *-fuzz.zip /chain/types/work_msg/ diff --git a/scripts/build-bundle.sh b/scripts/build-bundle.sh new file mode 100755 index 000000000..16cab49d1 --- /dev/null +++ b/scripts/build-bundle.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -ex + +ARCHS=( + "darwin" + "linux" +) + +REQUIRED=( + "ipfs" + "sha512sum" +) +for REQUIRE in "${REQUIRED[@]}" +do + command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" +done + +mkdir bundle +pushd bundle + +BINARIES=( + "lotus" + "lotus-storage-miner" + "lotus-seal-worker" +) + +export IPFS_PATH=`mktemp -d` +ipfs init +ipfs daemon & +PID="$!" +trap "kill -9 ${PID}" EXIT +sleep 30 + +for ARCH in "${ARCHS[@]}" +do + mkdir -p "${ARCH}/lotus" + pushd "${ARCH}" + for BINARY in "${BINARIES[@]}" + do + cp "../../${ARCH}/${BINARY}" "lotus/" + chmod +x "lotus/${BINARY}" + done + + tar -zcvf "../lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz" lotus + popd + rm -rf "${ARCH}" + + sha512sum "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz" | cut -d" " -f1 > "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz.sha512" + + ipfs add "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz" | cut -d" " -f2 > "lotus_${CIRCLE_TAG}_${ARCH}-amd64.tar.gz.cid" +done +popd diff --git a/scripts/publish-release.sh b/scripts/publish-release.sh new file mode 100755 index 000000000..103de1f0f --- /dev/null +++ b/scripts/publish-release.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -e + +pushd bundle + +# make sure we have a token set, api requests won't work otherwise +if [ -z "${GITHUB_TOKEN}" ]; then + echo "\${GITHUB_TOKEN} not set, publish failed" + exit 1 +fi + +REQUIRED=( + "jq" + "curl" +) +for REQUIRE in "${REQUIRED[@]}" +do + command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" +done + +#see if the release already exists by tag +RELEASE_RESPONSE=` + curl \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}" +` +RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` + +if [ "${RELEASE_ID}" = "null" ]; then + echo "creating release" + + RELEASE_DATA="{ + \"tag_name\": \"${CIRCLE_TAG}\", + \"target_commitish\": \"${CIRCLE_SHA1}\", + \"name\": \"${CIRCLE_TAG}\", + \"body\": \"\", + \"prerelease\": false + }" + + # create it if it doesn't exist yet + RELEASE_RESPONSE=` + curl \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "${RELEASE_DATA}" \ + "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases" + ` +else + echo "release already exists" +fi + +RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1` + +bundles=( + "lotus_${CIRCLE_TAG}_linux-amd64.tar.gz" + "lotus_${CIRCLE_TAG}_linux-amd64.tar.gz.cid" + "lotus_${CIRCLE_TAG}_linux-amd64.tar.gz.sha512" + "lotus_${CIRCLE_TAG}_darwin-amd64.tar.gz" + "lotus_${CIRCLE_TAG}_darwin-amd64.tar.gz.cid" + "lotus_${CIRCLE_TAG}_darwin-amd64.tar.gz.sha512" +) +for RELEASE_FILE in "${bundles[@]}" +do + echo "Uploading release bundle: ${RELEASE_FILE}" + curl \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/octet-stream" \ + --data-binary "@${RELEASE_FILE}" \ + "$RELEASE_UPLOAD_URL?name=$(basename "${RELEASE_FILE}")" + + echo "Release bundle uploaded: ${RELEASE_FILE}" +done + +popd + +miscellaneous=( + "README.md" + "LICENSE-MIT" + "LICENSE-APACHE" +) +for MISC in "${miscellaneous[@]}" +do + echo "Uploading release bundle: ${MISC}" + curl \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/octet-stream" \ + --data-binary "@${MISC}" \ + "$RELEASE_UPLOAD_URL?name=$(basename "${MISC}")" + + echo "Release bundle uploaded: ${MISC}" +done