version: 2.1 orbs: aws-cli: circleci/aws-cli@1.3.2 docker: circleci/docker@2.1.4 executors: golang: docker: # Must match GO_VERSION_MIN in project root - image: cimg/go:1.19.12 resource_class: medium+ golang-2xl: docker: # Must match GO_VERSION_MIN in project root - image: cimg/go:1.19.12 resource_class: 2xlarge ubuntu: docker: - image: ubuntu:20.04 commands: build-platform-specific: parameters: linux: default: true description: is a linux build environment? type: boolean darwin: default: false description: is a darwin build environment? type: boolean darwin-architecture: default: "amd64" description: which darwin architecture is being used? type: string steps: - checkout - git_fetch_all_tags - run: git submodule sync - run: git submodule update --init - when: condition: <<parameters.linux>> steps: - install-ubuntu-deps - check-go-version - when: condition: <<parameters.darwin>> steps: - run: name: Install Go command: | curl https://dl.google.com/go/go`cat GO_VERSION_MIN`.darwin-<<parameters.darwin-architecture>>.pkg -o /tmp/go.pkg && \ sudo installer -pkg /tmp/go.pkg -target / - run: name: Export Go command: | echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV - run: go version - run: name: Install dependencies with Homebrew command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config coreutils jq hwloc - run: name: Install Rust command: | curl https://sh.rustup.rs -sSf | sh -s -- -y - run: make deps download-params: steps: - restore_cache: name: Restore parameters cache keys: - 'v26-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache key: 'v26-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ install_ipfs: steps: - run: | curl -O https://dist.ipfs.tech/kubo/v0.16.0/kubo_v0.16.0_linux-amd64.tar.gz tar -xvzf kubo_v0.16.0_linux-amd64.tar.gz pushd kubo sudo bash install.sh popd rm -rf kubo rm kubo_v0.16.0_linux-amd64.tar.gz git_fetch_all_tags: steps: - run: name: fetch all tags command: | git fetch --all install-ubuntu-deps: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev check-go-version: steps: - run: | v=`go version | { read _ _ v _; echo ${v#go}; }` if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then echo "GO_VERSION_MIN file does not match the go version being used." echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." exit 1 fi jobs: build: executor: golang working_directory: ~/lotus steps: - checkout - git_fetch_all_tags - run: git submodule sync - run: git submodule update --init - install-ubuntu-deps - check-go-version - run: make deps lotus - persist_to_workspace: root: ~/ paths: - "lotus" mod-tidy-check: executor: golang working_directory: ~/lotus steps: - install-ubuntu-deps - attach_workspace: at: ~/ - run: go mod tidy -v - run: name: Check git diff command: | git --no-pager diff go.mod go.sum git --no-pager diff --quiet go.mod go.sum test: description: | Run tests with gotestsum. working_directory: ~/lotus parameters: &test-params executor: type: executor default: golang go-test-flags: type: string default: "-timeout 20m" description: Flags passed to go test. target: type: string default: "./..." description: Import paths of packages to be tested. proofs-log-test: type: string default: "0" get-params: type: boolean default: false suite: type: string default: unit description: Test suite name to report to CircleCI. executor: << parameters.executor >> steps: - install-ubuntu-deps - attach_workspace: at: ~/ - when: condition: << parameters.get-params >> steps: - download-params - run: name: go test environment: TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> SKIP_CONFORMANCE: "1" LOTUS_SRC_DIR: /home/circleci/project command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ --packages="<< parameters.target >>" \ -- << parameters.go-test-flags >> no_output_timeout: 30m - store_test_results: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/<< parameters.suite >>.json test-conformance: working_directory: ~/lotus description: | Run tests using a corpus of interoperable test vectors for Filecoin implementations to test their correctness and compliance with the Filecoin specifications. parameters: <<: *test-params vectors-branch: type: string default: "" description: | Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. executor: << parameters.executor >> steps: - install-ubuntu-deps - attach_workspace: at: ~/ - download-params - when: condition: not: equal: [ "", << parameters.vectors-branch >> ] steps: - run: name: checkout vectors branch command: | cd extern/test-vectors git fetch git checkout origin/<< parameters.vectors-branch >> - run: name: install statediff globally command: | ## statediff is optional; we succeed even if compilation fails. mkdir -p /tmp/statediff git clone https://github.com/filecoin-project/statediff.git /tmp/statediff cd /tmp/statediff go install ./cmd/statediff || exit 0 - run: name: go test environment: SKIP_CONFORMANCE: "0" command: | mkdir -p /tmp/test-reports mkdir -p /tmp/test-artifacts gotestsum \ --format pkgname-and-test-fails \ --junitfile /tmp/test-reports/junit.xml \ -- \ -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html no_output_timeout: 30m - store_test_results: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html build-linux-amd64: executor: golang steps: - build-platform-specific - run: make lotus lotus-miner lotus-worker - run: name: check tag and version output match command: ./scripts/version-check.sh ./lotus - run: | mkdir -p /tmp/workspace/linux_amd64_v1 && \ mv lotus lotus-miner lotus-worker /tmp/workspace/linux_amd64_v1/ - persist_to_workspace: root: /tmp/workspace paths: - linux_amd64_v1 build-darwin-amd64: description: build darwin lotus binary working_directory: ~/go/src/github.com/filecoin-project/lotus macos: xcode: "13.4.1" steps: - build-platform-specific: linux: false darwin: true darwin-architecture: amd64 - run: make lotus lotus-miner lotus-worker - run: otool -hv lotus - run: name: check tag and version output match command: ./scripts/version-check.sh ./lotus - run: | mkdir -p /tmp/workspace/darwin_amd64_v1 && \ mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_amd64_v1/ - persist_to_workspace: root: /tmp/workspace paths: - darwin_amd64_v1 build-darwin-arm64: description: self-hosted m1 runner working_directory: ~/go/src/github.com/filecoin-project/lotus machine: true resource_class: filecoin-project/self-hosted-m1 steps: - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - build-platform-specific: linux: false darwin: true darwin-architecture: arm64 - run: | export CPATH=$(brew --prefix)/include && export LIBRARY_PATH=$(brew --prefix)/lib && make lotus lotus-miner lotus-worker - run: otool -hv lotus - run: name: check tag and version output match command: ./scripts/version-check.sh ./lotus - run: | mkdir -p /tmp/workspace/darwin_arm64 && \ mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_arm64/ - persist_to_workspace: root: /tmp/workspace paths: - darwin_arm64 - run: command: make clean when: always - run: name: cleanup homebrew command: HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall pkg-config coreutils jq hwloc when: always release: executor: golang parameters: dry-run: default: false description: should this release actually publish it's artifacts? type: boolean steps: - checkout - run: | echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list sudo apt update sudo apt install goreleaser-pro - install_ipfs - attach_workspace: at: /tmp/workspace - when: condition: << parameters.dry-run >> steps: - run: goreleaser release --rm-dist --snapshot --debug - run: ./scripts/generate-checksums.sh - when: condition: not: << parameters.dry-run >> steps: - run: goreleaser release --rm-dist --debug - run: ./scripts/generate-checksums.sh - run: ./scripts/publish-checksums.sh gofmt: executor: golang working_directory: ~/lotus steps: - run: command: "! go fmt ./... 2>&1 | read" gen-check: executor: golang working_directory: ~/lotus steps: - install-ubuntu-deps - attach_workspace: at: ~/ - run: go install golang.org/x/tools/cmd/goimports - run: go install github.com/hannahhoward/cbor-gen-for - run: make gen - run: git --no-pager diff && git --no-pager diff --quiet - run: make docsgen-cli - run: git --no-pager diff && git --no-pager diff --quiet docs-check: executor: golang working_directory: ~/lotus steps: - install-ubuntu-deps - attach_workspace: at: ~/ - run: go install golang.org/x/tools/cmd/goimports - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - run: make docsgen - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet lint-all: description: | Run golangci-lint. working_directory: ~/lotus parameters: executor: type: executor default: golang args: type: string default: '' description: | Arguments to pass to golangci-lint executor: << parameters.executor >> steps: - install-ubuntu-deps - attach_workspace: at: ~/ - run: name: Lint command: | golangci-lint run -v --timeout 10m \ --concurrency 4 << parameters.args >> build-docker: description: > Publish to Dockerhub executor: docker/docker parameters: image: type: string default: lotus description: > Passed to the docker build process to determine which image in the Dockerfile should be built. Expected values are `lotus`, `lotus-all-in-one` network: type: string default: "mainnet" description: > Passed to the docker build process using GOFLAGS+=-tags=<<network>>. Expected values are `debug`, `2k`, `calibnet`, `butterflynet`, `interopnet`. channel: type: string default: "" description: > The release channel to use for this image. push: type: boolean default: false description: > When true, pushes the image to Dockerhub steps: - setup_remote_docker - checkout - git_fetch_all_tags - run: git submodule sync - run: git submodule update --init - docker/check: docker-username: DOCKERHUB_USERNAME docker-password: DOCKERHUB_PASSWORD - when: condition: equal: [ mainnet, <<parameters.network>> ] steps: - when: condition: <<parameters.push>> steps: - docker/build: image: filecoin/<<parameters.image>> extra_build_args: --target <<parameters.image>> tag: <<parameters.channel>> - run: name: Docker push command: | docker push filecoin/<<parameters.image>>:<<parameters.channel>> if [["[[ ! -z $CIRCLE_SHA ]]"]]; then docker image tag filecoin/<<parameters.image>>:<<parameters.channel>> filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}" docker push filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}" fi if [["[[ ! -z $CIRCLE_TAG ]]"]]; then docker image tag filecoin/<<parameters.image>>:<<parameters.channel>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}" docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}" fi - unless: condition: <<parameters.push>> steps: - docker/build: image: filecoin/<<parameters.image>> extra_build_args: --target <<parameters.image>> - when: condition: not: equal: [ mainnet, <<parameters.network>> ] steps: - when: condition: <<parameters.push>> steps: - docker/build: image: filecoin/<<parameters.image>> extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>> tag: <<parameters.channel>>-<<parameters.network>> - run: name: Docker push command: | docker push filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> if [["[[ ! -z $CIRCLE_SHA ]]"]]; then docker image tag filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}"-<<parameters.network>> docker push filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}"-<<parameters.network>> fi if [["[[ ! -z $CIRCLE_TAG ]]"]]; then docker image tag filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>> docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>> fi - unless: condition: <<parameters.push>> steps: - docker/build: image: filecoin/<<parameters.image>> extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>> workflows: ci: jobs: - build - lint-all: requires: - build - mod-tidy-check: requires: - build - gofmt: requires: - build - gen-check: requires: - build - docs-check: requires: - build [[- range $file := .ItestFiles -]] [[ with $name := $file | stripSuffix ]] - test: name: test-itest-[[ $name ]] requires: - build suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] executor: golang-2xl [[- end]] [[- if or (eq $name "wdpost") (eq $name "sector_pledge")]] get-params: true [[end]] [[- end ]][[- end]] [[- range $suite, $pkgs := .UnitSuites]] - test: name: test-[[ $suite ]] requires: - build suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" [[if eq $suite "unit-cli"]]get-params: true[[end]] [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]] [[- end]] - test: go-test-flags: "-run=TestMulticoreSDR" requires: - build suite: multicore-sdr-check target: "./storage/sealer/ffiwrapper" proofs-log-test: "1" - test-conformance: requires: - build suite: conformance target: "./conformance" release: jobs: - build-linux-amd64: name: "Build ( linux / amd64 )" filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-darwin-amd64: name: "Build ( darwin / amd64 )" filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-darwin-arm64: name: "Build ( darwin / arm64 )" filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - /^ci\/.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - release: name: "Release" requires: - "Build ( darwin / amd64 )" - "Build ( linux / amd64 )" - "Build ( darwin / arm64 )" filters: branches: ignore: - /^.*$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - release: name: "Release (dry-run)" dry-run: true requires: - "Build ( darwin / amd64 )" - "Build ( linux / amd64 )" - "Build ( darwin / arm64 )" filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - /^ci\/.*$/ [[- range .Networks]] - build-docker: name: "Docker push (lotus-all-in-one / stable / [[.]])" image: lotus-all-in-one channel: stable network: [[.]] push: true filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+$/ - build-docker: name: "Docker push (lotus-all-in-one / candidate / [[.]])" image: lotus-all-in-one channel: candidate network: [[.]] push: true filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+-rc\d+$/ - build-docker: name: "Docker push (lotus-all-in-one / edge / [[.]])" image: lotus-all-in-one channel: master network: [[.]] push: true filters: branches: only: - master - build-docker: name: "Docker build (lotus-all-in-one / [[.]])" image: lotus-all-in-one network: [[.]] push: false filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ [[- end]] - build-docker: name: "Docker push (lotus / stable / mainnet)" image: lotus channel: stable network: mainnet push: true filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+$/ - build-docker: name: "Docker push (lotus / candidate / mainnet)" image: lotus channel: candidate network: mainnet push: true filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+-rc\d+$/ - build-docker: name: "Docker push (lotus / master / mainnet)" image: lotus channel: master network: mainnet push: true filters: branches: only: - master - build-docker: name: "Docker build (lotus / mainnet)" image: lotus network: mainnet push: false filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ nightly: triggers: - schedule: cron: "0 0 * * *" filters: branches: only: - master jobs: [[- range .Networks]] - build-docker: name: "Docker (lotus-all-in-one / nightly / [[.]])" image: lotus-all-in-one channel: nightly network: [[.]] push: true [[- end]]