version: 2.1 orbs: go: gotest/tools@0.0.13 aws-cli: circleci/aws-cli@1.3.2 packer: salaxander/packer@0.0.3 executors: golang: docker: - image: circleci/golang:1.16.4 resource_class: 2xlarge ubuntu: docker: - image: ubuntu:20.04 commands: install-deps: steps: - go/install-ssh - go/install: {package: git} prepare: parameters: linux: default: true description: is a linux build environment? type: boolean darwin: default: false description: is a darwin build environment? type: boolean steps: - checkout - git_fetch_all_tags - checkout - when: condition: << parameters.linux >> steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev - run: git submodule sync - run: git submodule update --init download-params: steps: - restore_cache: name: Restore parameters cache keys: - 'v26-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache key: 'v26-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ install_ipfs: steps: - run: | apt update apt install -y wget wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512 if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ] then echo "ipfs failed checksum check" exit 1 fi tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz mv go-ipfs/ipfs /usr/local/bin/ipfs chmod +x /usr/local/bin/ipfs git_fetch_all_tags: steps: - run: name: fetch all tags command: | git fetch --all jobs: mod-tidy-check: executor: golang steps: - install-deps - prepare - go/mod-tidy-check build-all: executor: golang steps: - install-deps - prepare - run: sudo apt-get update - run: sudo apt-get install npm - run: command: make buildall - run: name: check tag and version output match command: ./scripts/version-check.sh ./lotus - store_artifacts: path: lotus - store_artifacts: path: lotus-miner - store_artifacts: path: lotus-worker - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/ - persist_to_workspace: root: "." paths: - linux build-debug: executor: golang steps: - install-deps - prepare - run: command: make debug test: description: | Run tests with gotestsum. parameters: &test-params executor: type: executor default: golang go-test-flags: type: string default: "-timeout 30m" description: Flags passed to go test. target: type: string default: "./..." description: Import paths of packages to be tested. proofs-log-test: type: string default: "0" suite: type: string default: unit description: Test suite name to report to CircleCI. gotestsum-format: type: string default: standard-verbose description: gotestsum format. https://github.com/gotestyourself/gotestsum#format coverage: type: string default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/... description: Coverage flag. Set to the empty string to disable. codecov-upload: type: boolean default: true description: | Upload coverage report to https://codecov.io/. Requires the codecov API token to be set as an environment variable for private projects. executor: << parameters.executor >> steps: - install-deps - prepare - run: command: make deps lotus no_output_timeout: 30m - download-params - go/install-gotestsum: gobin: $HOME/.local/bin version: 0.5.2 - run: name: go test environment: TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> SKIP_CONFORMANCE: "1" command: | mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ --format << parameters.gotestsum-format >> \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ -- \ << parameters.coverage >> \ << parameters.go-test-flags >> \ << parameters.target >> no_output_timeout: 30m - store_test_results: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/<< parameters.suite >>.json - when: condition: << parameters.codecov-upload >> steps: - go/install: {package: bash} - go/install: {package: curl} - run: shell: /bin/bash -eo pipefail command: | bash <(curl -s https://codecov.io/bash) test-conformance: description: | Run tests using a corpus of interoperable test vectors for Filecoin implementations to test their correctness and compliance with the Filecoin specifications. parameters: <<: *test-params vectors-branch: type: string default: "" description: | Branch on github.com/filecoin-project/test-vectors to checkout and test with. If empty (the default) the commit defined by the git submodule is used. executor: << parameters.executor >> steps: - install-deps - prepare - run: command: make deps lotus no_output_timeout: 30m - download-params - when: condition: not: equal: [ "", << parameters.vectors-branch >> ] steps: - run: name: checkout vectors branch command: | cd extern/test-vectors git fetch git checkout origin/<< parameters.vectors-branch >> - go/install-gotestsum: gobin: $HOME/.local/bin version: 0.5.2 - run: name: install statediff globally command: | ## statediff is optional; we succeed even if compilation fails. mkdir -p /tmp/statediff git clone https://github.com/filecoin-project/statediff.git /tmp/statediff cd /tmp/statediff go install ./cmd/statediff || exit 0 - run: name: go test environment: SKIP_CONFORMANCE: "0" command: | mkdir -p /tmp/test-reports mkdir -p /tmp/test-artifacts gotestsum \ --format pkgname-and-test-fails \ --junitfile /tmp/test-reports/junit.xml \ -- \ -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html no_output_timeout: 30m - store_test_results: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html build-ntwk-calibration: description: | Compile lotus binaries for the calibration network parameters: <<: *test-params executor: << parameters.executor >> steps: - install-deps - prepare - run: make calibnet - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet - persist_to_workspace: root: "." paths: - linux-calibrationnet build-ntwk-butterfly: description: | Compile lotus binaries for the butterfly network parameters: <<: *test-params executor: << parameters.executor >> steps: - install-deps - prepare - run: make butterflynet - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet - persist_to_workspace: root: "." paths: - linux-butterflynet build-lotus-soup: description: | Compile `lotus-soup` Testground test plan parameters: <<: *test-params executor: << parameters.executor >> steps: - install-deps - prepare - run: cd extern/filecoin-ffi && make - run: name: "go get lotus@master" command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy - run: name: "build lotus-soup testplan" command: pushd testplans/lotus-soup && go build -tags=testground . trigger-testplans: description: | Trigger `lotus-soup` test cases on TaaS parameters: <<: *test-params executor: << parameters.executor >> steps: - install-deps - prepare - run: name: "download testground" command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli - run: name: "prepare .env.toml" command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml - run: name: "prepare testground home dir and link test plans" command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup - run: name: "go get lotus@master" command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master - run: name: "trigger deals baseline testplan on taas" command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - run: name: "trigger payment channel stress testplan on taas" command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH build-macos: description: build darwin lotus binary macos: xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus steps: - prepare: linux: false darwin: true - run: name: Install go command: | curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \ sudo installer -pkg go1.16.4.darwin-amd64.pkg -target / - run: name: Install pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config - run: go version - run: name: Install Rust command: | curl https://sh.rustup.rs -sSf | sh -s -- -y - run: name: Install hwloc command: | mkdir ~/hwloc curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz cd ~/hwloc tar -xvzpf hwloc-2.4.1.tar.gz cd hwloc-2.4.1 ./configure && make && sudo make install - restore_cache: name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - install-deps - run: command: make build no_output_timeout: 30m - run: name: check tag and version output match command: ./scripts/version-check.sh ./lotus - store_artifacts: path: lotus - store_artifacts: path: lotus-miner - store_artifacts: path: lotus-worker - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ - persist_to_workspace: root: "." paths: - darwin - save_cache: name: save cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} paths: - "~/.rustup" - "~/.cargo" build-appimage: machine: image: ubuntu-2004:202111-02 steps: - checkout - attach_workspace: at: "." - run: name: install appimage-builder command: | # appimage-builder requires /dev/snd to exist. It creates containers during the testing phase # that pass sound devices from the host to the testing container. (hard coded!) # https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54 # Circleci doesn't provide a working sound device; this is enough to fake it. if [ ! -e /dev/snd ] then sudo mkdir /dev/snd sudo mknod /dev/snd/ControlC0 c 1 2 fi # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html sudo apt update sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage sudo chmod +x /usr/local/bin/appimagetool sudo pip3 install appimage-builder - run: name: install lotus dependencies command: sudo apt install ocl-icd-opencl-dev libhwloc-dev - run: name: build appimage command: | sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml make appimage - run: name: prepare workspace command: | mkdir appimage mv Lotus-*.AppImage appimage - persist_to_workspace: root: "." paths: - appimage gofmt: executor: golang steps: - install-deps - prepare - run: command: "! go fmt ./... 2>&1 | read" gen-check: executor: golang steps: - install-deps - prepare - run: make deps - run: go install golang.org/x/tools/cmd/goimports - run: go install github.com/hannahhoward/cbor-gen-for - run: make gen - run: git --no-pager diff - run: git --no-pager diff --quiet - run: make docsgen-cli - run: git --no-pager diff - run: git --no-pager diff --quiet docs-check: executor: golang steps: - install-deps - prepare - run: go install golang.org/x/tools/cmd/goimports - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - run: make deps - run: make docsgen - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - run: git --no-pager diff - run: diff ../pre-openrpc-full ../post-openrpc-full - run: diff ../pre-openrpc-miner ../post-openrpc-miner - run: diff ../pre-openrpc-worker ../post-openrpc-worker - run: git --no-pager diff --quiet lint: &lint description: | Run golangci-lint. parameters: executor: type: executor default: golang golangci-lint-version: type: string default: 1.27.0 concurrency: type: string default: '2' description: | Concurrency used to run linters. Defaults to 2 because NumCPU is not aware of container CPU limits. args: type: string default: '' description: | Arguments to pass to golangci-lint executor: << parameters.executor >> steps: - install-deps - prepare - run: command: make deps no_output_timeout: 30m - go/install-golangci-lint: gobin: $HOME/.local/bin version: << parameters.golangci-lint-version >> - run: name: Lint command: | $HOME/.local/bin/golangci-lint run -v --timeout 2m \ --concurrency << parameters.concurrency >> << parameters.args >> lint-all: <<: *lint publish: description: publish binary artifacts executor: ubuntu steps: - run: name: Install git jq curl command: apt update && apt install -y git jq curl - checkout - git_fetch_all_tags - checkout - install_ipfs - attach_workspace: at: "." - run: name: Create bundles command: ./scripts/build-bundle.sh - run: name: Publish release command: ./scripts/publish-release.sh publish-snapcraft: description: build and push snapcraft machine: image: ubuntu-2004:202104-01 resource_class: 2xlarge parameters: channel: type: string default: "edge" description: snapcraft channel steps: - checkout - run: name: install snapcraft command: sudo snap install snapcraft --classic - run: name: create snapcraft config file command: | mkdir -p ~/.config/snapcraft echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg - run: name: build snap command: snapcraft --use-lxd - run: name: publish snap command: snapcraft push *.snap --release << parameters.channel >> build-and-push-image: description: build and push docker images to public AWS ECR registry executor: aws-cli/default parameters: profile-name: type: string default: "default" description: AWS profile name to be configured. aws-access-key-id: type: env_var_name default: AWS_ACCESS_KEY_ID description: > AWS access key id for IAM role. Set this to the name of the environment variable you will set to hold this value, i.e. AWS_ACCESS_KEY. aws-secret-access-key: type: env_var_name default: AWS_SECRET_ACCESS_KEY description: > AWS secret key for IAM role. Set this to the name of the environment variable you will set to hold this value, i.e. AWS_SECRET_ACCESS_KEY. region: type: env_var_name default: AWS_REGION description: > Name of env var storing your AWS region information, defaults to AWS_REGION account-url: type: env_var_name default: AWS_ECR_ACCOUNT_URL description: > Env var storing Amazon ECR account URL that maps to an AWS account, e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com defaults to AWS_ECR_ACCOUNT_URL dockerfile: type: string default: Dockerfile description: Name of dockerfile to use. Defaults to Dockerfile. path: type: string default: . description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). extra-build-args: type: string default: "" description: > Extra flags to pass to docker build. For examples, see https://docs.docker.com/engine/reference/commandline/build repo: type: string description: Name of an Amazon ECR repository tag: type: string default: "latest" description: A comma-separated string containing docker image tags to build and push (default = latest) target: type: string default: "lotus-all-in-one" description: Docker target to build steps: - run: name: Confirm that environment variables are set command: | if [ -z "$AWS_ACCESS_KEY_ID" ]; then echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..." circleci-agent step halt fi - aws-cli/setup: profile-name: <> aws-access-key-id: <> aws-secret-access-key: <> aws-region: <> - run: name: Log into Amazon ECR command: | aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - checkout - setup_remote_docker: version: 19.03.13 docker_layer_caching: false - run: name: Build docker image command: | registry_id=$(echo $<> | sed "s;\..*;;g") docker_tag_args="" IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" for tag in "${DOCKER_TAGS[@]}"; do docker_tag_args="$docker_tag_args -t $<>/<>:$tag" done docker build \ <<#parameters.extra-build-args>><><> \ --target <> \ -f <>/<> \ $docker_tag_args \ <> - run: name: Push image to Amazon ECR command: | IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" for tag in "${DOCKER_TAGS[@]}"; do docker push $<>/<>:${tag} done publish-packer-mainnet: description: build and push AWS IAM and DigitalOcean droplet. executor: name: packer/default packer-version: 1.6.6 steps: - checkout - attach_workspace: at: "." - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" publish-packer-calibrationnet: description: build and push AWS IAM and DigitalOcean droplet. executor: name: packer/default packer-version: 1.6.6 steps: - checkout - attach_workspace: at: "." - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" publish-packer-butterflynet: description: build and push AWS IAM and DigitalOcean droplet. executor: name: packer/default packer-version: 1.6.6 steps: - checkout - attach_workspace: at: "." - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" publish-packer-snap: description: build packer image with snap. mainnet only. executor: name: packer/default packer-version: 1.6.6 steps: - checkout - attach_workspace: at: "." - packer/build: template: tools/packer/lotus-snap.pkr.hcl publish-dockerhub: description: publish to dockerhub machine: image: ubuntu-2004:202010-01 parameters: tag: type: string default: latest steps: - checkout - run: name: dockerhub login command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin - run: name: docker build command: | docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus . docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus . if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus . docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus . fi if [["[[ ! -z $CIRCLE_TAG ]]"]]; then docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus . docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus . fi - run: name: docker push command: | docker push filecoin/lotus:<< parameters.tag >> docker push filecoin/lotus-all-in-one:<< parameters.tag >> if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then docker push filecoin/lotus:$CIRCLE_SHA1 docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1 fi if [["[[ ! -z $CIRCLE_TAG ]]"]]; then docker push filecoin/lotus:$CIRCLE_TAG docker push filecoin/lotus-all-in-one:$CIRCLE_TAG fi workflows: version: 2.1 ci: jobs: - lint-all: concurrency: "16" # expend all docker 2xlarge CPUs. - mod-tidy-check - gofmt - gen-check - docs-check [[- range $file := .ItestFiles -]] [[ with $name := $file | stripSuffix ]] - test: name: test-itest-[[ $name ]] suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" [[ end ]] [[- end -]] [[range $suite, $pkgs := .UnitSuites]] - test: name: test-[[ $suite ]] suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" [[- end]] - test: go-test-flags: "-run=TestMulticoreSDR" suite: multicore-sdr-check target: "./extern/sector-storage/ffiwrapper" proofs-log-test: "1" - test-conformance: suite: conformance codecov-upload: false target: "./conformance" - test-conformance: name: test-conformance-bleeding-edge codecov-upload: false suite: conformance-bleeding-edge target: "./conformance" vectors-branch: specs-actors-v7 - trigger-testplans: filters: branches: only: - master - build-debug - build-all: filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-ntwk-calibration: filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-ntwk-butterfly: filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - build-macos: filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-appimage: filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish: requires: - build-all - build-macos - build-appimage filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-and-push-image: dockerfile: Dockerfile.lotus path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' target: lotus-all-in-one - build-and-push-image: dockerfile: Dockerfile.lotus path: . repo: lotus-test tag: '${CIRCLE_SHA1:0:8}' target: lotus-test - publish-packer-mainnet: requires: - build-all filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish-packer-calibrationnet: requires: - build-ntwk-calibration filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish-packer-butterflynet: requires: - build-ntwk-butterfly filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish-snapcraft: name: publish-snapcraft-stable channel: stable filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - publish-dockerhub: name: publish-dockerhub tag: stable filters: branches: ignore: - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ nightly: triggers: - schedule: cron: "0 0 * * *" filters: branches: only: - master jobs: - publish-snapcraft: name: publish-snapcraft-nightly channel: edge - publish-dockerhub: name: publish-dockerhub-nightly tag: nightly monthly: triggers: - schedule: cron: "0 0 1 * *" filters: branches: only: - master jobs: - publish-packer-snap