lotus/.circleci/template.yml
Phi-rjan 1a21b42fd7 Chore: Backports to the release/v1.26.0 branch (#11713)
* enable storing events (#11712)

* fix: commit batch: Always go through commit batcher (#11704)

* fix: commit batch: Always go through commit batcher

* fix sealing fsm tests

* sealing pipeline: Fix panic on padding pieces in WaitDeals (#11708)

* sealing pipeline: Fix panic on padding pieces in WaitDeals

* sealing pipeline: Catch panics

* sealing pipeline: Output DDO pieces in SectorStatus (#11709)

* sealing pipeline: Fix failing ProveCommit3 aggregate (#11710)

* itests: Repro failing ProveCommit3 aggregate

* commit batch: Correctly sort sectors in processBatchV2

* fix imports

* ci: Bigger instance for sector_pledge test

* itests: Use Must-Post mining in TestPledgeBatching

---------

Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
Co-authored-by: Łukasz Magiera <magik6k@users.noreply.github.com>
2024-03-22 07:00:28 +01:00

760 lines
24 KiB
YAML

version: 2.1
orbs:
aws-cli: circleci/aws-cli@4.1.1
docker: circleci/docker@2.3.0
executors:
golang:
docker:
# Must match GO_VERSION_MIN in project root
- image: cimg/go:1.20.7
resource_class: medium+
golang-2xl:
docker:
# Must match GO_VERSION_MIN in project root
- image: cimg/go:1.20.7
resource_class: 2xlarge
ubuntu:
docker:
- image: ubuntu:20.04
commands:
build-platform-specific:
parameters:
linux:
default: true
description: is a linux build environment?
type: boolean
darwin:
default: false
description: is a darwin build environment?
type: boolean
darwin-architecture:
default: "amd64"
description: which darwin architecture is being used?
type: string
steps:
- checkout
- git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- when:
condition: <<parameters.linux>>
steps:
- install-ubuntu-deps
- check-go-version
- when:
condition: <<parameters.darwin>>
steps:
- run:
name: Install Go
command: |
curl https://dl.google.com/go/go`cat GO_VERSION_MIN`.darwin-<<parameters.darwin-architecture>>.pkg -o /tmp/go.pkg && \
sudo installer -pkg /tmp/go.pkg -target /
- run:
name: Export Go
command: |
echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
- run: go version
- run:
name: Install dependencies with Homebrew
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config coreutils jq hwloc
- run:
name: Install Rust
command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y
- run: make deps
download-params:
steps:
- restore_cache:
name: Restore parameters cache
keys:
- 'v26-2k-lotus-params'
- run: ./lotus fetch-params 2048
- save_cache:
name: Save parameters cache
key: 'v26-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:
steps:
- run: |
curl -O https://dist.ipfs.tech/kubo/v0.16.0/kubo_v0.16.0_linux-amd64.tar.gz
tar -xvzf kubo_v0.16.0_linux-amd64.tar.gz
pushd kubo
sudo bash install.sh
popd
rm -rf kubo
rm kubo_v0.16.0_linux-amd64.tar.gz
git_fetch_all_tags:
steps:
- run:
name: fetch all tags
command: |
git fetch --all
install-ubuntu-deps:
steps:
- run: sudo apt install curl ca-certificates gnupg
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
check-go-version:
steps:
- run: |
v=`go version | { read _ _ v _; echo ${v#go}; }`
if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then
echo "GO_VERSION_MIN file does not match the go version being used."
echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v."
exit 1
fi
jobs:
build:
executor: golang
working_directory: ~/lotus
steps:
- checkout
- git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- install-ubuntu-deps
- check-go-version
- run: make deps lotus
- persist_to_workspace:
root: ~/
paths:
- "lotus"
mod-tidy-check:
executor: golang
working_directory: ~/lotus
steps:
- install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go mod tidy -v
- run:
name: Check git diff
command: |
git --no-pager diff go.mod go.sum
git --no-pager diff --quiet go.mod go.sum
test:
description: |
Run tests with gotestsum.
working_directory: ~/lotus
parameters: &test-params
resource_class:
type: string
default: medium+
go-test-flags:
type: string
default: "-timeout 20m"
description: Flags passed to go test.
target:
type: string
default: "./..."
description: Import paths of packages to be tested.
proofs-log-test:
type: string
default: "0"
get-params:
type: boolean
default: false
suite:
type: string
default: unit
description: Test suite name to report to CircleCI.
docker:
- image: cimg/go:[[ .GoVersion]]
environment:
LOTUS_HARMONYDB_HOSTS: yugabyte
- image: yugabytedb/yugabyte:2.18.0.0-b65
command: bin/yugabyted start --daemon=false
name: yugabyte
resource_class: << parameters.resource_class >>
steps:
- install-ubuntu-deps
- attach_workspace:
at: ~/
- when:
condition: << parameters.get-params >>
steps:
- download-params
- run:
name: go test
environment:
TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
SKIP_CONFORMANCE: "1"
LOTUS_SRC_DIR: /home/circleci/project
command: |
mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts
dockerize -wait tcp://yugabyte:5433 -timeout 3m
env
gotestsum \
--format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
--packages="<< parameters.target >>" \
-- << parameters.go-test-flags >>
no_output_timeout: 30m
- store_test_results:
path: /tmp/test-reports
- store_artifacts:
path: /tmp/test-artifacts/<< parameters.suite >>.json
test-conformance:
working_directory: ~/lotus
description: |
Run tests using a corpus of interoperable test vectors for Filecoin
implementations to test their correctness and compliance with the Filecoin
specifications.
parameters:
<<: *test-params
vectors-branch:
type: string
default: ""
description: |
Branch on github.com/filecoin-project/test-vectors to checkout and
test with. If empty (the default) the commit defined by the git
submodule is used.
docker:
- image: cimg/go:[[ .GoVersion]]
resource_class: << parameters.resource_class >>
steps:
- install-ubuntu-deps
- attach_workspace:
at: ~/
- download-params
- when:
condition:
not:
equal: [ "", << parameters.vectors-branch >> ]
steps:
- run:
name: checkout vectors branch
command: |
cd extern/test-vectors
git fetch
git checkout origin/<< parameters.vectors-branch >>
- run:
name: install statediff globally
command: |
## statediff is optional; we succeed even if compilation fails.
mkdir -p /tmp/statediff
git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
cd /tmp/statediff
go install ./cmd/statediff || exit 0
- run:
name: go test
environment:
SKIP_CONFORMANCE: "0"
command: |
mkdir -p /tmp/test-reports
mkdir -p /tmp/test-artifacts
gotestsum \
--format pkgname-and-test-fails \
--junitfile /tmp/test-reports/junit.xml \
-- \
-v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
no_output_timeout: 30m
- store_test_results:
path: /tmp/test-reports
- store_artifacts:
path: /tmp/test-artifacts/conformance-coverage.html
build-linux-amd64:
executor: golang
steps:
- build-platform-specific
- run: make lotus lotus-miner lotus-worker
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- run: |
mkdir -p /tmp/workspace/linux_amd64_v1 && \
mv lotus lotus-miner lotus-worker /tmp/workspace/linux_amd64_v1/
- persist_to_workspace:
root: /tmp/workspace
paths:
- linux_amd64_v1
build-darwin-amd64:
description: build darwin lotus binary
working_directory: ~/go/src/github.com/filecoin-project/lotus
macos:
xcode: "13.4.1"
steps:
- build-platform-specific:
linux: false
darwin: true
darwin-architecture: amd64
- run: make lotus lotus-miner lotus-worker
- run: otool -hv lotus
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- run: |
mkdir -p /tmp/workspace/darwin_amd64_v1 && \
mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_amd64_v1/
- persist_to_workspace:
root: /tmp/workspace
paths:
- darwin_amd64_v1
build-darwin-arm64:
description: self-hosted m1 runner
working_directory: ~/go/src/github.com/filecoin-project/lotus
machine: true
resource_class: filecoin-project/self-hosted-m1
steps:
- run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV"
- build-platform-specific:
linux: false
darwin: true
darwin-architecture: arm64
- run: |
export CPATH=$(brew --prefix)/include && export LIBRARY_PATH=$(brew --prefix)/lib && make lotus lotus-miner lotus-worker
- run: otool -hv lotus
- run:
name: check tag and version output match
command: ./scripts/version-check.sh ./lotus
- run: |
mkdir -p /tmp/workspace/darwin_arm64 && \
mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_arm64/
- persist_to_workspace:
root: /tmp/workspace
paths:
- darwin_arm64
- run:
command: make clean
when: always
- run:
name: cleanup homebrew
command: HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall pkg-config coreutils jq hwloc
when: always
release:
executor: golang
parameters:
dry-run:
default: false
description: should this release actually publish it's artifacts?
type: boolean
steps:
- checkout
- run: |
echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list
sudo apt update
sudo apt install goreleaser-pro
- install_ipfs
- attach_workspace:
at: /tmp/workspace
- when:
condition: << parameters.dry-run >>
steps:
- run: goreleaser release --rm-dist --snapshot --debug
- run: ./scripts/generate-checksums.sh
- when:
condition:
not: << parameters.dry-run >>
steps:
- run: goreleaser release --rm-dist --debug
- run: ./scripts/generate-checksums.sh
- run: ./scripts/publish-checksums.sh
gofmt:
executor: golang
working_directory: ~/lotus
steps:
- run:
command: "! go fmt ./... 2>&1 | read"
gen-check:
executor: golang
working_directory: ~/lotus
steps:
- install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go install golang.org/x/tools/cmd/goimports
- run: go install github.com/hannahhoward/cbor-gen-for
- run: make gen
- run: git --no-pager diff && git --no-pager diff --quiet
- run: make docsgen-cli
- run: git --no-pager diff && git --no-pager diff --quiet
docs-check:
executor: golang
working_directory: ~/lotus
steps:
- install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go install golang.org/x/tools/cmd/goimports
- run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
- run: make docsgen
- run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
- run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet
lint-all:
description: |
Run golangci-lint.
working_directory: ~/lotus
parameters:
args:
type: string
default: ''
description: |
Arguments to pass to golangci-lint
docker:
- image: cimg/go:[[ .GoVersion]]
resource_class: medium+
steps:
- install-ubuntu-deps
- attach_workspace:
at: ~/
- run:
name: Lint
command: |
golangci-lint run -v --timeout 10m \
--concurrency 4 << parameters.args >>
build-docker:
description: >
Publish to Dockerhub
executor: docker/docker
parameters:
image:
type: string
default: lotus
description: >
Passed to the docker build process to determine which image in the
Dockerfile should be built. Expected values are `lotus`,
`lotus-all-in-one`
network:
type: string
default: "mainnet"
description: >
Passed to the docker build process using GOFLAGS+=-tags=<<network>>.
Expected values are `debug`, `2k`, `calibnet`, `butterflynet`,
`interopnet`.
channel:
type: string
default: ""
description: >
The release channel to use for this image.
push:
type: boolean
default: false
description: >
When true, pushes the image to Dockerhub
steps:
- setup_remote_docker
- checkout
- git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- docker/check:
docker-username: DOCKERHUB_USERNAME
docker-password: DOCKERHUB_PASSWORD
- when:
condition:
equal: [ mainnet, <<parameters.network>> ]
steps:
- when:
condition: <<parameters.push>>
steps:
- docker/build:
image: filecoin/<<parameters.image>>
extra_build_args: --target <<parameters.image>>
tag: <<parameters.channel>>
- run:
name: Docker push
command: |
docker push filecoin/<<parameters.image>>:<<parameters.channel>>
if [["[[ ! -z $CIRCLE_SHA ]]"]]; then
docker image tag filecoin/<<parameters.image>>:<<parameters.channel>> filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}"
docker push filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}"
fi
if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
docker image tag filecoin/<<parameters.image>>:<<parameters.channel>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"
docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"
fi
- unless:
condition: <<parameters.push>>
steps:
- docker/build:
image: filecoin/<<parameters.image>>
extra_build_args: --target <<parameters.image>>
- when:
condition:
not:
equal: [ mainnet, <<parameters.network>> ]
steps:
- when:
condition: <<parameters.push>>
steps:
- docker/build:
image: filecoin/<<parameters.image>>
extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
tag: <<parameters.channel>>-<<parameters.network>>
- run:
name: Docker push
command: |
docker push filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>>
if [["[[ ! -z $CIRCLE_SHA ]]"]]; then
docker image tag filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}"-<<parameters.network>>
docker push filecoin/<<parameters.image>>:"${CIRCLE_SHA:0:7}"-<<parameters.network>>
fi
if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
docker image tag filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>>
docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>>
fi
- unless:
condition: <<parameters.push>>
steps:
- docker/build:
image: filecoin/<<parameters.image>>
extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
workflows:
ci:
jobs:
- build
- lint-all:
requires:
- build
- mod-tidy-check:
requires:
- build
- gofmt:
requires:
- build
- gen-check:
requires:
- build
- docs-check:
requires:
- build
[[- range $file := .ItestFiles -]]
[[ with $name := $file | stripSuffix ]]
- test:
name: test-itest-[[ $name ]]
requires:
- build
suite: itest-[[ $name ]]
target: "./itests/[[ $file ]]"
[[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config") (eq $name "sector_pledge")]]
resource_class: 2xlarge
[[- end]]
[[- if or (eq $name "wdpost") (eq $name "sector_pledge")]]
get-params: true
[[end]]
[[- end ]][[- end]]
[[- range $suite, $pkgs := .UnitSuites]]
- test:
name: test-[[ $suite ]]
requires:
- build
suite: utest-[[ $suite ]]
target: "[[ $pkgs ]]"
[[- if eq $suite "unit-storage"]]
get-params: true
[[- end -]]
[[- if eq $suite "unit-cli"]]
resource_class: 2xlarge
get-params: true
[[- end -]]
[[- if eq $suite "unit-rest"]]
resource_class: 2xlarge
[[- end -]]
[[- end]]
- test:
go-test-flags: "-run=TestMulticoreSDR"
requires:
- build
suite: multicore-sdr-check
target: "./storage/sealer/ffiwrapper"
proofs-log-test: "1"
- test-conformance:
requires:
- build
suite: conformance
target: "./conformance"
release:
jobs:
- build-linux-amd64:
name: "Build ( linux / amd64 )"
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
- /^ci\/.*$/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-darwin-amd64:
name: "Build ( darwin / amd64 )"
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
- /^ci\/.*$/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- build-darwin-arm64:
name: "Build ( darwin / arm64 )"
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
- /^ci\/.*$/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- release:
name: "Release"
requires:
- "Build ( darwin / amd64 )"
- "Build ( linux / amd64 )"
- "Build ( darwin / arm64 )"
filters:
branches:
ignore:
- /^.*$/
tags:
only:
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
- release:
name: "Release (dry-run)"
dry-run: true
requires:
- "Build ( darwin / amd64 )"
- "Build ( linux / amd64 )"
- "Build ( darwin / arm64 )"
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
- /^ci\/.*$/
[[- range .Networks]]
- build-docker:
name: "Docker push (lotus-all-in-one / stable / [[.]])"
image: lotus-all-in-one
channel: stable
network: [[.]]
push: true
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+$/
- build-docker:
name: "Docker push (lotus-all-in-one / candidate / [[.]])"
image: lotus-all-in-one
channel: candidate
network: [[.]]
push: true
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+-rc\d+$/
- build-docker:
name: "Docker push (lotus-all-in-one / edge / [[.]])"
image: lotus-all-in-one
channel: master
network: [[.]]
push: true
filters:
branches:
only:
- master
- build-docker:
name: "Docker build (lotus-all-in-one / [[.]])"
image: lotus-all-in-one
network: [[.]]
push: false
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
[[- end]]
- build-docker:
name: "Docker push (lotus / stable / mainnet)"
image: lotus
channel: stable
network: mainnet
push: true
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+$/
- build-docker:
name: "Docker push (lotus / candidate / mainnet)"
image: lotus
channel: candidate
network: mainnet
push: true
filters:
branches:
ignore:
- /.*/
tags:
only:
- /^v\d+\.\d+\.\d+-rc\d+$/
- build-docker:
name: "Docker push (lotus / master / mainnet)"
image: lotus
channel: master
network: mainnet
push: true
filters:
branches:
only:
- master
- build-docker:
name: "Docker build (lotus / mainnet)"
image: lotus
network: mainnet
push: false
filters:
branches:
only:
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
jobs:
[[- range .Networks]]
- build-docker:
name: "Docker (lotus-all-in-one / nightly / [[.]])"
image: lotus-all-in-one
channel: nightly
network: [[.]]
push: true
[[- end]]