cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #9910 from filecoin-project/gstuart/deps-caching

Browse Source 
feat: ci: make ci more efficient
This commit is contained in:
Łukasz Magiera 2023-01-13 20:08:20 +01:00 committed by GitHub
commit bb800fd78d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 358 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

369
.circleci/config.yml
View File

@ -5,6 +5,11 @@ orbs:
executors: executors:
golang: golang:
docker:
# Must match GO_VERSION_MIN in project root
- image: cimg/go:1.18.8
resource_class: medium+
golang-2xl:
docker: docker:
# Must match GO_VERSION_MIN in project root # Must match GO_VERSION_MIN in project root
- image: cimg/go:1.18.8 - image: cimg/go:1.18.8
@ -14,7 +19,7 @@ executors:
- image: ubuntu:20.04 - image: ubuntu:20.04
commands: commands:
prepare: build-platform-specific:
parameters: parameters:
linux: linux:
default: true default: true
@ -31,22 +36,13 @@ commands:
steps: steps:
- checkout - checkout
- git_fetch_all_tags - git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- when: - when:
condition: <<parameters.linux>> condition: <<parameters.linux>>
steps: steps:
- run: - install-ubuntu-deps
name: Check Go Version - check-go-version
command: |
v=`go version | { read _ _ v _; echo ${v#go}; }`
if [[ $v != `cat GO_VERSION_MIN` ]]; then
echo "GO_VERSION_MIN file does not match the go version being used."
echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v."
exit 1
fi
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
- run: sudo apt-get install python-is-python3
- when: - when:
condition: <<parameters.darwin>> condition: <<parameters.darwin>>
steps: steps:
@ -67,8 +63,7 @@ commands:
name: Install Rust name: Install Rust
command: | command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y curl https://sh.rustup.rs -sSf | sh -s -- -y
- run: git submodule sync - run: make deps lotus
- run: git submodule update --init
download-params: download-params:
steps: steps:
- restore_cache: - restore_cache:
@ -99,12 +94,43 @@ commands:
name: fetch all tags name: fetch all tags
command: | command: |
git fetch --all git fetch --all
install-ubuntu-deps:
steps:
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
check-go-version:
steps:
- run: |
v=`go version | { read _ _ v _; echo ${v#go}; }`
if [[ $v != `cat GO_VERSION_MIN` ]]; then
echo "GO_VERSION_MIN file does not match the go version being used."
echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v."
exit 1
fi
jobs: jobs:
build:
executor: golang
working_directory: ~/lotus
steps:
- checkout
- git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- install-ubuntu-deps
- check-go-version
- run: make deps lotus
- persist_to_workspace:
root: ~/
paths:
- "lotus"
mod-tidy-check: mod-tidy-check:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare - install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go mod tidy -v - run: go mod tidy -v
- run: - run:
name: Check git diff name: Check git diff
@ -115,13 +141,14 @@ jobs:
test: test:
description: | description: |
Run tests with gotestsum. Run tests with gotestsum.
working_directory: ~/lotus
parameters: &test-params parameters: &test-params
executor: executor:
type: executor type: executor
default: golang default: golang
go-test-flags: go-test-flags:
type: string type: string
default: "-timeout 30m" default: "-timeout 20m"
description: Flags passed to go test. description: Flags passed to go test.
target: target:
type: string type: string
@ -130,20 +157,21 @@ jobs:
proofs-log-test: proofs-log-test:
type: string type: string
default: "0" default: "0"
get-params:
type: boolean
default: false
suite: suite:
type: string type: string
default: unit default: unit
description: Test suite name to report to CircleCI. description: Test suite name to report to CircleCI.
gotestsum-format:
type: string
default: standard-verbose
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
executor: << parameters.executor >> executor: << parameters.executor >>
steps: steps:
- prepare - install-ubuntu-deps
- run: - attach_workspace:
command: make deps lotus at: ~/
no_output_timeout: 30m - when:
condition: << parameters.get-params >>
steps:
- download-params - download-params
- run: - run:
name: go test name: go test
@ -155,12 +183,11 @@ jobs:
mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts mkdir -p /tmp/test-artifacts
gotestsum \ gotestsum \
--format << parameters.gotestsum-format >> \ --format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-- \ --packages="<< parameters.target >>" \
<< parameters.go-test-flags >> \ -- << parameters.go-test-flags >>
<< parameters.target >>
no_output_timeout: 30m no_output_timeout: 30m
- store_test_results: - store_test_results:
path: /tmp/test-reports path: /tmp/test-reports
@ -168,6 +195,7 @@ jobs:
path: /tmp/test-artifacts/<< parameters.suite >>.json path: /tmp/test-artifacts/<< parameters.suite >>.json
test-conformance: test-conformance:
working_directory: ~/lotus
description: | description: |
Run tests using a corpus of interoperable test vectors for Filecoin Run tests using a corpus of interoperable test vectors for Filecoin
implementations to test their correctness and compliance with the Filecoin implementations to test their correctness and compliance with the Filecoin
@ -183,10 +211,9 @@ jobs:
submodule is used. submodule is used.
executor: << parameters.executor >> executor: << parameters.executor >>
steps: steps:
- prepare - install-ubuntu-deps
- run: - attach_workspace:
command: make deps lotus at: ~/
no_output_timeout: 30m
- download-params - download-params
- when: - when:
condition: condition:
@ -229,7 +256,7 @@ jobs:
build-linux-amd64: build-linux-amd64:
executor: golang executor: golang
steps: steps:
- prepare - build-platform-specific
- run: make lotus lotus-miner lotus-worker - run: make lotus lotus-miner lotus-worker
- run: - run:
name: check tag and version output match name: check tag and version output match
@ -248,7 +275,7 @@ jobs:
macos: macos:
xcode: "13.4.1" xcode: "13.4.1"
steps: steps:
- prepare: - build-platform-specific:
linux: false linux: false
darwin: true darwin: true
darwin-architecture: amd64 darwin-architecture: amd64
@ -272,7 +299,7 @@ jobs:
resource_class: filecoin-project/self-hosted-m1 resource_class: filecoin-project/self-hosted-m1
steps: steps:
- run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV"
- prepare: - build-platform-specific:
linux: false linux: false
darwin: true darwin: true
darwin-architecture: arm64 darwin-architecture: arm64
@ -380,16 +407,18 @@ jobs:
gofmt: gofmt:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare
- run: - run:
command: "! go fmt ./... 2>&1 | read" command: "! go fmt ./... 2>&1 | read"
gen-check: gen-check:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare - install-ubuntu-deps
- run: make deps - attach_workspace:
at: ~/
- run: go install golang.org/x/tools/cmd/goimports - run: go install golang.org/x/tools/cmd/goimports
- run: go install github.com/hannahhoward/cbor-gen-for - run: go install github.com/hannahhoward/cbor-gen-for
- run: make gen - run: make gen
@ -399,32 +428,29 @@ jobs:
docs-check: docs-check:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare - install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go install golang.org/x/tools/cmd/goimports - run: go install golang.org/x/tools/cmd/goimports
- run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
- run: make deps
- run: make docsgen - run: make docsgen
- run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
- run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet - run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet
lint: &lint lint-all:
description: | description: |
Run golangci-lint. Run golangci-lint.
working_directory: ~/lotus
parameters: parameters:
executor: executor:
type: executor type: executor
default: golang default: golang
concurrency:
type: string
default: '2'
description: |
Concurrency used to run linters. Defaults to 2 because NumCPU is not
aware of container CPU limits.
args: args:
type: string type: string
default: '' default: ''
@ -432,17 +458,14 @@ jobs:
Arguments to pass to golangci-lint Arguments to pass to golangci-lint
executor: << parameters.executor >> executor: << parameters.executor >>
steps: steps:
- prepare - install-ubuntu-deps
- run: - attach_workspace:
command: make deps at: ~/
no_output_timeout: 30m
- run: - run:
name: Lint name: Lint
command: | command: |
golangci-lint run -v --timeout 2m \ golangci-lint run -v --timeout 10m \
--concurrency << parameters.concurrency >> << parameters.args >> --concurrency 4 << parameters.args >>
lint-all:
<<: *lint
publish: publish:
description: publish binary artifacts description: publish binary artifacts
@ -462,7 +485,6 @@ jobs:
command: apt update && apt install -y git jq curl sudo command: apt update && apt install -y git jq curl sudo
- checkout - checkout
- git_fetch_all_tags - git_fetch_all_tags
- checkout
- install_ipfs - install_ipfs
- attach_workspace: - attach_workspace:
at: /tmp/workspace at: /tmp/workspace
@ -606,356 +628,449 @@ jobs:
extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>> extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
workflows: workflows:
version: 2.1
ci: ci:
jobs: jobs:
- build
- lint-all: - lint-all:
concurrency: "16" # expend all docker 2xlarge CPUs. requires:
- mod-tidy-check - build
- gofmt - mod-tidy-check:
- gen-check requires:
- docs-check - build
- gofmt:
requires:
- build
- gen-check:
requires:
- build
- docs-check:
requires:
- build
- test: - test:
name: test-itest-api name: test-itest-api
requires:
- build
suite: itest-api suite: itest-api
target: "./itests/api_test.go" target: "./itests/api_test.go"
- test: - test:
name: test-itest-batch_deal name: test-itest-batch_deal
requires:
- build
suite: itest-batch_deal suite: itest-batch_deal
target: "./itests/batch_deal_test.go" target: "./itests/batch_deal_test.go"
- test: - test:
name: test-itest-ccupgrade name: test-itest-ccupgrade
requires:
- build
suite: itest-ccupgrade suite: itest-ccupgrade
target: "./itests/ccupgrade_test.go" target: "./itests/ccupgrade_test.go"
- test: - test:
name: test-itest-cli name: test-itest-cli
requires:
- build
suite: itest-cli suite: itest-cli
target: "./itests/cli_test.go" target: "./itests/cli_test.go"
- test: - test:
name: test-itest-deadlines name: test-itest-deadlines
requires:
- build
suite: itest-deadlines suite: itest-deadlines
target: "./itests/deadlines_test.go" target: "./itests/deadlines_test.go"
- test: - test:
name: test-itest-deals_512mb name: test-itest-deals_512mb
requires:
- build
suite: itest-deals_512mb suite: itest-deals_512mb
target: "./itests/deals_512mb_test.go" target: "./itests/deals_512mb_test.go"
- test: - test:
name: test-itest-deals_anycid name: test-itest-deals_anycid
requires:
- build
suite: itest-deals_anycid suite: itest-deals_anycid
target: "./itests/deals_anycid_test.go" target: "./itests/deals_anycid_test.go"
- test: - test:
name: test-itest-deals_concurrent name: test-itest-deals_concurrent
requires:
- build
suite: itest-deals_concurrent suite: itest-deals_concurrent
target: "./itests/deals_concurrent_test.go" target: "./itests/deals_concurrent_test.go"
executor: golang-2xl
- test: - test:
name: test-itest-deals_invalid_utf8_label name: test-itest-deals_invalid_utf8_label
requires:
- build
suite: itest-deals_invalid_utf8_label suite: itest-deals_invalid_utf8_label
target: "./itests/deals_invalid_utf8_label_test.go" target: "./itests/deals_invalid_utf8_label_test.go"
- test: - test:
name: test-itest-deals_max_staging_deals name: test-itest-deals_max_staging_deals
requires:
- build
suite: itest-deals_max_staging_deals suite: itest-deals_max_staging_deals
target: "./itests/deals_max_staging_deals_test.go" target: "./itests/deals_max_staging_deals_test.go"
- test: - test:
name: test-itest-deals_offline name: test-itest-deals_offline
requires:
- build
suite: itest-deals_offline suite: itest-deals_offline
target: "./itests/deals_offline_test.go" target: "./itests/deals_offline_test.go"
- test: - test:
name: test-itest-deals_padding name: test-itest-deals_padding
requires:
- build
suite: itest-deals_padding suite: itest-deals_padding
target: "./itests/deals_padding_test.go" target: "./itests/deals_padding_test.go"
- test: - test:
name: test-itest-deals_partial_retrieval_dm-level name: test-itest-deals_partial_retrieval_dm-level
requires:
- build
suite: itest-deals_partial_retrieval_dm-level suite: itest-deals_partial_retrieval_dm-level
target: "./itests/deals_partial_retrieval_dm-level_test.go" target: "./itests/deals_partial_retrieval_dm-level_test.go"
- test: - test:
name: test-itest-deals_partial_retrieval name: test-itest-deals_partial_retrieval
requires:
- build
suite: itest-deals_partial_retrieval suite: itest-deals_partial_retrieval
target: "./itests/deals_partial_retrieval_test.go" target: "./itests/deals_partial_retrieval_test.go"
- test: - test:
name: test-itest-deals_power name: test-itest-deals_power
requires:
- build
suite: itest-deals_power suite: itest-deals_power
target: "./itests/deals_power_test.go" target: "./itests/deals_power_test.go"
- test: - test:
name: test-itest-deals_pricing name: test-itest-deals_pricing
requires:
- build
suite: itest-deals_pricing suite: itest-deals_pricing
target: "./itests/deals_pricing_test.go" target: "./itests/deals_pricing_test.go"
- test: - test:
name: test-itest-deals_publish name: test-itest-deals_publish
requires:
- build
suite: itest-deals_publish suite: itest-deals_publish
target: "./itests/deals_publish_test.go" target: "./itests/deals_publish_test.go"
- test: - test:
name: test-itest-deals_remote_retrieval name: test-itest-deals_remote_retrieval
requires:
- build
suite: itest-deals_remote_retrieval suite: itest-deals_remote_retrieval
target: "./itests/deals_remote_retrieval_test.go" target: "./itests/deals_remote_retrieval_test.go"
- test: - test:
name: test-itest-deals_retry_deal_no_funds name: test-itest-deals_retry_deal_no_funds
requires:
- build
suite: itest-deals_retry_deal_no_funds suite: itest-deals_retry_deal_no_funds
target: "./itests/deals_retry_deal_no_funds_test.go" target: "./itests/deals_retry_deal_no_funds_test.go"
- test: - test:
name: test-itest-deals name: test-itest-deals
requires:
- build
suite: itest-deals suite: itest-deals
target: "./itests/deals_test.go" target: "./itests/deals_test.go"
- test: - test:
name: test-itest-dup_mpool_messages name: test-itest-dup_mpool_messages
requires:
- build
suite: itest-dup_mpool_messages suite: itest-dup_mpool_messages
target: "./itests/dup_mpool_messages_test.go" target: "./itests/dup_mpool_messages_test.go"
- test: - test:
name: test-itest-gas_estimation name: test-itest-gas_estimation
requires:
- build
suite: itest-gas_estimation suite: itest-gas_estimation
target: "./itests/gas_estimation_test.go" target: "./itests/gas_estimation_test.go"
- test: - test:
name: test-itest-gateway name: test-itest-gateway
requires:
- build
suite: itest-gateway suite: itest-gateway
target: "./itests/gateway_test.go" target: "./itests/gateway_test.go"
- test: - test:
name: test-itest-get_messages_in_ts name: test-itest-get_messages_in_ts
requires:
- build
suite: itest-get_messages_in_ts suite: itest-get_messages_in_ts
target: "./itests/get_messages_in_ts_test.go" target: "./itests/get_messages_in_ts_test.go"
- test: - test:
name: test-itest-lite_migration name: test-itest-lite_migration
requires:
- build
suite: itest-lite_migration suite: itest-lite_migration
target: "./itests/lite_migration_test.go" target: "./itests/lite_migration_test.go"
- test: - test:
name: test-itest-lookup_robust_address name: test-itest-lookup_robust_address
requires:
- build
suite: itest-lookup_robust_address suite: itest-lookup_robust_address
target: "./itests/lookup_robust_address_test.go" target: "./itests/lookup_robust_address_test.go"
- test: - test:
name: test-itest-mempool name: test-itest-mempool
requires:
- build
suite: itest-mempool suite: itest-mempool
target: "./itests/mempool_test.go" target: "./itests/mempool_test.go"
- test: - test:
name: test-itest-migration_nv17 name: test-itest-migration_nv17
requires:
- build
suite: itest-migration_nv17 suite: itest-migration_nv17
target: "./itests/migration_nv17_test.go" target: "./itests/migration_nv17_test.go"
- test: - test:
name: test-itest-mpool_msg_uuid name: test-itest-mpool_msg_uuid
requires:
- build
suite: itest-mpool_msg_uuid suite: itest-mpool_msg_uuid
target: "./itests/mpool_msg_uuid_test.go" target: "./itests/mpool_msg_uuid_test.go"
- test: - test:
name: test-itest-mpool_push_with_uuid name: test-itest-mpool_push_with_uuid
requires:
- build
suite: itest-mpool_push_with_uuid suite: itest-mpool_push_with_uuid
target: "./itests/mpool_push_with_uuid_test.go" target: "./itests/mpool_push_with_uuid_test.go"
- test: - test:
name: test-itest-multisig name: test-itest-multisig
requires:
- build
suite: itest-multisig suite: itest-multisig
target: "./itests/multisig_test.go" target: "./itests/multisig_test.go"
- test: - test:
name: test-itest-net name: test-itest-net
requires:
- build
suite: itest-net suite: itest-net
target: "./itests/net_test.go" target: "./itests/net_test.go"
- test: - test:
name: test-itest-nonce name: test-itest-nonce
requires:
- build
suite: itest-nonce suite: itest-nonce
target: "./itests/nonce_test.go" target: "./itests/nonce_test.go"
- test: - test:
name: test-itest-path_detach_redeclare name: test-itest-path_detach_redeclare
requires:
- build
suite: itest-path_detach_redeclare suite: itest-path_detach_redeclare
target: "./itests/path_detach_redeclare_test.go" target: "./itests/path_detach_redeclare_test.go"
- test: - test:
name: test-itest-path_type_filters name: test-itest-path_type_filters
requires:
- build
suite: itest-path_type_filters suite: itest-path_type_filters
target: "./itests/path_type_filters_test.go" target: "./itests/path_type_filters_test.go"
- test: - test:
name: test-itest-paych_api name: test-itest-paych_api
requires:
- build
suite: itest-paych_api suite: itest-paych_api
target: "./itests/paych_api_test.go" target: "./itests/paych_api_test.go"
- test: - test:
name: test-itest-paych_cli name: test-itest-paych_cli
requires:
- build
suite: itest-paych_cli suite: itest-paych_cli
target: "./itests/paych_cli_test.go" target: "./itests/paych_cli_test.go"
- test: - test:
name: test-itest-pending_deal_allocation name: test-itest-pending_deal_allocation
requires:
- build
suite: itest-pending_deal_allocation suite: itest-pending_deal_allocation
target: "./itests/pending_deal_allocation_test.go" target: "./itests/pending_deal_allocation_test.go"
- test: - test:
name: test-itest-raft_messagesigner name: test-itest-raft_messagesigner
requires:
- build
suite: itest-raft_messagesigner suite: itest-raft_messagesigner
target: "./itests/raft_messagesigner_test.go" target: "./itests/raft_messagesigner_test.go"
- test: - test:
name: test-itest-remove_verifreg_datacap name: test-itest-remove_verifreg_datacap
requires:
- build
suite: itest-remove_verifreg_datacap suite: itest-remove_verifreg_datacap
target: "./itests/remove_verifreg_datacap_test.go" target: "./itests/remove_verifreg_datacap_test.go"
- test: - test:
name: test-itest-sdr_upgrade name: test-itest-sdr_upgrade
requires:
- build
suite: itest-sdr_upgrade suite: itest-sdr_upgrade
target: "./itests/sdr_upgrade_test.go" target: "./itests/sdr_upgrade_test.go"
- test: - test:
name: test-itest-sector_finalize_early name: test-itest-sector_finalize_early
requires:
- build
suite: itest-sector_finalize_early suite: itest-sector_finalize_early
target: "./itests/sector_finalize_early_test.go" target: "./itests/sector_finalize_early_test.go"
- test: - test:
name: test-itest-sector_import_full name: test-itest-sector_import_full
requires:
- build
suite: itest-sector_import_full suite: itest-sector_import_full
target: "./itests/sector_import_full_test.go" target: "./itests/sector_import_full_test.go"
- test: - test:
name: test-itest-sector_import_simple name: test-itest-sector_import_simple
requires:
- build
suite: itest-sector_import_simple suite: itest-sector_import_simple
target: "./itests/sector_import_simple_test.go" target: "./itests/sector_import_simple_test.go"
- test: - test:
name: test-itest-sector_make_cc_avail name: test-itest-sector_make_cc_avail
requires:
- build
suite: itest-sector_make_cc_avail suite: itest-sector_make_cc_avail
target: "./itests/sector_make_cc_avail_test.go" target: "./itests/sector_make_cc_avail_test.go"
- test: - test:
name: test-itest-sector_miner_collateral name: test-itest-sector_miner_collateral
requires:
- build
suite: itest-sector_miner_collateral suite: itest-sector_miner_collateral
target: "./itests/sector_miner_collateral_test.go" target: "./itests/sector_miner_collateral_test.go"
- test: - test:
name: test-itest-sector_numassign name: test-itest-sector_numassign
requires:
- build
suite: itest-sector_numassign suite: itest-sector_numassign
target: "./itests/sector_numassign_test.go" target: "./itests/sector_numassign_test.go"
- test: - test:
name: test-itest-sector_pledge name: test-itest-sector_pledge
requires:
- build
suite: itest-sector_pledge suite: itest-sector_pledge
target: "./itests/sector_pledge_test.go" target: "./itests/sector_pledge_test.go"
- test: - test:
name: test-itest-sector_prefer_no_upgrade name: test-itest-sector_prefer_no_upgrade
requires:
- build
suite: itest-sector_prefer_no_upgrade suite: itest-sector_prefer_no_upgrade
target: "./itests/sector_prefer_no_upgrade_test.go" target: "./itests/sector_prefer_no_upgrade_test.go"
- test: - test:
name: test-itest-sector_revert_available name: test-itest-sector_revert_available
requires:
- build
suite: itest-sector_revert_available suite: itest-sector_revert_available
target: "./itests/sector_revert_available_test.go" target: "./itests/sector_revert_available_test.go"
- test: - test:
name: test-itest-sector_terminate name: test-itest-sector_terminate
requires:
- build
suite: itest-sector_terminate suite: itest-sector_terminate
target: "./itests/sector_terminate_test.go" target: "./itests/sector_terminate_test.go"
- test: - test:
name: test-itest-sector_unseal name: test-itest-sector_unseal
requires:
- build
suite: itest-sector_unseal suite: itest-sector_unseal
target: "./itests/sector_unseal_test.go" target: "./itests/sector_unseal_test.go"
- test: - test:
name: test-itest-self_sent_txn name: test-itest-self_sent_txn
requires:
- build
suite: itest-self_sent_txn suite: itest-self_sent_txn
target: "./itests/self_sent_txn_test.go" target: "./itests/self_sent_txn_test.go"
- test: - test:
name: test-itest-splitstore name: test-itest-splitstore
requires:
- build
suite: itest-splitstore suite: itest-splitstore
target: "./itests/splitstore_test.go" target: "./itests/splitstore_test.go"
- test: - test:
name: test-itest-tape name: test-itest-tape
requires:
- build
suite: itest-tape suite: itest-tape
target: "./itests/tape_test.go" target: "./itests/tape_test.go"
- test: - test:
name: test-itest-verifreg name: test-itest-verifreg
requires:
- build
suite: itest-verifreg suite: itest-verifreg
target: "./itests/verifreg_test.go" target: "./itests/verifreg_test.go"
- test: - test:
name: test-itest-wdpost_config name: test-itest-wdpost_config
requires:
- build
suite: itest-wdpost_config suite: itest-wdpost_config
target: "./itests/wdpost_config_test.go" target: "./itests/wdpost_config_test.go"
- test: - test:
name: test-itest-wdpost_dispute name: test-itest-wdpost_dispute
requires:
- build
suite: itest-wdpost_dispute suite: itest-wdpost_dispute
target: "./itests/wdpost_dispute_test.go" target: "./itests/wdpost_dispute_test.go"
- test: - test:
name: test-itest-wdpost_no_miner_storage name: test-itest-wdpost_no_miner_storage
requires:
- build
suite: itest-wdpost_no_miner_storage suite: itest-wdpost_no_miner_storage
target: "./itests/wdpost_no_miner_storage_test.go" target: "./itests/wdpost_no_miner_storage_test.go"
- test: - test:
name: test-itest-wdpost name: test-itest-wdpost
requires:
- build
suite: itest-wdpost suite: itest-wdpost
target: "./itests/wdpost_test.go" target: "./itests/wdpost_test.go"
- test: - test:
name: test-itest-wdpost_worker_config name: test-itest-wdpost_worker_config
requires:
- build
suite: itest-wdpost_worker_config suite: itest-wdpost_worker_config
target: "./itests/wdpost_worker_config_test.go" target: "./itests/wdpost_worker_config_test.go"
executor: golang-2xl
- test: - test:
name: test-itest-worker name: test-itest-worker
requires:
- build
suite: itest-worker suite: itest-worker
target: "./itests/worker_test.go" target: "./itests/worker_test.go"
executor: golang-2xl
- test: - test:
name: test-itest-worker_upgrade name: test-itest-worker_upgrade
requires:
- build
suite: itest-worker_upgrade suite: itest-worker_upgrade
target: "./itests/worker_upgrade_test.go" target: "./itests/worker_upgrade_test.go"
- test: - test:
name: test-unit-cli name: test-unit-cli
requires:
- build
suite: utest-unit-cli suite: utest-unit-cli
target: "./cli/... ./cmd/... ./api/..." target: "./cli/... ./cmd/... ./api/..."
get-params: true
- test: - test:
name: test-unit-node name: test-unit-node
requires:
- build
suite: utest-unit-node suite: utest-unit-node
target: "./node/..." target: "./node/..."
- test: - test:
name: test-unit-rest name: test-unit-rest
requires:
- build
suite: utest-unit-rest suite: utest-unit-rest
target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..." target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..."
executor: golang-2xl
- test: - test:
name: test-unit-storage name: test-unit-storage
requires:
- build
suite: utest-unit-storage suite: utest-unit-storage
target: "./storage/... ./extern/..." target: "./storage/... ./extern/..."
- test: - test:
go-test-flags: "-run=TestMulticoreSDR" go-test-flags: "-run=TestMulticoreSDR"
requires:
- build
suite: multicore-sdr-check suite: multicore-sdr-check
target: "./storage/sealer/ffiwrapper" target: "./storage/sealer/ffiwrapper"
proofs-log-test: "1" proofs-log-test: "1"
- test-conformance: - test-conformance:
requires:
- build
suite: conformance suite: conformance
target: "./conformance" target: "./conformance"
- test-conformance: - test-conformance:
name: test-conformance-bleeding-edge name: test-conformance-bleeding-edge
requires:
- build
suite: conformance-bleeding-edge suite: conformance-bleeding-edge
target: "./conformance" target: "./conformance"
vectors-branch: specs-actors-v7 vectors-branch: specs-actors-v7

179
.circleci/template.yml
View File

@ -5,6 +5,11 @@ orbs:
executors: executors:
golang: golang:
docker:
# Must match GO_VERSION_MIN in project root
- image: cimg/go:1.18.8
resource_class: medium+
golang-2xl:
docker: docker:
# Must match GO_VERSION_MIN in project root # Must match GO_VERSION_MIN in project root
- image: cimg/go:1.18.8 - image: cimg/go:1.18.8
@ -14,7 +19,7 @@ executors:
- image: ubuntu:20.04 - image: ubuntu:20.04
commands: commands:
prepare: build-platform-specific:
parameters: parameters:
linux: linux:
default: true default: true
@ -31,22 +36,13 @@ commands:
steps: steps:
- checkout - checkout
- git_fetch_all_tags - git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- when: - when:
condition: <<parameters.linux>> condition: <<parameters.linux>>
steps: steps:
- run: - install-ubuntu-deps
name: Check Go Version - check-go-version
command: |
v=`go version | { read _ _ v _; echo ${v#go}; }`
if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then
echo "GO_VERSION_MIN file does not match the go version being used."
echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v."
exit 1
fi
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
- run: sudo apt-get install python-is-python3
- when: - when:
condition: <<parameters.darwin>> condition: <<parameters.darwin>>
steps: steps:
@ -67,8 +63,7 @@ commands:
name: Install Rust name: Install Rust
command: | command: |
curl https://sh.rustup.rs -sSf | sh -s -- -y curl https://sh.rustup.rs -sSf | sh -s -- -y
- run: git submodule sync - run: make deps lotus
- run: git submodule update --init
download-params: download-params:
steps: steps:
- restore_cache: - restore_cache:
@ -99,12 +94,43 @@ commands:
name: fetch all tags name: fetch all tags
command: | command: |
git fetch --all git fetch --all
install-ubuntu-deps:
steps:
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
check-go-version:
steps:
- run: |
v=`go version | { read _ _ v _; echo ${v#go}; }`
if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then
echo "GO_VERSION_MIN file does not match the go version being used."
echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v."
exit 1
fi
jobs: jobs:
build:
executor: golang
working_directory: ~/lotus
steps:
- checkout
- git_fetch_all_tags
- run: git submodule sync
- run: git submodule update --init
- install-ubuntu-deps
- check-go-version
- run: make deps lotus
- persist_to_workspace:
root: ~/
paths:
- "lotus"
mod-tidy-check: mod-tidy-check:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare - install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go mod tidy -v - run: go mod tidy -v
- run: - run:
name: Check git diff name: Check git diff
@ -115,13 +141,14 @@ jobs:
test: test:
description: | description: |
Run tests with gotestsum. Run tests with gotestsum.
working_directory: ~/lotus
parameters: &test-params parameters: &test-params
executor: executor:
type: executor type: executor
default: golang default: golang
go-test-flags: go-test-flags:
type: string type: string
default: "-timeout 30m" default: "-timeout 20m"
description: Flags passed to go test. description: Flags passed to go test.
target: target:
type: string type: string
@ -130,20 +157,21 @@ jobs:
proofs-log-test: proofs-log-test:
type: string type: string
default: "0" default: "0"
get-params:
type: boolean
default: false
suite: suite:
type: string type: string
default: unit default: unit
description: Test suite name to report to CircleCI. description: Test suite name to report to CircleCI.
gotestsum-format:
type: string
default: standard-verbose
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
executor: << parameters.executor >> executor: << parameters.executor >>
steps: steps:
- prepare - install-ubuntu-deps
- run: - attach_workspace:
command: make deps lotus at: ~/
no_output_timeout: 30m - when:
condition: << parameters.get-params >>
steps:
- download-params - download-params
- run: - run:
name: go test name: go test
@ -155,12 +183,11 @@ jobs:
mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-reports/<< parameters.suite >>
mkdir -p /tmp/test-artifacts mkdir -p /tmp/test-artifacts
gotestsum \ gotestsum \
--format << parameters.gotestsum-format >> \ --format standard-verbose \
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
--jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
-- \ --packages="<< parameters.target >>" \
<< parameters.go-test-flags >> \ -- << parameters.go-test-flags >>
<< parameters.target >>
no_output_timeout: 30m no_output_timeout: 30m
- store_test_results: - store_test_results:
path: /tmp/test-reports path: /tmp/test-reports
@ -168,6 +195,7 @@ jobs:
path: /tmp/test-artifacts/<< parameters.suite >>.json path: /tmp/test-artifacts/<< parameters.suite >>.json
test-conformance: test-conformance:
working_directory: ~/lotus
description: | description: |
Run tests using a corpus of interoperable test vectors for Filecoin Run tests using a corpus of interoperable test vectors for Filecoin
implementations to test their correctness and compliance with the Filecoin implementations to test their correctness and compliance with the Filecoin
@ -183,10 +211,9 @@ jobs:
submodule is used. submodule is used.
executor: << parameters.executor >> executor: << parameters.executor >>
steps: steps:
- prepare - install-ubuntu-deps
- run: - attach_workspace:
command: make deps lotus at: ~/
no_output_timeout: 30m
- download-params - download-params
- when: - when:
condition: condition:
@ -229,7 +256,7 @@ jobs:
build-linux-amd64: build-linux-amd64:
executor: golang executor: golang
steps: steps:
- prepare - build-platform-specific
- run: make lotus lotus-miner lotus-worker - run: make lotus lotus-miner lotus-worker
- run: - run:
name: check tag and version output match name: check tag and version output match
@ -248,7 +275,7 @@ jobs:
macos: macos:
xcode: "13.4.1" xcode: "13.4.1"
steps: steps:
- prepare: - build-platform-specific:
linux: false linux: false
darwin: true darwin: true
darwin-architecture: amd64 darwin-architecture: amd64
@ -272,7 +299,7 @@ jobs:
resource_class: filecoin-project/self-hosted-m1 resource_class: filecoin-project/self-hosted-m1
steps: steps:
- run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV"
- prepare: - build-platform-specific:
linux: false linux: false
darwin: true darwin: true
darwin-architecture: arm64 darwin-architecture: arm64
@ -380,16 +407,18 @@ jobs:
gofmt: gofmt:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare
- run: - run:
command: "! go fmt ./... 2>&1 | read" command: "! go fmt ./... 2>&1 | read"
gen-check: gen-check:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare - install-ubuntu-deps
- run: make deps - attach_workspace:
at: ~/
- run: go install golang.org/x/tools/cmd/goimports - run: go install golang.org/x/tools/cmd/goimports
- run: go install github.com/hannahhoward/cbor-gen-for - run: go install github.com/hannahhoward/cbor-gen-for
- run: make gen - run: make gen
@ -399,32 +428,29 @@ jobs:
docs-check: docs-check:
executor: golang executor: golang
working_directory: ~/lotus
steps: steps:
- prepare - install-ubuntu-deps
- attach_workspace:
at: ~/
- run: go install golang.org/x/tools/cmd/goimports - run: go install golang.org/x/tools/cmd/goimports
- run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
- run: make deps
- run: make docsgen - run: make docsgen
- run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
- run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
- run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
- run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet - run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet
lint: &lint lint-all:
description: | description: |
Run golangci-lint. Run golangci-lint.
working_directory: ~/lotus
parameters: parameters:
executor: executor:
type: executor type: executor
default: golang default: golang
concurrency:
type: string
default: '2'
description: |
Concurrency used to run linters. Defaults to 2 because NumCPU is not
aware of container CPU limits.
args: args:
type: string type: string
default: '' default: ''
@ -432,17 +458,14 @@ jobs:
Arguments to pass to golangci-lint Arguments to pass to golangci-lint
executor: << parameters.executor >> executor: << parameters.executor >>
steps: steps:
- prepare - install-ubuntu-deps
- run: - attach_workspace:
command: make deps at: ~/
no_output_timeout: 30m
- run: - run:
name: Lint name: Lint
command: | command: |
golangci-lint run -v --timeout 2m \ golangci-lint run -v --timeout 10m \
--concurrency << parameters.concurrency >> << parameters.args >> --concurrency 4 << parameters.args >>
lint-all:
<<: *lint
publish: publish:
description: publish binary artifacts description: publish binary artifacts
@ -462,7 +485,6 @@ jobs:
command: apt update && apt install -y git jq curl sudo command: apt update && apt install -y git jq curl sudo
- checkout - checkout
- git_fetch_all_tags - git_fetch_all_tags
- checkout
- install_ipfs - install_ipfs
- attach_workspace: - attach_workspace:
at: /tmp/workspace at: /tmp/workspace
@ -606,41 +628,64 @@ jobs:
extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>> extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
workflows: workflows:
version: 2.1
ci: ci:
jobs: jobs:
- build
- lint-all: - lint-all:
concurrency: "16" # expend all docker 2xlarge CPUs. requires:
- mod-tidy-check - build
- gofmt - mod-tidy-check:
- gen-check requires:
- docs-check - build
- gofmt:
requires:
- build
- gen-check:
requires:
- build
- docs-check:
requires:
- build
[[- range $file := .ItestFiles -]] [[- range $file := .ItestFiles -]]
[[ with $name := $file | stripSuffix ]] [[ with $name := $file | stripSuffix ]]
- test: - test:
name: test-itest-[[ $name ]] name: test-itest-[[ $name ]]
requires:
- build
suite: itest-[[ $name ]] suite: itest-[[ $name ]]
target: "./itests/[[ $file ]]" target: "./itests/[[ $file ]]"
[[ end ]] [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]]
[[- end -]] executor: golang-2xl
[[- end]]
[[- end ]][[- end]]
[[range $suite, $pkgs := .UnitSuites]] [[- range $suite, $pkgs := .UnitSuites]]
- test: - test:
name: test-[[ $suite ]] name: test-[[ $suite ]]
requires:
- build
suite: utest-[[ $suite ]] suite: utest-[[ $suite ]]
target: "[[ $pkgs ]]" target: "[[ $pkgs ]]"
[[if eq $suite "unit-cli"]]get-params: true[[end]]
[[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]]
[[- end]] [[- end]]
- test: - test:
go-test-flags: "-run=TestMulticoreSDR" go-test-flags: "-run=TestMulticoreSDR"
requires:
- build
suite: multicore-sdr-check suite: multicore-sdr-check
target: "./storage/sealer/ffiwrapper" target: "./storage/sealer/ffiwrapper"
proofs-log-test: "1" proofs-log-test: "1"
- test-conformance: - test-conformance:
requires:
- build
suite: conformance suite: conformance
target: "./conformance" target: "./conformance"
- test-conformance: - test-conformance:
name: test-conformance-bleeding-edge name: test-conformance-bleeding-edge
requires:
- build
suite: conformance-bleeding-edge suite: conformance-bleeding-edge
target: "./conformance" target: "./conformance"
vectors-branch: specs-actors-v7 vectors-branch: specs-actors-v7