From 7c448afda897852ab685376d15bd6faa725b147d Mon Sep 17 00:00:00 2001 From: Geoff Stuart Date: Mon, 19 Dec 2022 01:17:00 -0500 Subject: [PATCH 1/4] make build step --- .circleci/config.yml | 342 +++++++++++++++++++++++++++++++++++------ .circleci/template.yml | 147 ++++++++++++------ 2 files changed, 395 insertions(+), 94 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d690a6d54..bfb947de3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,11 @@ orbs: executors: golang: + docker: + # Must match GO_VERSION_MIN in project root + - image: cimg/go:1.18.8 + resource_class: large + golang-2xl: docker: # Must match GO_VERSION_MIN in project root - image: cimg/go:1.18.8 @@ -14,7 +19,7 @@ executors: - image: ubuntu:20.04 commands: - prepare: + build-platform-specific: parameters: linux: default: true @@ -34,6 +39,7 @@ commands: - when: condition: <> steps: + - prepare - run: name: Check Go Version command: | @@ -43,10 +49,6 @@ commands: echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." exit 1 fi - - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev - - run: sudo apt-get install python-is-python3 - - when: condition: <> steps: @@ -69,6 +71,7 @@ commands: curl https://sh.rustup.rs -sSf | sh -s -- -y - run: git submodule sync - run: git submodule update --init + - run: make deps lotus download-params: steps: - restore_cache: @@ -77,7 +80,7 @@ commands: - 'v26-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - - run: ./lotus fetch-params 2048 + - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache key: 'v26-2k-lotus-params' @@ -99,12 +102,42 @@ commands: name: fetch all tags command: | git fetch --all + prepare: + steps: + - run: sudo apt-get update + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev jobs: + build: + executor: golang + working_directory: ~/lotus + steps: + - checkout + - git_fetch_all_tags + - run: + name: Check Go Version + command: | + v=`go version | { read _ _ v _; echo ${v#go}; }` + if [[ $v != `cat GO_VERSION_MIN` ]]; then + echo "GO_VERSION_MIN file does not match the go version being used." + echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." + exit 1 + fi + - prepare + - run: git submodule sync + - run: git submodule update --init + - run: make deps lotus + - persist_to_workspace: + root: ~/ + paths: + - "lotus" mod-tidy-check: executor: golang + working_directory: ~/lotus steps: - prepare + - attach_workspace: + at: ~/ - run: go mod tidy -v - run: name: Check git diff @@ -115,13 +148,14 @@ jobs: test: description: | Run tests with gotestsum. + working_directory: ~/lotus parameters: &test-params executor: type: executor default: golang go-test-flags: type: string - default: "-timeout 30m" + default: "-timeout 20m" description: Flags passed to go test. target: type: string @@ -130,21 +164,22 @@ jobs: proofs-log-test: type: string default: "0" + get-params: + type: boolean + default: false suite: type: string default: unit description: Test suite name to report to CircleCI. - gotestsum-format: - type: string - default: standard-verbose - description: gotestsum format. https://github.com/gotestyourself/gotestsum#format executor: << parameters.executor >> steps: - prepare - - run: - command: make deps lotus - no_output_timeout: 30m - - download-params + - attach_workspace: + at: ~/ + - when: + condition: << parameters.get-params >> + steps: + - download-params - run: name: go test environment: @@ -155,7 +190,7 @@ jobs: mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ - --format << parameters.gotestsum-format >> \ + --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ -- \ @@ -168,6 +203,7 @@ jobs: path: /tmp/test-artifacts/<< parameters.suite >>.json test-conformance: + working_directory: ~/lotus description: | Run tests using a corpus of interoperable test vectors for Filecoin implementations to test their correctness and compliance with the Filecoin @@ -184,9 +220,8 @@ jobs: executor: << parameters.executor >> steps: - prepare - - run: - command: make deps lotus - no_output_timeout: 30m + - attach_workspace: + at: ~/ - download-params - when: condition: @@ -229,7 +264,7 @@ jobs: build-linux-amd64: executor: golang steps: - - prepare + - build-platform-specific - run: make lotus lotus-miner lotus-worker - run: name: check tag and version output match @@ -248,7 +283,7 @@ jobs: macos: xcode: "13.4.1" steps: - - prepare: + - build-platform-specific: linux: false darwin: true darwin-architecture: amd64 @@ -272,7 +307,7 @@ jobs: resource_class: filecoin-project/self-hosted-m1 steps: - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - - prepare: + - build-platform-specific: linux: false darwin: true darwin-architecture: arm64 @@ -380,16 +415,18 @@ jobs: gofmt: executor: golang + working_directory: ~/lotus steps: - - prepare - run: command: "! go fmt ./... 2>&1 | read" gen-check: executor: golang + working_directory: ~/lotus steps: - prepare - - run: make deps + - attach_workspace: + at: ~/ - run: go install golang.org/x/tools/cmd/goimports - run: go install github.com/hannahhoward/cbor-gen-for - run: make gen @@ -399,32 +436,29 @@ jobs: docs-check: executor: golang + working_directory: ~/lotus steps: - prepare + - attach_workspace: + at: ~/ - run: go install golang.org/x/tools/cmd/goimports - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - - run: make deps - run: make docsgen - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet - lint: &lint + lint-all: description: | Run golangci-lint. + working_directory: ~/lotus parameters: executor: type: executor default: golang - concurrency: - type: string - default: '2' - description: | - Concurrency used to run linters. Defaults to 2 because NumCPU is not - aware of container CPU limits. args: type: string default: '' @@ -433,16 +467,13 @@ jobs: executor: << parameters.executor >> steps: - prepare - - run: - command: make deps - no_output_timeout: 30m + - attach_workspace: + at: ~/ - run: name: Lint command: | - golangci-lint run -v --timeout 2m \ - --concurrency << parameters.concurrency >> << parameters.args >> - lint-all: - <<: *lint + golangci-lint run -v --timeout 10m \ + --concurrency 4 << parameters.args >> publish: description: publish binary artifacts @@ -462,7 +493,6 @@ jobs: command: apt update && apt install -y git jq curl sudo - checkout - git_fetch_all_tags - - checkout - install_ipfs - attach_workspace: at: /tmp/workspace @@ -606,356 +636,576 @@ jobs: extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> workflows: - version: 2.1 ci: jobs: + - build - lint-all: - concurrency: "16" # expend all docker 2xlarge CPUs. - - mod-tidy-check - - gofmt - - gen-check - - docs-check + requires: + - build + - mod-tidy-check: + requires: + - build + - gofmt: + requires: + - build + - gen-check: + requires: + - build + - docs-check: + requires: + - build - test: name: test-itest-api + requires: + - build suite: itest-api target: "./itests/api_test.go" + - test: name: test-itest-batch_deal + requires: + - build suite: itest-batch_deal target: "./itests/batch_deal_test.go" + - test: name: test-itest-ccupgrade + requires: + - build suite: itest-ccupgrade target: "./itests/ccupgrade_test.go" + - test: name: test-itest-cli + requires: + - build suite: itest-cli target: "./itests/cli_test.go" + - test: name: test-itest-deadlines + requires: + - build suite: itest-deadlines target: "./itests/deadlines_test.go" + - test: name: test-itest-deals_512mb + requires: + - build suite: itest-deals_512mb target: "./itests/deals_512mb_test.go" + - test: name: test-itest-deals_anycid + requires: + - build suite: itest-deals_anycid target: "./itests/deals_anycid_test.go" + - test: name: test-itest-deals_concurrent + requires: + - build suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" + executor: golang-2xl - test: name: test-itest-deals_invalid_utf8_label + requires: + - build suite: itest-deals_invalid_utf8_label target: "./itests/deals_invalid_utf8_label_test.go" + - test: name: test-itest-deals_max_staging_deals + requires: + - build suite: itest-deals_max_staging_deals target: "./itests/deals_max_staging_deals_test.go" + - test: name: test-itest-deals_offline + requires: + - build suite: itest-deals_offline target: "./itests/deals_offline_test.go" + - test: name: test-itest-deals_padding + requires: + - build suite: itest-deals_padding target: "./itests/deals_padding_test.go" + - test: name: test-itest-deals_partial_retrieval_dm-level + requires: + - build suite: itest-deals_partial_retrieval_dm-level target: "./itests/deals_partial_retrieval_dm-level_test.go" + - test: name: test-itest-deals_partial_retrieval + requires: + - build suite: itest-deals_partial_retrieval target: "./itests/deals_partial_retrieval_test.go" + - test: name: test-itest-deals_power + requires: + - build suite: itest-deals_power target: "./itests/deals_power_test.go" + - test: name: test-itest-deals_pricing + requires: + - build suite: itest-deals_pricing target: "./itests/deals_pricing_test.go" + - test: name: test-itest-deals_publish + requires: + - build suite: itest-deals_publish target: "./itests/deals_publish_test.go" + - test: name: test-itest-deals_remote_retrieval + requires: + - build suite: itest-deals_remote_retrieval target: "./itests/deals_remote_retrieval_test.go" + - test: name: test-itest-deals_retry_deal_no_funds + requires: + - build suite: itest-deals_retry_deal_no_funds target: "./itests/deals_retry_deal_no_funds_test.go" + - test: name: test-itest-deals + requires: + - build suite: itest-deals target: "./itests/deals_test.go" + - test: name: test-itest-dup_mpool_messages + requires: + - build suite: itest-dup_mpool_messages target: "./itests/dup_mpool_messages_test.go" + - test: name: test-itest-gas_estimation + requires: + - build suite: itest-gas_estimation target: "./itests/gas_estimation_test.go" + - test: name: test-itest-gateway + requires: + - build suite: itest-gateway target: "./itests/gateway_test.go" + - test: name: test-itest-get_messages_in_ts + requires: + - build suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" + - test: name: test-itest-lite_migration + requires: + - build suite: itest-lite_migration target: "./itests/lite_migration_test.go" + - test: name: test-itest-lookup_robust_address + requires: + - build suite: itest-lookup_robust_address target: "./itests/lookup_robust_address_test.go" + - test: name: test-itest-mempool + requires: + - build suite: itest-mempool target: "./itests/mempool_test.go" + - test: name: test-itest-migration_nv17 + requires: + - build suite: itest-migration_nv17 target: "./itests/migration_nv17_test.go" + - test: name: test-itest-mpool_msg_uuid + requires: + - build suite: itest-mpool_msg_uuid target: "./itests/mpool_msg_uuid_test.go" + - test: name: test-itest-mpool_push_with_uuid + requires: + - build suite: itest-mpool_push_with_uuid target: "./itests/mpool_push_with_uuid_test.go" + - test: name: test-itest-multisig + requires: + - build suite: itest-multisig target: "./itests/multisig_test.go" + - test: name: test-itest-net + requires: + - build suite: itest-net target: "./itests/net_test.go" + - test: name: test-itest-nonce + requires: + - build suite: itest-nonce target: "./itests/nonce_test.go" + - test: name: test-itest-path_detach_redeclare + requires: + - build suite: itest-path_detach_redeclare target: "./itests/path_detach_redeclare_test.go" + - test: name: test-itest-path_type_filters + requires: + - build suite: itest-path_type_filters target: "./itests/path_type_filters_test.go" + - test: name: test-itest-paych_api + requires: + - build suite: itest-paych_api target: "./itests/paych_api_test.go" + - test: name: test-itest-paych_cli + requires: + - build suite: itest-paych_cli target: "./itests/paych_cli_test.go" + - test: name: test-itest-pending_deal_allocation + requires: + - build suite: itest-pending_deal_allocation target: "./itests/pending_deal_allocation_test.go" + - test: name: test-itest-raft_messagesigner + requires: + - build suite: itest-raft_messagesigner target: "./itests/raft_messagesigner_test.go" + - test: name: test-itest-remove_verifreg_datacap + requires: + - build suite: itest-remove_verifreg_datacap target: "./itests/remove_verifreg_datacap_test.go" + - test: name: test-itest-sdr_upgrade + requires: + - build suite: itest-sdr_upgrade target: "./itests/sdr_upgrade_test.go" + - test: name: test-itest-sector_finalize_early + requires: + - build suite: itest-sector_finalize_early target: "./itests/sector_finalize_early_test.go" + - test: name: test-itest-sector_import_full + requires: + - build suite: itest-sector_import_full target: "./itests/sector_import_full_test.go" + - test: name: test-itest-sector_import_simple + requires: + - build suite: itest-sector_import_simple target: "./itests/sector_import_simple_test.go" + - test: name: test-itest-sector_make_cc_avail + requires: + - build suite: itest-sector_make_cc_avail target: "./itests/sector_make_cc_avail_test.go" + - test: name: test-itest-sector_miner_collateral + requires: + - build suite: itest-sector_miner_collateral target: "./itests/sector_miner_collateral_test.go" + - test: name: test-itest-sector_numassign + requires: + - build suite: itest-sector_numassign target: "./itests/sector_numassign_test.go" + - test: name: test-itest-sector_pledge + requires: + - build suite: itest-sector_pledge target: "./itests/sector_pledge_test.go" + - test: name: test-itest-sector_prefer_no_upgrade + requires: + - build suite: itest-sector_prefer_no_upgrade target: "./itests/sector_prefer_no_upgrade_test.go" + - test: name: test-itest-sector_revert_available + requires: + - build suite: itest-sector_revert_available target: "./itests/sector_revert_available_test.go" + - test: name: test-itest-sector_terminate + requires: + - build suite: itest-sector_terminate target: "./itests/sector_terminate_test.go" + - test: name: test-itest-sector_unseal + requires: + - build suite: itest-sector_unseal target: "./itests/sector_unseal_test.go" + - test: name: test-itest-self_sent_txn + requires: + - build suite: itest-self_sent_txn target: "./itests/self_sent_txn_test.go" + - test: name: test-itest-splitstore + requires: + - build suite: itest-splitstore target: "./itests/splitstore_test.go" + - test: name: test-itest-tape + requires: + - build suite: itest-tape target: "./itests/tape_test.go" + - test: name: test-itest-verifreg + requires: + - build suite: itest-verifreg target: "./itests/verifreg_test.go" + - test: name: test-itest-wdpost_config + requires: + - build suite: itest-wdpost_config target: "./itests/wdpost_config_test.go" + - test: name: test-itest-wdpost_dispute + requires: + - build suite: itest-wdpost_dispute target: "./itests/wdpost_dispute_test.go" + - test: name: test-itest-wdpost_no_miner_storage + requires: + - build suite: itest-wdpost_no_miner_storage target: "./itests/wdpost_no_miner_storage_test.go" + - test: name: test-itest-wdpost + requires: + - build suite: itest-wdpost target: "./itests/wdpost_test.go" + - test: name: test-itest-wdpost_worker_config + requires: + - build suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" + executor: golang-2xl - test: name: test-itest-worker + requires: + - build suite: itest-worker target: "./itests/worker_test.go" + executor: golang-2xl - test: name: test-itest-worker_upgrade + requires: + - build suite: itest-worker_upgrade target: "./itests/worker_upgrade_test.go" + - test: name: test-unit-cli + requires: + - build suite: utest-unit-cli target: "./cli/... ./cmd/... ./api/..." + get-params: true + - test: name: test-unit-node + requires: + - build suite: utest-unit-node target: "./node/..." + + - test: name: test-unit-rest + requires: + - build suite: utest-unit-rest target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..." + + executor: golang-2xl - test: name: test-unit-storage + requires: + - build suite: utest-unit-storage target: "./storage/... ./extern/..." + + - test: go-test-flags: "-run=TestMulticoreSDR" + requires: + - build suite: multicore-sdr-check target: "./storage/sealer/ffiwrapper" proofs-log-test: "1" - test-conformance: + requires: + - build suite: conformance target: "./conformance" - test-conformance: name: test-conformance-bleeding-edge + requires: + - build suite: conformance-bleeding-edge target: "./conformance" vectors-branch: specs-actors-v7 diff --git a/.circleci/template.yml b/.circleci/template.yml index 6906f13f4..ba71ccc73 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -5,6 +5,11 @@ orbs: executors: golang: + docker: + # Must match GO_VERSION_MIN in project root + - image: cimg/go:1.18.8 + resource_class: large + golang-2xl: docker: # Must match GO_VERSION_MIN in project root - image: cimg/go:1.18.8 @@ -14,7 +19,7 @@ executors: - image: ubuntu:20.04 commands: - prepare: + build-platform-specific: parameters: linux: default: true @@ -34,6 +39,7 @@ commands: - when: condition: <> steps: + - prepare - run: name: Check Go Version command: | @@ -43,10 +49,6 @@ commands: echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." exit 1 fi - - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev - - run: sudo apt-get install python-is-python3 - - when: condition: <> steps: @@ -69,6 +71,7 @@ commands: curl https://sh.rustup.rs -sSf | sh -s -- -y - run: git submodule sync - run: git submodule update --init + - run: make deps lotus download-params: steps: - restore_cache: @@ -77,7 +80,7 @@ commands: - 'v26-2k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - - run: ./lotus fetch-params 2048 + - run: ./lotus fetch-params 2048 - save_cache: name: Save parameters cache key: 'v26-2k-lotus-params' @@ -99,12 +102,42 @@ commands: name: fetch all tags command: | git fetch --all + prepare: + steps: + - run: sudo apt-get update + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev jobs: + build: + executor: golang + working_directory: ~/lotus + steps: + - checkout + - git_fetch_all_tags + - run: + name: Check Go Version + command: | + v=`go version | { read _ _ v _; echo ${v#go}; }` + if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then + echo "GO_VERSION_MIN file does not match the go version being used." + echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." + exit 1 + fi + - prepare + - run: git submodule sync + - run: git submodule update --init + - run: make deps lotus + - persist_to_workspace: + root: ~/ + paths: + - "lotus" mod-tidy-check: executor: golang + working_directory: ~/lotus steps: - prepare + - attach_workspace: + at: ~/ - run: go mod tidy -v - run: name: Check git diff @@ -115,13 +148,14 @@ jobs: test: description: | Run tests with gotestsum. + working_directory: ~/lotus parameters: &test-params executor: type: executor default: golang go-test-flags: type: string - default: "-timeout 30m" + default: "-timeout 20m" description: Flags passed to go test. target: type: string @@ -130,21 +164,22 @@ jobs: proofs-log-test: type: string default: "0" + get-params: + type: boolean + default: false suite: type: string default: unit description: Test suite name to report to CircleCI. - gotestsum-format: - type: string - default: standard-verbose - description: gotestsum format. https://github.com/gotestyourself/gotestsum#format executor: << parameters.executor >> steps: - prepare - - run: - command: make deps lotus - no_output_timeout: 30m - - download-params + - attach_workspace: + at: ~/ + - when: + condition: << parameters.get-params >> + steps: + - download-params - run: name: go test environment: @@ -155,7 +190,7 @@ jobs: mkdir -p /tmp/test-reports/<< parameters.suite >> mkdir -p /tmp/test-artifacts gotestsum \ - --format << parameters.gotestsum-format >> \ + --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ -- \ @@ -168,6 +203,7 @@ jobs: path: /tmp/test-artifacts/<< parameters.suite >>.json test-conformance: + working_directory: ~/lotus description: | Run tests using a corpus of interoperable test vectors for Filecoin implementations to test their correctness and compliance with the Filecoin @@ -184,9 +220,8 @@ jobs: executor: << parameters.executor >> steps: - prepare - - run: - command: make deps lotus - no_output_timeout: 30m + - attach_workspace: + at: ~/ - download-params - when: condition: @@ -229,7 +264,7 @@ jobs: build-linux-amd64: executor: golang steps: - - prepare + - build-platform-specific - run: make lotus lotus-miner lotus-worker - run: name: check tag and version output match @@ -248,7 +283,7 @@ jobs: macos: xcode: "13.4.1" steps: - - prepare: + - build-platform-specific: linux: false darwin: true darwin-architecture: amd64 @@ -272,7 +307,7 @@ jobs: resource_class: filecoin-project/self-hosted-m1 steps: - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - - prepare: + - build-platform-specific: linux: false darwin: true darwin-architecture: arm64 @@ -380,16 +415,18 @@ jobs: gofmt: executor: golang + working_directory: ~/lotus steps: - - prepare - run: command: "! go fmt ./... 2>&1 | read" gen-check: executor: golang + working_directory: ~/lotus steps: - prepare - - run: make deps + - attach_workspace: + at: ~/ - run: go install golang.org/x/tools/cmd/goimports - run: go install github.com/hannahhoward/cbor-gen-for - run: make gen @@ -399,32 +436,29 @@ jobs: docs-check: executor: golang + working_directory: ~/lotus steps: - prepare + - attach_workspace: + at: ~/ - run: go install golang.org/x/tools/cmd/goimports - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - - run: make deps - run: make docsgen - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - run: diff ../pre-openrpc-full ../post-openrpc-full && diff ../pre-openrpc-miner ../post-openrpc-miner && diff ../pre-openrpc-worker ../post-openrpc-worker && git --no-pager diff && git --no-pager diff --quiet - lint: &lint + lint-all: description: | Run golangci-lint. + working_directory: ~/lotus parameters: executor: type: executor default: golang - concurrency: - type: string - default: '2' - description: | - Concurrency used to run linters. Defaults to 2 because NumCPU is not - aware of container CPU limits. args: type: string default: '' @@ -433,16 +467,13 @@ jobs: executor: << parameters.executor >> steps: - prepare - - run: - command: make deps - no_output_timeout: 30m + - attach_workspace: + at: ~/ - run: name: Lint command: | - golangci-lint run -v --timeout 2m \ - --concurrency << parameters.concurrency >> << parameters.args >> - lint-all: - <<: *lint + golangci-lint run -v --timeout 10m \ + --concurrency 4 << parameters.args >> publish: description: publish binary artifacts @@ -462,7 +493,6 @@ jobs: command: apt update && apt install -y git jq curl sudo - checkout - git_fetch_all_tags - - checkout - install_ipfs - attach_workspace: at: /tmp/workspace @@ -606,41 +636,62 @@ jobs: extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> workflows: - version: 2.1 ci: jobs: + - build - lint-all: - concurrency: "16" # expend all docker 2xlarge CPUs. - - mod-tidy-check - - gofmt - - gen-check - - docs-check + requires: + - build + - mod-tidy-check: + requires: + - build + - gofmt: + requires: + - build + - gen-check: + requires: + - build + - docs-check: + requires: + - build [[- range $file := .ItestFiles -]] [[ with $name := $file | stripSuffix ]] - test: name: test-itest-[[ $name ]] + requires: + - build suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" - [[ end ]] - [[- end -]] + [[if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]]executor: golang-2xl[[end]] + [[ end ]][[- end -]] [[range $suite, $pkgs := .UnitSuites]] - test: name: test-[[ $suite ]] + requires: + - build suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" + [[if eq $suite "unit-cli"]]get-params: true[[end]] + [[if eq $suite "unit-rest"]]executor: golang-2xl[[end]] [[- end]] - test: go-test-flags: "-run=TestMulticoreSDR" + requires: + - build suite: multicore-sdr-check target: "./storage/sealer/ffiwrapper" proofs-log-test: "1" - test-conformance: + requires: + - build suite: conformance target: "./conformance" - test-conformance: name: test-conformance-bleeding-edge + requires: + - build suite: conformance-bleeding-edge target: "./conformance" vectors-branch: specs-actors-v7 From f9404aa55e8d62578371e99e0818c06059b1e477 Mon Sep 17 00:00:00 2001 From: Geoff Stuart Date: Mon, 19 Dec 2022 16:18:07 -0500 Subject: [PATCH 2/4] Try medium+ --- .circleci/config.yml | 2 +- .circleci/template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bfb947de3..11b21ae37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ executors: docker: # Must match GO_VERSION_MIN in project root - image: cimg/go:1.18.8 - resource_class: large + resource_class: medium+ golang-2xl: docker: # Must match GO_VERSION_MIN in project root diff --git a/.circleci/template.yml b/.circleci/template.yml index ba71ccc73..f135c7cb8 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -8,7 +8,7 @@ executors: docker: # Must match GO_VERSION_MIN in project root - image: cimg/go:1.18.8 - resource_class: large + resource_class: medium+ golang-2xl: docker: # Must match GO_VERSION_MIN in project root From 3bae3a3bea58372e415c5b23cd4b28093614a16e Mon Sep 17 00:00:00 2001 From: Geoff Stuart Date: Tue, 20 Dec 2022 10:03:22 -0500 Subject: [PATCH 3/4] Reorg some jobs --- .circleci/config.yml | 178 +++++------------------------------------ .circleci/template.yml | 61 +++++++------- 2 files changed, 50 insertions(+), 189 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11b21ae37..964c86b05 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,19 +36,13 @@ commands: steps: - checkout - git_fetch_all_tags + - run: git submodule sync + - run: git submodule update --init - when: condition: <> steps: - - prepare - - run: - name: Check Go Version - command: | - v=`go version | { read _ _ v _; echo ${v#go}; }` - if [[ $v != `cat GO_VERSION_MIN` ]]; then - echo "GO_VERSION_MIN file does not match the go version being used." - echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." - exit 1 - fi + - install-ubuntu-deps + - check-go-version - when: condition: <> steps: @@ -69,8 +63,6 @@ commands: name: Install Rust command: | curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: git submodule sync - - run: git submodule update --init - run: make deps lotus download-params: steps: @@ -102,10 +94,19 @@ commands: name: fetch all tags command: | git fetch --all - prepare: + install-ubuntu-deps: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + check-go-version: + steps: + - run: | + v=`go version | { read _ _ v _; echo ${v#go}; }` + if [[ $v != `cat GO_VERSION_MIN` ]]; then + echo "GO_VERSION_MIN file does not match the go version being used." + echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." + exit 1 + fi jobs: build: @@ -114,18 +115,10 @@ jobs: steps: - checkout - git_fetch_all_tags - - run: - name: Check Go Version - command: | - v=`go version | { read _ _ v _; echo ${v#go}; }` - if [[ $v != `cat GO_VERSION_MIN` ]]; then - echo "GO_VERSION_MIN file does not match the go version being used." - echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." - exit 1 - fi - - prepare - run: git submodule sync - run: git submodule update --init + - install-ubuntu-deps + - check-go-version - run: make deps lotus - persist_to_workspace: root: ~/ @@ -135,7 +128,7 @@ jobs: executor: golang working_directory: ~/lotus steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: go mod tidy -v @@ -173,7 +166,7 @@ jobs: description: Test suite name to report to CircleCI. executor: << parameters.executor >> steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - when: @@ -219,7 +212,7 @@ jobs: submodule is used. executor: << parameters.executor >> steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - download-params @@ -424,7 +417,7 @@ jobs: executor: golang working_directory: ~/lotus steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: go install golang.org/x/tools/cmd/goimports @@ -438,7 +431,7 @@ jobs: executor: golang working_directory: ~/lotus steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: go install golang.org/x/tools/cmd/goimports @@ -466,7 +459,7 @@ jobs: Arguments to pass to golangci-lint executor: << parameters.executor >> steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: @@ -660,56 +653,42 @@ workflows: - build suite: itest-api target: "./itests/api_test.go" - - - test: name: test-itest-batch_deal requires: - build suite: itest-batch_deal target: "./itests/batch_deal_test.go" - - - test: name: test-itest-ccupgrade requires: - build suite: itest-ccupgrade target: "./itests/ccupgrade_test.go" - - - test: name: test-itest-cli requires: - build suite: itest-cli target: "./itests/cli_test.go" - - - test: name: test-itest-deadlines requires: - build suite: itest-deadlines target: "./itests/deadlines_test.go" - - - test: name: test-itest-deals_512mb requires: - build suite: itest-deals_512mb target: "./itests/deals_512mb_test.go" - - - test: name: test-itest-deals_anycid requires: - build suite: itest-deals_anycid target: "./itests/deals_anycid_test.go" - - - test: name: test-itest-deals_concurrent requires: @@ -717,423 +696,318 @@ workflows: suite: itest-deals_concurrent target: "./itests/deals_concurrent_test.go" executor: golang-2xl - - test: name: test-itest-deals_invalid_utf8_label requires: - build suite: itest-deals_invalid_utf8_label target: "./itests/deals_invalid_utf8_label_test.go" - - - test: name: test-itest-deals_max_staging_deals requires: - build suite: itest-deals_max_staging_deals target: "./itests/deals_max_staging_deals_test.go" - - - test: name: test-itest-deals_offline requires: - build suite: itest-deals_offline target: "./itests/deals_offline_test.go" - - - test: name: test-itest-deals_padding requires: - build suite: itest-deals_padding target: "./itests/deals_padding_test.go" - - - test: name: test-itest-deals_partial_retrieval_dm-level requires: - build suite: itest-deals_partial_retrieval_dm-level target: "./itests/deals_partial_retrieval_dm-level_test.go" - - - test: name: test-itest-deals_partial_retrieval requires: - build suite: itest-deals_partial_retrieval target: "./itests/deals_partial_retrieval_test.go" - - - test: name: test-itest-deals_power requires: - build suite: itest-deals_power target: "./itests/deals_power_test.go" - - - test: name: test-itest-deals_pricing requires: - build suite: itest-deals_pricing target: "./itests/deals_pricing_test.go" - - - test: name: test-itest-deals_publish requires: - build suite: itest-deals_publish target: "./itests/deals_publish_test.go" - - - test: name: test-itest-deals_remote_retrieval requires: - build suite: itest-deals_remote_retrieval target: "./itests/deals_remote_retrieval_test.go" - - - test: name: test-itest-deals_retry_deal_no_funds requires: - build suite: itest-deals_retry_deal_no_funds target: "./itests/deals_retry_deal_no_funds_test.go" - - - test: name: test-itest-deals requires: - build suite: itest-deals target: "./itests/deals_test.go" - - - test: name: test-itest-dup_mpool_messages requires: - build suite: itest-dup_mpool_messages target: "./itests/dup_mpool_messages_test.go" - - - test: name: test-itest-gas_estimation requires: - build suite: itest-gas_estimation target: "./itests/gas_estimation_test.go" - - - test: name: test-itest-gateway requires: - build suite: itest-gateway target: "./itests/gateway_test.go" - - - test: name: test-itest-get_messages_in_ts requires: - build suite: itest-get_messages_in_ts target: "./itests/get_messages_in_ts_test.go" - - - test: name: test-itest-lite_migration requires: - build suite: itest-lite_migration target: "./itests/lite_migration_test.go" - - - test: name: test-itest-lookup_robust_address requires: - build suite: itest-lookup_robust_address target: "./itests/lookup_robust_address_test.go" - - - test: name: test-itest-mempool requires: - build suite: itest-mempool target: "./itests/mempool_test.go" - - - test: name: test-itest-migration_nv17 requires: - build suite: itest-migration_nv17 target: "./itests/migration_nv17_test.go" - - - test: name: test-itest-mpool_msg_uuid requires: - build suite: itest-mpool_msg_uuid target: "./itests/mpool_msg_uuid_test.go" - - - test: name: test-itest-mpool_push_with_uuid requires: - build suite: itest-mpool_push_with_uuid target: "./itests/mpool_push_with_uuid_test.go" - - - test: name: test-itest-multisig requires: - build suite: itest-multisig target: "./itests/multisig_test.go" - - - test: name: test-itest-net requires: - build suite: itest-net target: "./itests/net_test.go" - - - test: name: test-itest-nonce requires: - build suite: itest-nonce target: "./itests/nonce_test.go" - - - test: name: test-itest-path_detach_redeclare requires: - build suite: itest-path_detach_redeclare target: "./itests/path_detach_redeclare_test.go" - - - test: name: test-itest-path_type_filters requires: - build suite: itest-path_type_filters target: "./itests/path_type_filters_test.go" - - - test: name: test-itest-paych_api requires: - build suite: itest-paych_api target: "./itests/paych_api_test.go" - - - test: name: test-itest-paych_cli requires: - build suite: itest-paych_cli target: "./itests/paych_cli_test.go" - - - test: name: test-itest-pending_deal_allocation requires: - build suite: itest-pending_deal_allocation target: "./itests/pending_deal_allocation_test.go" - - - test: name: test-itest-raft_messagesigner requires: - build suite: itest-raft_messagesigner target: "./itests/raft_messagesigner_test.go" - - - test: name: test-itest-remove_verifreg_datacap requires: - build suite: itest-remove_verifreg_datacap target: "./itests/remove_verifreg_datacap_test.go" - - - test: name: test-itest-sdr_upgrade requires: - build suite: itest-sdr_upgrade target: "./itests/sdr_upgrade_test.go" - - - test: name: test-itest-sector_finalize_early requires: - build suite: itest-sector_finalize_early target: "./itests/sector_finalize_early_test.go" - - - test: name: test-itest-sector_import_full requires: - build suite: itest-sector_import_full target: "./itests/sector_import_full_test.go" - - - test: name: test-itest-sector_import_simple requires: - build suite: itest-sector_import_simple target: "./itests/sector_import_simple_test.go" - - - test: name: test-itest-sector_make_cc_avail requires: - build suite: itest-sector_make_cc_avail target: "./itests/sector_make_cc_avail_test.go" - - - test: name: test-itest-sector_miner_collateral requires: - build suite: itest-sector_miner_collateral target: "./itests/sector_miner_collateral_test.go" - - - test: name: test-itest-sector_numassign requires: - build suite: itest-sector_numassign target: "./itests/sector_numassign_test.go" - - - test: name: test-itest-sector_pledge requires: - build suite: itest-sector_pledge target: "./itests/sector_pledge_test.go" - - - test: name: test-itest-sector_prefer_no_upgrade requires: - build suite: itest-sector_prefer_no_upgrade target: "./itests/sector_prefer_no_upgrade_test.go" - - - test: name: test-itest-sector_revert_available requires: - build suite: itest-sector_revert_available target: "./itests/sector_revert_available_test.go" - - - test: name: test-itest-sector_terminate requires: - build suite: itest-sector_terminate target: "./itests/sector_terminate_test.go" - - - test: name: test-itest-sector_unseal requires: - build suite: itest-sector_unseal target: "./itests/sector_unseal_test.go" - - - test: name: test-itest-self_sent_txn requires: - build suite: itest-self_sent_txn target: "./itests/self_sent_txn_test.go" - - - test: name: test-itest-splitstore requires: - build suite: itest-splitstore target: "./itests/splitstore_test.go" - - - test: name: test-itest-tape requires: - build suite: itest-tape target: "./itests/tape_test.go" - - - test: name: test-itest-verifreg requires: - build suite: itest-verifreg target: "./itests/verifreg_test.go" - - - test: name: test-itest-wdpost_config requires: - build suite: itest-wdpost_config target: "./itests/wdpost_config_test.go" - - - test: name: test-itest-wdpost_dispute requires: - build suite: itest-wdpost_dispute target: "./itests/wdpost_dispute_test.go" - - - test: name: test-itest-wdpost_no_miner_storage requires: - build suite: itest-wdpost_no_miner_storage target: "./itests/wdpost_no_miner_storage_test.go" - - - test: name: test-itest-wdpost requires: - build suite: itest-wdpost target: "./itests/wdpost_test.go" - - - test: name: test-itest-wdpost_worker_config requires: @@ -1141,7 +1015,6 @@ workflows: suite: itest-wdpost_worker_config target: "./itests/wdpost_worker_config_test.go" executor: golang-2xl - - test: name: test-itest-worker requires: @@ -1149,15 +1022,12 @@ workflows: suite: itest-worker target: "./itests/worker_test.go" executor: golang-2xl - - test: name: test-itest-worker_upgrade requires: - build suite: itest-worker_upgrade target: "./itests/worker_upgrade_test.go" - - - test: name: test-unit-cli requires: @@ -1165,7 +1035,6 @@ workflows: suite: utest-unit-cli target: "./cli/... ./cmd/... ./api/..." get-params: true - - test: name: test-unit-node requires: @@ -1173,14 +1042,12 @@ workflows: suite: utest-unit-node target: "./node/..." - - test: name: test-unit-rest requires: - build suite: utest-unit-rest target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..." - executor: golang-2xl - test: name: test-unit-storage @@ -1189,7 +1056,6 @@ workflows: suite: utest-unit-storage target: "./storage/... ./extern/..." - - test: go-test-flags: "-run=TestMulticoreSDR" requires: diff --git a/.circleci/template.yml b/.circleci/template.yml index f135c7cb8..6bb688d27 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -36,19 +36,13 @@ commands: steps: - checkout - git_fetch_all_tags + - run: git submodule sync + - run: git submodule update --init - when: condition: <> steps: - - prepare - - run: - name: Check Go Version - command: | - v=`go version | { read _ _ v _; echo ${v#go}; }` - if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then - echo "GO_VERSION_MIN file does not match the go version being used." - echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." - exit 1 - fi + - install-ubuntu-deps + - check-go-version - when: condition: <> steps: @@ -69,8 +63,6 @@ commands: name: Install Rust command: | curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: git submodule sync - - run: git submodule update --init - run: make deps lotus download-params: steps: @@ -102,10 +94,19 @@ commands: name: fetch all tags command: | git fetch --all - prepare: + install-ubuntu-deps: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + check-go-version: + steps: + - run: | + v=`go version | { read _ _ v _; echo ${v#go}; }` + if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then + echo "GO_VERSION_MIN file does not match the go version being used." + echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." + exit 1 + fi jobs: build: @@ -114,18 +115,10 @@ jobs: steps: - checkout - git_fetch_all_tags - - run: - name: Check Go Version - command: | - v=`go version | { read _ _ v _; echo ${v#go}; }` - if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then - echo "GO_VERSION_MIN file does not match the go version being used." - echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." - exit 1 - fi - - prepare - run: git submodule sync - run: git submodule update --init + - install-ubuntu-deps + - check-go-version - run: make deps lotus - persist_to_workspace: root: ~/ @@ -135,7 +128,7 @@ jobs: executor: golang working_directory: ~/lotus steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: go mod tidy -v @@ -173,7 +166,7 @@ jobs: description: Test suite name to report to CircleCI. executor: << parameters.executor >> steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - when: @@ -219,7 +212,7 @@ jobs: submodule is used. executor: << parameters.executor >> steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - download-params @@ -424,7 +417,7 @@ jobs: executor: golang working_directory: ~/lotus steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: go install golang.org/x/tools/cmd/goimports @@ -438,7 +431,7 @@ jobs: executor: golang working_directory: ~/lotus steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: go install golang.org/x/tools/cmd/goimports @@ -466,7 +459,7 @@ jobs: Arguments to pass to golangci-lint executor: << parameters.executor >> steps: - - prepare + - install-ubuntu-deps - attach_workspace: at: ~/ - run: @@ -663,10 +656,12 @@ workflows: - build suite: itest-[[ $name ]] target: "./itests/[[ $file ]]" - [[if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]]executor: golang-2xl[[end]] - [[ end ]][[- end -]] + [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config")]] + executor: golang-2xl + [[- end]] + [[- end ]][[- end]] - [[range $suite, $pkgs := .UnitSuites]] + [[- range $suite, $pkgs := .UnitSuites]] - test: name: test-[[ $suite ]] requires: @@ -674,7 +669,7 @@ workflows: suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" [[if eq $suite "unit-cli"]]get-params: true[[end]] - [[if eq $suite "unit-rest"]]executor: golang-2xl[[end]] + [[- if eq $suite "unit-rest"]]executor: golang-2xl[[end]] [[- end]] - test: go-test-flags: "-run=TestMulticoreSDR" From fd1e0ad7a3566dc69b4cfcb00f8c97778292ab89 Mon Sep 17 00:00:00 2001 From: Geoff Stuart Date: Tue, 20 Dec 2022 12:45:06 -0500 Subject: [PATCH 4/4] rerun fails --- .circleci/config.yml | 5 ++--- .circleci/template.yml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 964c86b05..ce1561e66 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -186,9 +186,8 @@ jobs: --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ - -- \ - << parameters.go-test-flags >> \ - << parameters.target >> + --packages="<< parameters.target >>" \ + -- << parameters.go-test-flags >> no_output_timeout: 30m - store_test_results: path: /tmp/test-reports diff --git a/.circleci/template.yml b/.circleci/template.yml index 6bb688d27..1b79e595c 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -186,9 +186,8 @@ jobs: --format standard-verbose \ --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ - -- \ - << parameters.go-test-flags >> \ - << parameters.target >> + --packages="<< parameters.target >>" \ + -- << parameters.go-test-flags >> no_output_timeout: 30m - store_test_results: path: /tmp/test-reports