From 3ee09115e6a63062746d63a0b85b6a58abda7891 Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 16 May 2024 12:31:08 +0100 Subject: [PATCH] ci: deprecate circle ci in favour of github actions (#11786) --- .circleci/config.yml | 1383 ------------------ .circleci/gen.go | 156 -- .circleci/template.yml | 753 ---------- .github/actions/export-circle-env/action.yml | 14 - .github/workflows/docker.yml | 4 +- .github/workflows/release.yml | 6 +- .github/workflows/test.yml | 14 - .goreleaser.yaml | 6 +- Makefile | 7 +- README.md | 14 +- scripts/publish-checksums.sh | 19 +- scripts/version-check.sh | 6 +- storage/sealer/README.md | 3 +- 13 files changed, 32 insertions(+), 2353 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/gen.go delete mode 100644 .circleci/template.yml delete mode 100644 .github/actions/export-circle-env/action.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 75d747c69..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1383 +0,0 @@ -version: 2.1 -orbs: - aws-cli: circleci/aws-cli@4.1.1 - docker: circleci/docker@2.3.0 - -executors: - golang: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.21.7 - resource_class: medium+ - golang-2xl: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.21.7 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 - -commands: - build-platform-specific: - parameters: - linux: - default: true - description: is a linux build environment? - type: boolean - darwin: - default: false - description: is a darwin build environment? - type: boolean - darwin-architecture: - default: "amd64" - description: which darwin architecture is being used? - type: string - steps: - - checkout - - git_fetch_all_tags - - run: git submodule sync - - run: git submodule update --init - - when: - condition: <> - steps: - - install-ubuntu-deps - - check-go-version - - when: - condition: <> - steps: - - run: - name: Install Go - command: | - curl https://dl.google.com/go/go`cat GO_VERSION_MIN`.darwin-<>.pkg -o /tmp/go.pkg && \ - sudo installer -pkg /tmp/go.pkg -target / - - run: - name: Export Go - command: | - echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV - - run: go version - - run: - name: Install dependencies with Homebrew - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config coreutils jq hwloc - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: make deps - download-params: - steps: - - restore_cache: - name: Restore parameters cache - keys: - - 'v26-2k-lotus-params' - - run: ./lotus fetch-params 2048 - - save_cache: - name: Save parameters cache - key: 'v26-2k-lotus-params' - paths: - - /var/tmp/filecoin-proof-parameters/ - install_ipfs: - steps: - - run: | - curl -O https://dist.ipfs.tech/kubo/v0.16.0/kubo_v0.16.0_linux-amd64.tar.gz - tar -xvzf kubo_v0.16.0_linux-amd64.tar.gz - pushd kubo - sudo bash install.sh - popd - rm -rf kubo - rm kubo_v0.16.0_linux-amd64.tar.gz - git_fetch_all_tags: - steps: - - run: - name: fetch all tags - command: | - git fetch --all - install-ubuntu-deps: - steps: - - run: sudo apt install curl ca-certificates gnupg - - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev - check-go-version: - steps: - - run: | - v=`go version | { read _ _ v _; echo ${v#go}; }` - if [[ $v != `cat GO_VERSION_MIN` ]]; then - echo "GO_VERSION_MIN file does not match the go version being used." - echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." - exit 1 - fi - -jobs: - build: - executor: golang - working_directory: ~/lotus - steps: - - checkout - - git_fetch_all_tags - - run: git submodule sync - - run: git submodule update --init - - install-ubuntu-deps - - check-go-version - - run: make deps lotus - - persist_to_workspace: - root: ~/ - paths: - - "lotus" - mod-tidy-check: - executor: golang - working_directory: ~/lotus - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: go mod tidy -v - - run: - name: Check git diff - command: | - git --no-pager diff go.mod go.sum - git --no-pager diff --quiet go.mod go.sum - - test: - description: | - Run tests with gotestsum. - working_directory: ~/lotus - parameters: &test-params - resource_class: - type: string - default: medium+ - go-test-flags: - type: string - default: "-timeout 20m" - description: Flags passed to go test. - target: - type: string - default: "./..." - description: Import paths of packages to be tested. - proofs-log-test: - type: string - default: "0" - get-params: - type: boolean - default: false - suite: - type: string - default: unit - description: Test suite name to report to CircleCI. - docker: - - image: cimg/go:1.21 - environment: - LOTUS_HARMONYDB_HOSTS: yugabyte - - image: yugabytedb/yugabyte:2.18.0.0-b65 - command: bin/yugabyted start --daemon=false - name: yugabyte - resource_class: << parameters.resource_class >> - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - when: - condition: << parameters.get-params >> - steps: - - download-params - - run: - name: go test - environment: - TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> - SKIP_CONFORMANCE: "1" - LOTUS_SRC_DIR: /home/circleci/project - command: | - mkdir -p /tmp/test-reports/<< parameters.suite >> - mkdir -p /tmp/test-artifacts - dockerize -wait tcp://yugabyte:5433 -timeout 3m - env - gotestsum \ - --format standard-verbose \ - --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ - --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ - --packages="<< parameters.target >>" \ - -- << parameters.go-test-flags >> - no_output_timeout: 30m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-artifacts/<< parameters.suite >>.json - - test-conformance: - working_directory: ~/lotus - description: | - Run tests using a corpus of interoperable test vectors for Filecoin - implementations to test their correctness and compliance with the Filecoin - specifications. - parameters: - <<: *test-params - vectors-branch: - type: string - default: "" - description: | - Branch on github.com/filecoin-project/test-vectors to checkout and - test with. If empty (the default) the commit defined by the git - submodule is used. - docker: - - image: cimg/go:1.21 - resource_class: << parameters.resource_class >> - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - download-params - - when: - condition: - not: - equal: [ "", << parameters.vectors-branch >> ] - steps: - - run: - name: checkout vectors branch - command: | - cd extern/test-vectors - git fetch - git checkout origin/<< parameters.vectors-branch >> - - run: - name: install statediff globally - command: | - ## statediff is optional; we succeed even if compilation fails. - mkdir -p /tmp/statediff - git clone https://github.com/filecoin-project/statediff.git /tmp/statediff - cd /tmp/statediff - go install ./cmd/statediff || exit 0 - - run: - name: go test - environment: - SKIP_CONFORMANCE: "0" - command: | - mkdir -p /tmp/test-reports - mkdir -p /tmp/test-artifacts - gotestsum \ - --format pkgname-and-test-fails \ - --junitfile /tmp/test-reports/junit.xml \ - -- \ - -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ - go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html - no_output_timeout: 30m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-artifacts/conformance-coverage.html - - build-linux-amd64: - executor: golang - steps: - - build-platform-specific - - run: make lotus lotus-miner lotus-worker - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - run: | - mkdir -p /tmp/workspace/linux_amd64_v1 && \ - mv lotus lotus-miner lotus-worker /tmp/workspace/linux_amd64_v1/ - - persist_to_workspace: - root: /tmp/workspace - paths: - - linux_amd64_v1 - - build-darwin-amd64: - description: build darwin lotus binary - working_directory: ~/go/src/github.com/filecoin-project/lotus - macos: - xcode: "13.4.1" - steps: - - build-platform-specific: - linux: false - darwin: true - darwin-architecture: amd64 - - run: make lotus lotus-miner lotus-worker - - run: otool -hv lotus - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - run: | - mkdir -p /tmp/workspace/darwin_amd64_v1 && \ - mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_amd64_v1/ - - persist_to_workspace: - root: /tmp/workspace - paths: - - darwin_amd64_v1 - - build-darwin-arm64: - description: self-hosted m1 runner - working_directory: ~/go/src/github.com/filecoin-project/lotus - machine: true - resource_class: filecoin-project/self-hosted-m1 - steps: - - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - - build-platform-specific: - linux: false - darwin: true - darwin-architecture: arm64 - - run: | - export CPATH=$(brew --prefix)/include && export LIBRARY_PATH=$(brew --prefix)/lib && make lotus lotus-miner lotus-worker - - run: otool -hv lotus - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - run: | - mkdir -p /tmp/workspace/darwin_arm64 && \ - mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_arm64/ - - persist_to_workspace: - root: /tmp/workspace - paths: - - darwin_arm64 - - run: - command: make clean - when: always - - run: - name: cleanup homebrew - command: HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall pkg-config coreutils jq hwloc - when: always - - release: - executor: golang - parameters: - dry-run: - default: false - description: should this release actually publish it's artifacts? - type: boolean - steps: - - checkout - - run: | - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list - sudo apt update - sudo apt install goreleaser-pro - - install_ipfs - - attach_workspace: - at: /tmp/workspace - - when: - condition: << parameters.dry-run >> - steps: - - run: goreleaser release --rm-dist --snapshot --debug - - run: ./scripts/generate-checksums.sh - - when: - condition: - not: << parameters.dry-run >> - steps: - - run: goreleaser release --rm-dist --debug - - run: ./scripts/generate-checksums.sh - - run: ./scripts/publish-checksums.sh - - gofmt: - executor: golang - working_directory: ~/lotus - steps: - - run: - command: "! go fmt ./... 2>&1 | read" - - gen-check: - executor: golang - working_directory: ~/lotus - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: go install golang.org/x/tools/cmd/goimports - - run: go install github.com/hannahhoward/cbor-gen-for - - run: make gen - - run: git --no-pager diff && git --no-pager diff --quiet - - run: make docsgen-cli - - run: git --no-pager diff && git --no-pager diff --quiet - - docs-check: - executor: golang - working_directory: ~/lotus - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: go install golang.org/x/tools/cmd/goimports - - run: make docsgen - - run: git --no-pager diff && git --no-pager diff --quiet - - lint-all: - description: | - Run golangci-lint. - working_directory: ~/lotus - parameters: - args: - type: string - default: '' - description: | - Arguments to pass to golangci-lint - docker: - - image: cimg/go:1.21 - resource_class: medium+ - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: - name: Lint - command: | - golangci-lint run -v --timeout 10m \ - --concurrency 4 << parameters.args >> - - build-docker: - description: > - Publish to Dockerhub - executor: docker/docker - parameters: - image: - type: string - default: lotus - description: > - Passed to the docker build process to determine which image in the - Dockerfile should be built. Expected values are `lotus`, - `lotus-all-in-one` - network: - type: string - default: "mainnet" - description: > - Passed to the docker build process using GOFLAGS+=-tags=<>. - Expected values are `debug`, `2k`, `calibnet`, `butterflynet`, - `interopnet`. - channel: - type: string - default: "" - description: > - The release channel to use for this image. - push: - type: boolean - default: false - description: > - When true, pushes the image to Dockerhub - steps: - - setup_remote_docker - - checkout - - git_fetch_all_tags - - run: git submodule sync - - run: git submodule update --init - - - docker/check: - docker-username: DOCKERHUB_USERNAME - docker-password: DOCKERHUB_PASSWORD - - when: - condition: - equal: [ mainnet, <> ] - steps: - - when: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> - tag: <> - - run: - name: Docker push - command: | - docker push filecoin/<>:<> - if [[ ! -z $CIRCLE_SHA ]]; then - docker image tag filecoin/<>:<> filecoin/<>:"${CIRCLE_SHA:0:7}" - docker push filecoin/<>:"${CIRCLE_SHA:0:7}" - fi - if [[ ! -z $CIRCLE_TAG ]]; then - docker image tag filecoin/<>:<> filecoin/<>:"${CIRCLE_TAG}" - docker push filecoin/<>:"${CIRCLE_TAG}" - fi - - unless: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> - - when: - condition: - not: - equal: [ mainnet, <> ] - steps: - - when: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> - tag: <>-<> - - run: - name: Docker push - command: | - docker push filecoin/<>:<>-<> - if [[ ! -z $CIRCLE_SHA ]]; then - docker image tag filecoin/<>:<>-<> filecoin/<>:"${CIRCLE_SHA:0:7}"-<> - docker push filecoin/<>:"${CIRCLE_SHA:0:7}"-<> - fi - if [[ ! -z $CIRCLE_TAG ]]; then - docker image tag filecoin/<>:<>-<> filecoin/<>:"${CIRCLE_TAG}"-<> - docker push filecoin/<>:"${CIRCLE_TAG}"-<> - fi - - unless: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> - -workflows: - ci: - jobs: - - build - - lint-all: - requires: - - build - - mod-tidy-check: - requires: - - build - - gofmt: - requires: - - build - - gen-check: - requires: - - build - - docs-check: - requires: - - build - - test: - name: test-itest-api - requires: - - build - suite: itest-api - target: "./itests/api_test.go" - - test: - name: test-itest-batch_deal - requires: - - build - suite: itest-batch_deal - target: "./itests/batch_deal_test.go" - - test: - name: test-itest-cli - requires: - - build - suite: itest-cli - target: "./itests/cli_test.go" - - test: - name: test-itest-curio - requires: - - build - suite: itest-curio - target: "./itests/curio_test.go" - - test: - name: test-itest-deadlines - requires: - - build - suite: itest-deadlines - target: "./itests/deadlines_test.go" - - test: - name: test-itest-deals_512mb - requires: - - build - suite: itest-deals_512mb - target: "./itests/deals_512mb_test.go" - - test: - name: test-itest-deals_anycid - requires: - - build - suite: itest-deals_anycid - target: "./itests/deals_anycid_test.go" - - test: - name: test-itest-deals_concurrent - requires: - - build - suite: itest-deals_concurrent - target: "./itests/deals_concurrent_test.go" - resource_class: 2xlarge - - test: - name: test-itest-deals_invalid_utf8_label - requires: - - build - suite: itest-deals_invalid_utf8_label - target: "./itests/deals_invalid_utf8_label_test.go" - - test: - name: test-itest-deals_max_staging_deals - requires: - - build - suite: itest-deals_max_staging_deals - target: "./itests/deals_max_staging_deals_test.go" - - test: - name: test-itest-deals_offline - requires: - - build - suite: itest-deals_offline - target: "./itests/deals_offline_test.go" - - test: - name: test-itest-deals_padding - requires: - - build - suite: itest-deals_padding - target: "./itests/deals_padding_test.go" - - test: - name: test-itest-deals_partial_retrieval_dm-level - requires: - - build - suite: itest-deals_partial_retrieval_dm-level - target: "./itests/deals_partial_retrieval_dm-level_test.go" - - test: - name: test-itest-deals_partial_retrieval - requires: - - build - suite: itest-deals_partial_retrieval - target: "./itests/deals_partial_retrieval_test.go" - - test: - name: test-itest-deals_power - requires: - - build - suite: itest-deals_power - target: "./itests/deals_power_test.go" - - test: - name: test-itest-deals_pricing - requires: - - build - suite: itest-deals_pricing - target: "./itests/deals_pricing_test.go" - - test: - name: test-itest-deals_publish - requires: - - build - suite: itest-deals_publish - target: "./itests/deals_publish_test.go" - - test: - name: test-itest-deals_remote_retrieval - requires: - - build - suite: itest-deals_remote_retrieval - target: "./itests/deals_remote_retrieval_test.go" - - test: - name: test-itest-deals_retry_deal_no_funds - requires: - - build - suite: itest-deals_retry_deal_no_funds - target: "./itests/deals_retry_deal_no_funds_test.go" - - test: - name: test-itest-deals - requires: - - build - suite: itest-deals - target: "./itests/deals_test.go" - - test: - name: test-itest-decode_params - requires: - - build - suite: itest-decode_params - target: "./itests/decode_params_test.go" - - test: - name: test-itest-direct_data_onboard - requires: - - build - suite: itest-direct_data_onboard - target: "./itests/direct_data_onboard_test.go" - - test: - name: test-itest-direct_data_onboard_verified - requires: - - build - suite: itest-direct_data_onboard_verified - target: "./itests/direct_data_onboard_verified_test.go" - - test: - name: test-itest-dup_mpool_messages - requires: - - build - suite: itest-dup_mpool_messages - target: "./itests/dup_mpool_messages_test.go" - - test: - name: test-itest-eth_account_abstraction - requires: - - build - suite: itest-eth_account_abstraction - target: "./itests/eth_account_abstraction_test.go" - - test: - name: test-itest-eth_api - requires: - - build - suite: itest-eth_api - target: "./itests/eth_api_test.go" - - test: - name: test-itest-eth_balance - requires: - - build - suite: itest-eth_balance - target: "./itests/eth_balance_test.go" - - test: - name: test-itest-eth_block_hash - requires: - - build - suite: itest-eth_block_hash - target: "./itests/eth_block_hash_test.go" - - test: - name: test-itest-eth_bytecode - requires: - - build - suite: itest-eth_bytecode - target: "./itests/eth_bytecode_test.go" - - test: - name: test-itest-eth_config - requires: - - build - suite: itest-eth_config - target: "./itests/eth_config_test.go" - - test: - name: test-itest-eth_conformance - requires: - - build - suite: itest-eth_conformance - target: "./itests/eth_conformance_test.go" - - test: - name: test-itest-eth_deploy - requires: - - build - suite: itest-eth_deploy - target: "./itests/eth_deploy_test.go" - - test: - name: test-itest-eth_fee_history - requires: - - build - suite: itest-eth_fee_history - target: "./itests/eth_fee_history_test.go" - - test: - name: test-itest-eth_filter - requires: - - build - suite: itest-eth_filter - target: "./itests/eth_filter_test.go" - - test: - name: test-itest-eth_hash_lookup - requires: - - build - suite: itest-eth_hash_lookup - target: "./itests/eth_hash_lookup_test.go" - - test: - name: test-itest-eth_transactions - requires: - - build - suite: itest-eth_transactions - target: "./itests/eth_transactions_test.go" - - test: - name: test-itest-fevm_address - requires: - - build - suite: itest-fevm_address - target: "./itests/fevm_address_test.go" - - test: - name: test-itest-fevm_events - requires: - - build - suite: itest-fevm_events - target: "./itests/fevm_events_test.go" - - test: - name: test-itest-fevm - requires: - - build - suite: itest-fevm - target: "./itests/fevm_test.go" - - test: - name: test-itest-gas_estimation - requires: - - build - suite: itest-gas_estimation - target: "./itests/gas_estimation_test.go" - - test: - name: test-itest-gateway - requires: - - build - suite: itest-gateway - target: "./itests/gateway_test.go" - - test: - name: test-itest-get_messages_in_ts - requires: - - build - suite: itest-get_messages_in_ts - target: "./itests/get_messages_in_ts_test.go" - - test: - name: test-itest-harmonydb - requires: - - build - suite: itest-harmonydb - target: "./itests/harmonydb_test.go" - - test: - name: test-itest-harmonytask - requires: - - build - suite: itest-harmonytask - target: "./itests/harmonytask_test.go" - - test: - name: test-itest-lite_migration - requires: - - build - suite: itest-lite_migration - target: "./itests/lite_migration_test.go" - - test: - name: test-itest-lookup_robust_address - requires: - - build - suite: itest-lookup_robust_address - target: "./itests/lookup_robust_address_test.go" - - test: - name: test-itest-mempool - requires: - - build - suite: itest-mempool - target: "./itests/mempool_test.go" - - test: - name: test-itest-migration - requires: - - build - suite: itest-migration - target: "./itests/migration_test.go" - - test: - name: test-itest-mpool_msg_uuid - requires: - - build - suite: itest-mpool_msg_uuid - target: "./itests/mpool_msg_uuid_test.go" - - test: - name: test-itest-mpool_push_with_uuid - requires: - - build - suite: itest-mpool_push_with_uuid - target: "./itests/mpool_push_with_uuid_test.go" - - test: - name: test-itest-msgindex - requires: - - build - suite: itest-msgindex - target: "./itests/msgindex_test.go" - - test: - name: test-itest-multisig - requires: - - build - suite: itest-multisig - target: "./itests/multisig_test.go" - - test: - name: test-itest-net - requires: - - build - suite: itest-net - target: "./itests/net_test.go" - - test: - name: test-itest-nonce - requires: - - build - suite: itest-nonce - target: "./itests/nonce_test.go" - - test: - name: test-itest-path_detach_redeclare - requires: - - build - suite: itest-path_detach_redeclare - target: "./itests/path_detach_redeclare_test.go" - - test: - name: test-itest-path_type_filters - requires: - - build - suite: itest-path_type_filters - target: "./itests/path_type_filters_test.go" - - test: - name: test-itest-paych_api - requires: - - build - suite: itest-paych_api - target: "./itests/paych_api_test.go" - - test: - name: test-itest-paych_cli - requires: - - build - suite: itest-paych_cli - target: "./itests/paych_cli_test.go" - - test: - name: test-itest-pending_deal_allocation - requires: - - build - suite: itest-pending_deal_allocation - target: "./itests/pending_deal_allocation_test.go" - - test: - name: test-itest-remove_verifreg_datacap - requires: - - build - suite: itest-remove_verifreg_datacap - target: "./itests/remove_verifreg_datacap_test.go" - - test: - name: test-itest-sealing_resources - requires: - - build - suite: itest-sealing_resources - target: "./itests/sealing_resources_test.go" - - test: - name: test-itest-sector_finalize_early - requires: - - build - suite: itest-sector_finalize_early - target: "./itests/sector_finalize_early_test.go" - - test: - name: test-itest-sector_import_full - requires: - - build - suite: itest-sector_import_full - target: "./itests/sector_import_full_test.go" - - test: - name: test-itest-sector_import_simple - requires: - - build - suite: itest-sector_import_simple - target: "./itests/sector_import_simple_test.go" - - test: - name: test-itest-sector_miner_collateral - requires: - - build - suite: itest-sector_miner_collateral - target: "./itests/sector_miner_collateral_test.go" - - test: - name: test-itest-sector_numassign - requires: - - build - suite: itest-sector_numassign - target: "./itests/sector_numassign_test.go" - - test: - name: test-itest-sector_pledge - requires: - - build - suite: itest-sector_pledge - target: "./itests/sector_pledge_test.go" - resource_class: 2xlarge - get-params: true - - - test: - name: test-itest-sector_terminate - requires: - - build - suite: itest-sector_terminate - target: "./itests/sector_terminate_test.go" - - test: - name: test-itest-sector_unseal - requires: - - build - suite: itest-sector_unseal - target: "./itests/sector_unseal_test.go" - - test: - name: test-itest-self_sent_txn - requires: - - build - suite: itest-self_sent_txn - target: "./itests/self_sent_txn_test.go" - - test: - name: test-itest-splitstore - requires: - - build - suite: itest-splitstore - target: "./itests/splitstore_test.go" - - test: - name: test-itest-supply - requires: - - build - suite: itest-supply - target: "./itests/supply_test.go" - - test: - name: test-itest-verifreg - requires: - - build - suite: itest-verifreg - target: "./itests/verifreg_test.go" - - test: - name: test-itest-wdpost_config - requires: - - build - suite: itest-wdpost_config - target: "./itests/wdpost_config_test.go" - - test: - name: test-itest-wdpost_dispute - requires: - - build - suite: itest-wdpost_dispute - target: "./itests/wdpost_dispute_test.go" - - test: - name: test-itest-wdpost_no_miner_storage - requires: - - build - suite: itest-wdpost_no_miner_storage - target: "./itests/wdpost_no_miner_storage_test.go" - - test: - name: test-itest-wdpost - requires: - - build - suite: itest-wdpost - target: "./itests/wdpost_test.go" - get-params: true - - - test: - name: test-itest-wdpost_worker_config - requires: - - build - suite: itest-wdpost_worker_config - target: "./itests/wdpost_worker_config_test.go" - resource_class: 2xlarge - - test: - name: test-itest-worker - requires: - - build - suite: itest-worker - target: "./itests/worker_test.go" - resource_class: 2xlarge - - test: - name: test-itest-worker_upgrade - requires: - - build - suite: itest-worker_upgrade - target: "./itests/worker_upgrade_test.go" - - test: - name: test-unit-cli - requires: - - build - suite: utest-unit-cli - target: "./cli/... ./cmd/... ./api/..." - resource_class: 2xlarge - get-params: true - - test: - name: test-unit-node - requires: - - build - suite: utest-unit-node - target: "./node/..." - - test: - name: test-unit-rest - requires: - - build - suite: utest-unit-rest - target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./curiosrc/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..." - resource_class: 2xlarge - - test: - name: test-unit-storage - requires: - - build - suite: utest-unit-storage - target: "./storage/... ./extern/..." - get-params: true - - test: - go-test-flags: "-run=TestMulticoreSDR" - requires: - - build - suite: multicore-sdr-check - target: "./storage/sealer/ffiwrapper" - proofs-log-test: "1" - - test-conformance: - requires: - - build - suite: conformance - target: "./conformance" - - release: - jobs: - - build-linux-amd64: - name: "Build ( linux / amd64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-darwin-amd64: - name: "Build ( darwin / amd64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-darwin-arm64: - name: "Build ( darwin / arm64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - release: - name: "Release" - requires: - - "Build ( darwin / amd64 )" - - "Build ( linux / amd64 )" - - "Build ( darwin / arm64 )" - filters: - branches: - ignore: - - /^.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - release: - name: "Release (dry-run)" - dry-run: true - requires: - - "Build ( darwin / amd64 )" - - "Build ( linux / amd64 )" - - "Build ( darwin / arm64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - - build-docker: - name: "Docker push (lotus-all-in-one / stable / mainnet)" - image: lotus-all-in-one - channel: stable - network: mainnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / candidate / mainnet)" - image: lotus-all-in-one - channel: candidate - network: mainnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / edge / mainnet)" - image: lotus-all-in-one - channel: master - network: mainnet - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus-all-in-one / mainnet)" - image: lotus-all-in-one - network: mainnet - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-docker: - name: "Docker push (lotus-all-in-one / stable / butterflynet)" - image: lotus-all-in-one - channel: stable - network: butterflynet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / candidate / butterflynet)" - image: lotus-all-in-one - channel: candidate - network: butterflynet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / edge / butterflynet)" - image: lotus-all-in-one - channel: master - network: butterflynet - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus-all-in-one / butterflynet)" - image: lotus-all-in-one - network: butterflynet - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-docker: - name: "Docker push (lotus-all-in-one / stable / calibnet)" - image: lotus-all-in-one - channel: stable - network: calibnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / candidate / calibnet)" - image: lotus-all-in-one - channel: candidate - network: calibnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / edge / calibnet)" - image: lotus-all-in-one - channel: master - network: calibnet - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus-all-in-one / calibnet)" - image: lotus-all-in-one - network: calibnet - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-docker: - name: "Docker push (lotus-all-in-one / stable / debug)" - image: lotus-all-in-one - channel: stable - network: debug - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / candidate / debug)" - image: lotus-all-in-one - channel: candidate - network: debug - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / edge / debug)" - image: lotus-all-in-one - channel: master - network: debug - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus-all-in-one / debug)" - image: lotus-all-in-one - network: debug - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-docker: - name: "Docker push (lotus / stable / mainnet)" - image: lotus - channel: stable - network: mainnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus / candidate / mainnet)" - image: lotus - channel: candidate - network: mainnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus / master / mainnet)" - image: lotus - channel: master - network: mainnet - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus / mainnet)" - image: lotus - network: mainnet - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - build-docker: - name: "Docker (lotus-all-in-one / nightly / mainnet)" - image: lotus-all-in-one - channel: nightly - network: mainnet - push: true - - build-docker: - name: "Docker (lotus-all-in-one / nightly / butterflynet)" - image: lotus-all-in-one - channel: nightly - network: butterflynet - push: true - - build-docker: - name: "Docker (lotus-all-in-one / nightly / calibnet)" - image: lotus-all-in-one - channel: nightly - network: calibnet - push: true - - build-docker: - name: "Docker (lotus-all-in-one / nightly / debug)" - image: lotus-all-in-one - channel: nightly - network: debug - push: true diff --git a/.circleci/gen.go b/.circleci/gen.go deleted file mode 100644 index 19329247a..000000000 --- a/.circleci/gen.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -import ( - "embed" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "text/template" -) - -var GoVersion = "" // from init below. Ex: 1.19.7 - -//go:generate go run ./gen.go .. - -//go:embed template.yml -var templateFile embed.FS - -func init() { - b, err := os.ReadFile("../go.mod") - if err != nil { - panic("cannot find go.mod in parent folder") - } - for _, line := range strings.Split(string(b), "\n") { - if strings.HasPrefix(line, "go ") { - GoVersion = line[3:] - } - } -} - -type ( - dirs = []string - suite = string -) - -// groupedUnitTests maps suite names to top-level directories that should be -// included in that suite. The program adds an implicit group "rest" that -// includes all other top-level directories. -var groupedUnitTests = map[suite]dirs{ - "unit-node": {"node"}, - "unit-storage": {"storage", "extern"}, - "unit-cli": {"cli", "cmd", "api"}, -} - -func main() { - if len(os.Args) != 2 { - panic("expected path to repo as argument") - } - - repo := os.Args[1] - - tmpl := template.New("template.yml") - tmpl.Delims("[[", "]]") - tmpl.Funcs(template.FuncMap{ - "stripSuffix": func(in string) string { - return strings.TrimSuffix(in, "_test.go") - }, - }) - tmpl = template.Must(tmpl.ParseFS(templateFile, "*")) - - // list all itests. - itests, err := filepath.Glob(filepath.Join(repo, "./itests/*_test.go")) - if err != nil { - panic(err) - } - - // strip the dir from all entries. - for i, f := range itests { - itests[i] = filepath.Base(f) - } - - // calculate the exclusion set of unit test directories to exclude because - // they are already included in a grouped suite. - var excluded = map[string]struct{}{} - for _, ss := range groupedUnitTests { - for _, s := range ss { - e, err := filepath.Abs(filepath.Join(repo, s)) - if err != nil { - panic(err) - } - // Redundantly flag both absolute and relative paths as excluded - excluded[filepath.Join(repo, s)] = struct{}{} - excluded[e] = struct{}{} - } - } - - // all unit tests top-level dirs that are not itests, nor included in other suites. - var rest = map[string]struct{}{} - err = filepath.Walk(repo, func(path string, f os.FileInfo, err error) error { - // include all tests that aren't in the itests directory. - if strings.Contains(path, "itests") { - return filepath.SkipDir - } - // exclude all tests included in other suites - if f.IsDir() { - if _, ok := excluded[path]; ok { - return filepath.SkipDir - } - } - if strings.HasSuffix(path, "_test.go") { - rel, err := filepath.Rel(repo, path) - if err != nil { - panic(err) - } - // take the first directory - rest[strings.Split(rel, string(os.PathSeparator))[0]] = struct{}{} - } - return err - }) - if err != nil { - panic(err) - } - - // add other directories to a 'rest' suite. - for k := range rest { - groupedUnitTests["unit-rest"] = append(groupedUnitTests["unit-rest"], k) - } - - // map iteration guarantees no order, so sort the array in-place. - sort.Strings(groupedUnitTests["unit-rest"]) - - // form the input data. - type data struct { - Networks []string - ItestFiles []string - UnitSuites map[string]string - GoVersion string - } - in := data{ - Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, - ItestFiles: itests, - UnitSuites: func() map[string]string { - ret := make(map[string]string) - for name, dirs := range groupedUnitTests { - for i, d := range dirs { - dirs[i] = fmt.Sprintf("./%s/...", d) // turn into package - } - ret[name] = strings.Join(dirs, " ") - } - return ret - }(), - GoVersion: GoVersion, - } - - out, err := os.Create("./config.yml") - if err != nil { - panic(err) - } - defer out.Close() - - // execute the template. - if err := tmpl.Execute(out, in); err != nil { - panic(err) - } -} diff --git a/.circleci/template.yml b/.circleci/template.yml deleted file mode 100644 index b7462b027..000000000 --- a/.circleci/template.yml +++ /dev/null @@ -1,753 +0,0 @@ -version: 2.1 -orbs: - aws-cli: circleci/aws-cli@4.1.1 - docker: circleci/docker@2.3.0 - -executors: - golang: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.21.7 - resource_class: medium+ - golang-2xl: - docker: - # Must match GO_VERSION_MIN in project root - - image: cimg/go:1.21.7 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 - -commands: - build-platform-specific: - parameters: - linux: - default: true - description: is a linux build environment? - type: boolean - darwin: - default: false - description: is a darwin build environment? - type: boolean - darwin-architecture: - default: "amd64" - description: which darwin architecture is being used? - type: string - steps: - - checkout - - git_fetch_all_tags - - run: git submodule sync - - run: git submodule update --init - - when: - condition: <> - steps: - - install-ubuntu-deps - - check-go-version - - when: - condition: <> - steps: - - run: - name: Install Go - command: | - curl https://dl.google.com/go/go`cat GO_VERSION_MIN`.darwin-<>.pkg -o /tmp/go.pkg && \ - sudo installer -pkg /tmp/go.pkg -target / - - run: - name: Export Go - command: | - echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV - - run: go version - - run: - name: Install dependencies with Homebrew - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config coreutils jq hwloc - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: make deps - download-params: - steps: - - restore_cache: - name: Restore parameters cache - keys: - - 'v26-2k-lotus-params' - - run: ./lotus fetch-params 2048 - - save_cache: - name: Save parameters cache - key: 'v26-2k-lotus-params' - paths: - - /var/tmp/filecoin-proof-parameters/ - install_ipfs: - steps: - - run: | - curl -O https://dist.ipfs.tech/kubo/v0.16.0/kubo_v0.16.0_linux-amd64.tar.gz - tar -xvzf kubo_v0.16.0_linux-amd64.tar.gz - pushd kubo - sudo bash install.sh - popd - rm -rf kubo - rm kubo_v0.16.0_linux-amd64.tar.gz - git_fetch_all_tags: - steps: - - run: - name: fetch all tags - command: | - git fetch --all - install-ubuntu-deps: - steps: - - run: sudo apt install curl ca-certificates gnupg - - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev - check-go-version: - steps: - - run: | - v=`go version | { read _ _ v _; echo ${v#go}; }` - if [["[[ $v != `cat GO_VERSION_MIN` ]]"]]; then - echo "GO_VERSION_MIN file does not match the go version being used." - echo "Please update image to cimg/go:`cat GO_VERSION_MIN` or update GO_VERSION_MIN to $v." - exit 1 - fi - -jobs: - build: - executor: golang - working_directory: ~/lotus - steps: - - checkout - - git_fetch_all_tags - - run: git submodule sync - - run: git submodule update --init - - install-ubuntu-deps - - check-go-version - - run: make deps lotus - - persist_to_workspace: - root: ~/ - paths: - - "lotus" - mod-tidy-check: - executor: golang - working_directory: ~/lotus - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: go mod tidy -v - - run: - name: Check git diff - command: | - git --no-pager diff go.mod go.sum - git --no-pager diff --quiet go.mod go.sum - - test: - description: | - Run tests with gotestsum. - working_directory: ~/lotus - parameters: &test-params - resource_class: - type: string - default: medium+ - go-test-flags: - type: string - default: "-timeout 20m" - description: Flags passed to go test. - target: - type: string - default: "./..." - description: Import paths of packages to be tested. - proofs-log-test: - type: string - default: "0" - get-params: - type: boolean - default: false - suite: - type: string - default: unit - description: Test suite name to report to CircleCI. - docker: - - image: cimg/go:[[ .GoVersion]] - environment: - LOTUS_HARMONYDB_HOSTS: yugabyte - - image: yugabytedb/yugabyte:2.18.0.0-b65 - command: bin/yugabyted start --daemon=false - name: yugabyte - resource_class: << parameters.resource_class >> - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - when: - condition: << parameters.get-params >> - steps: - - download-params - - run: - name: go test - environment: - TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> - SKIP_CONFORMANCE: "1" - LOTUS_SRC_DIR: /home/circleci/project - command: | - mkdir -p /tmp/test-reports/<< parameters.suite >> - mkdir -p /tmp/test-artifacts - dockerize -wait tcp://yugabyte:5433 -timeout 3m - env - gotestsum \ - --format standard-verbose \ - --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ - --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ - --packages="<< parameters.target >>" \ - -- << parameters.go-test-flags >> - no_output_timeout: 30m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-artifacts/<< parameters.suite >>.json - - test-conformance: - working_directory: ~/lotus - description: | - Run tests using a corpus of interoperable test vectors for Filecoin - implementations to test their correctness and compliance with the Filecoin - specifications. - parameters: - <<: *test-params - vectors-branch: - type: string - default: "" - description: | - Branch on github.com/filecoin-project/test-vectors to checkout and - test with. If empty (the default) the commit defined by the git - submodule is used. - docker: - - image: cimg/go:[[ .GoVersion]] - resource_class: << parameters.resource_class >> - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - download-params - - when: - condition: - not: - equal: [ "", << parameters.vectors-branch >> ] - steps: - - run: - name: checkout vectors branch - command: | - cd extern/test-vectors - git fetch - git checkout origin/<< parameters.vectors-branch >> - - run: - name: install statediff globally - command: | - ## statediff is optional; we succeed even if compilation fails. - mkdir -p /tmp/statediff - git clone https://github.com/filecoin-project/statediff.git /tmp/statediff - cd /tmp/statediff - go install ./cmd/statediff || exit 0 - - run: - name: go test - environment: - SKIP_CONFORMANCE: "0" - command: | - mkdir -p /tmp/test-reports - mkdir -p /tmp/test-artifacts - gotestsum \ - --format pkgname-and-test-fails \ - --junitfile /tmp/test-reports/junit.xml \ - -- \ - -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ - go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html - no_output_timeout: 30m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-artifacts/conformance-coverage.html - - build-linux-amd64: - executor: golang - steps: - - build-platform-specific - - run: make lotus lotus-miner lotus-worker - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - run: | - mkdir -p /tmp/workspace/linux_amd64_v1 && \ - mv lotus lotus-miner lotus-worker /tmp/workspace/linux_amd64_v1/ - - persist_to_workspace: - root: /tmp/workspace - paths: - - linux_amd64_v1 - - build-darwin-amd64: - description: build darwin lotus binary - working_directory: ~/go/src/github.com/filecoin-project/lotus - macos: - xcode: "13.4.1" - steps: - - build-platform-specific: - linux: false - darwin: true - darwin-architecture: amd64 - - run: make lotus lotus-miner lotus-worker - - run: otool -hv lotus - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - run: | - mkdir -p /tmp/workspace/darwin_amd64_v1 && \ - mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_amd64_v1/ - - persist_to_workspace: - root: /tmp/workspace - paths: - - darwin_amd64_v1 - - build-darwin-arm64: - description: self-hosted m1 runner - working_directory: ~/go/src/github.com/filecoin-project/lotus - machine: true - resource_class: filecoin-project/self-hosted-m1 - steps: - - run: echo 'export PATH=/opt/homebrew/bin:"$PATH"' >> "$BASH_ENV" - - build-platform-specific: - linux: false - darwin: true - darwin-architecture: arm64 - - run: | - export CPATH=$(brew --prefix)/include && export LIBRARY_PATH=$(brew --prefix)/lib && make lotus lotus-miner lotus-worker - - run: otool -hv lotus - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - run: | - mkdir -p /tmp/workspace/darwin_arm64 && \ - mv lotus lotus-miner lotus-worker /tmp/workspace/darwin_arm64/ - - persist_to_workspace: - root: /tmp/workspace - paths: - - darwin_arm64 - - run: - command: make clean - when: always - - run: - name: cleanup homebrew - command: HOMEBREW_NO_AUTO_UPDATE=1 brew uninstall pkg-config coreutils jq hwloc - when: always - - release: - executor: golang - parameters: - dry-run: - default: false - description: should this release actually publish it's artifacts? - type: boolean - steps: - - checkout - - run: | - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list - sudo apt update - sudo apt install goreleaser-pro - - install_ipfs - - attach_workspace: - at: /tmp/workspace - - when: - condition: << parameters.dry-run >> - steps: - - run: goreleaser release --rm-dist --snapshot --debug - - run: ./scripts/generate-checksums.sh - - when: - condition: - not: << parameters.dry-run >> - steps: - - run: goreleaser release --rm-dist --debug - - run: ./scripts/generate-checksums.sh - - run: ./scripts/publish-checksums.sh - - gofmt: - executor: golang - working_directory: ~/lotus - steps: - - run: - command: "! go fmt ./... 2>&1 | read" - - gen-check: - executor: golang - working_directory: ~/lotus - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: go install golang.org/x/tools/cmd/goimports - - run: go install github.com/hannahhoward/cbor-gen-for - - run: make gen - - run: git --no-pager diff && git --no-pager diff --quiet - - run: make docsgen-cli - - run: git --no-pager diff && git --no-pager diff --quiet - - docs-check: - executor: golang - working_directory: ~/lotus - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: go install golang.org/x/tools/cmd/goimports - - run: make docsgen - - run: git --no-pager diff && git --no-pager diff --quiet - - lint-all: - description: | - Run golangci-lint. - working_directory: ~/lotus - parameters: - args: - type: string - default: '' - description: | - Arguments to pass to golangci-lint - docker: - - image: cimg/go:[[ .GoVersion]] - resource_class: medium+ - steps: - - install-ubuntu-deps - - attach_workspace: - at: ~/ - - run: - name: Lint - command: | - golangci-lint run -v --timeout 10m \ - --concurrency 4 << parameters.args >> - - build-docker: - description: > - Publish to Dockerhub - executor: docker/docker - parameters: - image: - type: string - default: lotus - description: > - Passed to the docker build process to determine which image in the - Dockerfile should be built. Expected values are `lotus`, - `lotus-all-in-one` - network: - type: string - default: "mainnet" - description: > - Passed to the docker build process using GOFLAGS+=-tags=<>. - Expected values are `debug`, `2k`, `calibnet`, `butterflynet`, - `interopnet`. - channel: - type: string - default: "" - description: > - The release channel to use for this image. - push: - type: boolean - default: false - description: > - When true, pushes the image to Dockerhub - steps: - - setup_remote_docker - - checkout - - git_fetch_all_tags - - run: git submodule sync - - run: git submodule update --init - - - docker/check: - docker-username: DOCKERHUB_USERNAME - docker-password: DOCKERHUB_PASSWORD - - when: - condition: - equal: [ mainnet, <> ] - steps: - - when: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> - tag: <> - - run: - name: Docker push - command: | - docker push filecoin/<>:<> - if [["[[ ! -z $CIRCLE_SHA ]]"]]; then - docker image tag filecoin/<>:<> filecoin/<>:"${CIRCLE_SHA:0:7}" - docker push filecoin/<>:"${CIRCLE_SHA:0:7}" - fi - if [["[[ ! -z $CIRCLE_TAG ]]"]]; then - docker image tag filecoin/<>:<> filecoin/<>:"${CIRCLE_TAG}" - docker push filecoin/<>:"${CIRCLE_TAG}" - fi - - unless: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> - - when: - condition: - not: - equal: [ mainnet, <> ] - steps: - - when: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> - tag: <>-<> - - run: - name: Docker push - command: | - docker push filecoin/<>:<>-<> - if [["[[ ! -z $CIRCLE_SHA ]]"]]; then - docker image tag filecoin/<>:<>-<> filecoin/<>:"${CIRCLE_SHA:0:7}"-<> - docker push filecoin/<>:"${CIRCLE_SHA:0:7}"-<> - fi - if [["[[ ! -z $CIRCLE_TAG ]]"]]; then - docker image tag filecoin/<>:<>-<> filecoin/<>:"${CIRCLE_TAG}"-<> - docker push filecoin/<>:"${CIRCLE_TAG}"-<> - fi - - unless: - condition: <> - steps: - - docker/build: - image: filecoin/<> - extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> - -workflows: - ci: - jobs: - - build - - lint-all: - requires: - - build - - mod-tidy-check: - requires: - - build - - gofmt: - requires: - - build - - gen-check: - requires: - - build - - docs-check: - requires: - - build - - [[- range $file := .ItestFiles -]] - [[ with $name := $file | stripSuffix ]] - - test: - name: test-itest-[[ $name ]] - requires: - - build - suite: itest-[[ $name ]] - target: "./itests/[[ $file ]]" - [[- if or (eq $name "worker") (eq $name "deals_concurrent") (eq $name "wdpost_worker_config") (eq $name "sector_pledge")]] - resource_class: 2xlarge - [[- end]] - [[- if or (eq $name "wdpost") (eq $name "sector_pledge")]] - get-params: true - [[end]] - [[- end ]][[- end]] - - [[- range $suite, $pkgs := .UnitSuites]] - - test: - name: test-[[ $suite ]] - requires: - - build - suite: utest-[[ $suite ]] - target: "[[ $pkgs ]]" - [[- if eq $suite "unit-storage"]] - get-params: true - [[- end -]] - [[- if eq $suite "unit-cli"]] - resource_class: 2xlarge - get-params: true - [[- end -]] - [[- if eq $suite "unit-rest"]] - resource_class: 2xlarge - [[- end -]] - [[- end]] - - test: - go-test-flags: "-run=TestMulticoreSDR" - requires: - - build - suite: multicore-sdr-check - target: "./storage/sealer/ffiwrapper" - proofs-log-test: "1" - - test-conformance: - requires: - - build - suite: conformance - target: "./conformance" - - release: - jobs: - - build-linux-amd64: - name: "Build ( linux / amd64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-darwin-amd64: - name: "Build ( darwin / amd64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-darwin-arm64: - name: "Build ( darwin / arm64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - release: - name: "Release" - requires: - - "Build ( darwin / amd64 )" - - "Build ( linux / amd64 )" - - "Build ( darwin / arm64 )" - filters: - branches: - ignore: - - /^.*$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - release: - name: "Release (dry-run)" - dry-run: true - requires: - - "Build ( darwin / amd64 )" - - "Build ( linux / amd64 )" - - "Build ( darwin / arm64 )" - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - /^ci\/.*$/ - [[- range .Networks]] - - build-docker: - name: "Docker push (lotus-all-in-one / stable / [[.]])" - image: lotus-all-in-one - channel: stable - network: [[.]] - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / candidate / [[.]])" - image: lotus-all-in-one - channel: candidate - network: [[.]] - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus-all-in-one / edge / [[.]])" - image: lotus-all-in-one - channel: master - network: [[.]] - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus-all-in-one / [[.]])" - image: lotus-all-in-one - network: [[.]] - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - [[- end]] - - build-docker: - name: "Docker push (lotus / stable / mainnet)" - image: lotus - channel: stable - network: mainnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - build-docker: - name: "Docker push (lotus / candidate / mainnet)" - image: lotus - channel: candidate - network: mainnet - push: true - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - - build-docker: - name: "Docker push (lotus / master / mainnet)" - image: lotus - channel: master - network: mainnet - push: true - filters: - branches: - only: - - master - - build-docker: - name: "Docker build (lotus / mainnet)" - image: lotus - network: mainnet - push: false - filters: - branches: - only: - - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - [[- range .Networks]] - - build-docker: - name: "Docker (lotus-all-in-one / nightly / [[.]])" - image: lotus-all-in-one - channel: nightly - network: [[.]] - push: true - [[- end]] diff --git a/.github/actions/export-circle-env/action.yml b/.github/actions/export-circle-env/action.yml deleted file mode 100644 index ded4bbb05..000000000 --- a/.github/actions/export-circle-env/action.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Export Circle Env -description: Export CircleCI environment variables for Filecoin Lotus - -runs: - using: composite - steps: - - run: | - if [[ "$GITHUB_REF" == refs/tags/* ]]; then - echo "CIRCLE_TAG=${GITHUB_REF#refs/tags/}" | tee -a $GITHUB_ENV - fi - echo "CIRCLE_PROJECT_USERNAME=$GITHUB_REPOSITORY_OWNER" | tee -a $GITHUB_ENV - echo "CIRCLE_PROJECT_REPONAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" | tee -a $GITHUB_ENV - echo "CIRCLE_SHA1=$GITHUB_SHA" | tee -a $GITHUB_ENV - shell: bash diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9e6bf59f1..f6afe68bf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,9 +35,7 @@ jobs: - image: lotus network: mainnet env: - # Do not publish until CircleCI is deprecated - PUBLISH: false - # PUBLISH: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }} + PUBLISH: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }} steps: - id: channel env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd8a9cfbc..797473bb3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,6 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'recursive' - - uses: ./.github/actions/export-circle-env - uses: ./.github/actions/install-system-dependencies - uses: ./.github/actions/install-go - env: @@ -68,9 +67,7 @@ jobs: runs-on: ubuntu-latest needs: [build] env: - # Do not publish until CircleCI is deprecated - PUBLISH: false - # PUBLISH: ${{ startsWith(github.ref, 'refs/tags/') }} + PUBLISH: ${{ startsWith(github.ref, 'refs/tags/') }} steps: - uses: actions/checkout@v4 with: @@ -88,7 +85,6 @@ jobs: with: name: lotus-macOS-ARM64 path: darwin_arm64 - - uses: ./.github/actions/export-circle-env - uses: ./.github/actions/install-go - uses: ipfs/download-ipfs-distribution-action@v1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f63e4c42b..4553158f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,14 +58,6 @@ jobs: # to extend the available runner capacity (60 default hosted runners). # We use self-hosted 4xlarge (16 CPU, 32 RAM; and 2xlarge - 8 CPU, 16 RAM) self-hosted # to support resource intensive jobs. - # In CircleCI, the jobs defaulted to running on medium+ runners (3 CPU, 6 RAM). - # The following jobs were scheduled to run on 2xlarge runners (16 CPU, 32 RAM): - # - itest-deals_concurrent (✅) - # - itest-sector_pledge (✅) - # - itest-wdpost_worker_config (❌) - # - itest-worker (✅) - # - unit-cli (❌) - # - unit-rest (❌) runners: | { "itest-deals_concurrent": ["self-hosted", "linux", "x64", "4xlarge"], @@ -122,15 +114,9 @@ jobs: "unit-node": ["self-hosted", "linux", "x64", "xlarge"] } # A list of test groups that require YugabyteDB to be running - # In CircleCI, all jobs had yugabytedb running as a sidecar. yugabytedb: | ["itest-harmonydb", "itest-harmonytask", "itest-curio"] # A list of test groups that require Proof Parameters to be fetched - # In CircleCI, only the following jobs had get-params set: - # - unit-cli (✅) - # - unit-storage (✅) - # - itest-sector_pledge (✅) - # - itest-wdpost (✅) parameters: | [ "conformance", diff --git a/.goreleaser.yaml b/.goreleaser.yaml index f855ee969..bf6200714 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -27,7 +27,7 @@ builds: - goos: linux goarch: arm64 prebuilt: - path: '{{ if index .Env "GITHUB_WORKSPACE" }}{{ .Env.GITHUB_WORKSPACE }}{{ else }}/tmp/workspace{{ end }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus' + path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus' - id: lotus-miner binary: lotus-miner builder: prebuilt @@ -43,7 +43,7 @@ builds: - goos: linux goarch: arm64 prebuilt: - path: '{{ if index .Env "GITHUB_WORKSPACE" }}{{ .Env.GITHUB_WORKSPACE }}{{ else }}/tmp/workspace{{ end }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner' + path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-miner' - id: lotus-worker binary: lotus-worker builder: prebuilt @@ -59,7 +59,7 @@ builds: - goos: linux goarch: arm64 prebuilt: - path: '{{ if index .Env "GITHUB_WORKSPACE" }}{{ .Env.GITHUB_WORKSPACE }}{{ else }}/tmp/workspace{{ end }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker' + path: '{{ .Env.GITHUB_WORKSPACE }}/{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ . }}{{ end }}/lotus-worker' archives: - id: primary diff --git a/Makefile b/Makefile index f37e25750..901c8dc00 100644 --- a/Makefile +++ b/Makefile @@ -343,7 +343,7 @@ actors-code-gen: $(GOCC) run ./chain/actors/agen $(GOCC) fmt ./... -actors-gen: actors-code-gen +actors-gen: actors-code-gen $(GOCC) run ./scripts/fiximports .PHONY: actors-gen @@ -406,7 +406,7 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin fiximports: $(GOCC) run ./scripts/fiximports -gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen circleci +gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen $(GOCC) run ./scripts/fiximports @echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO RUN 'make docsgen-cli'" .PHONY: gen @@ -428,9 +428,6 @@ docsgen-cli: lotus lotus-miner lotus-worker curio sptool print-%: @echo $*=$($*) -circleci: - go generate -x ./.circleci - ### Curio devnet images curio_docker_user?=curio curio_base_image=$(curio_docker_user)/curio-all-in-one:latest-debug diff --git a/README.md b/README.md index dd4ff3b54..561b20166 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,12 @@

Project Lotus - 莲

- - - - +![example workflow](https://github.com/github/docs/actions/workflows/main.yml/badge.svg) + + + + +

@@ -19,7 +21,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation > Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases). - + For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme. ## Reporting a Vulnerability @@ -96,7 +98,7 @@ Once all the dependencies are installed, you can build and install the Lotus sui git clone https://github.com/filecoin-project/lotus.git cd lotus/ ``` - + Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the latest release[ here](https://github.com/filecoin-project/lotus/releases). 2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases). diff --git a/scripts/publish-checksums.sh b/scripts/publish-checksums.sh index 346ca9b4c..8e5cdfe35 100755 --- a/scripts/publish-checksums.sh +++ b/scripts/publish-checksums.sh @@ -9,6 +9,11 @@ if [ -z "${GITHUB_TOKEN}" ]; then exit 1 fi +if [ "$GITHUB_REF" != refs/tags/* ]; then + echo "$GITHUB_REF is not a tag, publish failed" + exit 1 +fi + REQUIRED=( "jq" "curl" @@ -18,12 +23,14 @@ do command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" done +GITHUB_TAG="${GITHUB_REF#refs/tags/}" + #see if the release already exists by tag RELEASE_RESPONSE=` curl \ --fail \ --header "Authorization: token ${GITHUB_TOKEN}" \ - "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}" + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${GITHUB_TAG}" ` RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` @@ -32,16 +39,16 @@ if [ "${RELEASE_ID}" = "null" ]; then COND_CREATE_DISCUSSION="" PRERELEASE=true - if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ ${GITHUB_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\"," PRERELEASE=false fi RELEASE_DATA="{ - \"tag_name\": \"${CIRCLE_TAG}\", - \"target_commitish\": \"${CIRCLE_SHA1}\", + \"tag_name\": \"${GITHUB_TAG}\", + \"target_commitish\": \"${GITHUB_SHA}\", ${COND_CREATE_DISCUSSION} - \"name\": \"${CIRCLE_TAG}\", + \"name\": \"${GITHUB_TAG}\", \"body\": \"\", \"prerelease\": ${PRERELEASE} }" @@ -54,7 +61,7 @@ if [ "${RELEASE_ID}" = "null" ]; then --header "Authorization: token ${GITHUB_TOKEN}" \ --header "Content-Type: application/json" \ --data "${RELEASE_DATA}" \ - "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases" + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases" ` else echo "release already exists" diff --git a/scripts/version-check.sh b/scripts/version-check.sh index 4f424ca0c..20aeda4cc 100755 --- a/scripts/version-check.sh +++ b/scripts/version-check.sh @@ -32,8 +32,8 @@ function validate_lotus_version_matches_tag(){ _lotus_path=$1 -if [[ ! -z "${CIRCLE_TAG}" ]]; then - validate_lotus_version_matches_tag "${_lotus_path}" "${CIRCLE_TAG}" +if [[ "$GITHUB_REF" != refs/tags/* ]]; then + validate_lotus_version_matches_tag "${_lotus_path}" "${GITHUB_REF#refs/tags/}" else - echo "No CI tag found. Skipping version check." + echo "$GITHUB_REF is not a tag, skipping version check" fi diff --git a/storage/sealer/README.md b/storage/sealer/README.md index 83fa3ea5f..b90a9ebd0 100644 --- a/storage/sealer/README.md +++ b/storage/sealer/README.md @@ -1,7 +1,6 @@ # sector-storage [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![CircleCI](https://circleci.com/gh/filecoin-project/sector-storage.svg?style=svg)](https://circleci.com/gh/filecoin-project/sector-storage) [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) > a concrete implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface @@ -35,7 +34,7 @@ The Local store implements SectorProvider for paths mounted in the local filesystem. Paths can be shared between nodes, and support shared filesystems such as NFS. -stores.Local implements all native filesystem-related operations +stores.Local implements all native filesystem-related operations ### `paths.Remote`