From 51e51d1b902fa93def7a387f9071c75bcc29b5a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 22 Jun 2021 20:52:14 +0100 Subject: [PATCH 01/11] dynamic circleci config. --- .circleci/config.yml | 972 +---------------------------------------- .circleci/gen.go | 124 ++++++ .circleci/go.mod | 1 + .circleci/template.yml | 915 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 1054 insertions(+), 958 deletions(-) create mode 100644 .circleci/gen.go create mode 100644 .circleci/go.mod create mode 100644 .circleci/template.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index d76be4bdc..b2de04b79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,972 +1,28 @@ version: 2.1 +setup: true orbs: + continuation: circleci/continuation@0.2.0 go: gotest/tools@0.0.13 - aws-cli: circleci/aws-cli@1.3.2 - packer: salaxander/packer@0.0.3 - executors: golang: docker: - image: circleci/golang:1.16.4 - resource_class: 2xlarge - ubuntu: - docker: - - image: ubuntu:20.04 - -commands: - install-deps: +jobs: + generate-config: + executor: golang steps: - go/install-ssh - - go/install: {package: git} - prepare: - parameters: - linux: - default: true - description: is a linux build environment? - type: boolean - darwin: - default: false - description: is a darwin build environment? - type: boolean - steps: - - checkout - - git_fetch_all_tags - - checkout - - when: - condition: << parameters.linux >> - steps: - - run: sudo apt-get update - - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev - - run: git submodule sync - - run: git submodule update --init - download-params: - steps: - - restore_cache: - name: Restore parameters cache - keys: - - 'v25-2k-lotus-params' - paths: - - /var/tmp/filecoin-proof-parameters/ - - run: ./lotus fetch-params 2048 - - save_cache: - name: Save parameters cache - key: 'v25-2k-lotus-params' - paths: - - /var/tmp/filecoin-proof-parameters/ - install_ipfs: - steps: - - run: | - apt update - apt install -y wget - wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz - wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512 - if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ] - then - echo "ipfs failed checksum check" - exit 1 - fi - tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz - mv go-ipfs/ipfs /usr/local/bin/ipfs - chmod +x /usr/local/bin/ipfs - git_fetch_all_tags: - steps: - - run: - name: fetch all tags - command: | - git fetch --all - -jobs: - mod-tidy-check: - executor: golang - steps: - - install-deps - - prepare - - go/mod-tidy-check - - build-all: - executor: golang - steps: - - install-deps - - prepare - - run: sudo apt-get update - - run: sudo apt-get install npm - - run: - command: make buildall - - store_artifacts: - path: lotus - - store_artifacts: - path: lotus-miner - - store_artifacts: - path: lotus-worker - - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/ - - persist_to_workspace: - root: "." - paths: - - linux - - build-debug: - executor: golang - steps: - - install-deps - - prepare - - run: - command: make debug - - test: &test - description: | - Run tests with gotestsum. - parameters: &test-params - executor: - type: executor - default: golang - go-test-flags: - type: string - default: "-timeout 30m" - description: Flags passed to go test. - packages: - type: string - default: "./..." - description: Import paths of packages to be tested. - winpost-test: - type: string - default: "0" - deadline-test: - type: string - default: "0" - proofs-log-test: - type: string - default: "0" - test-suite-name: - type: string - default: unit - description: Test suite name to report to CircleCI. - gotestsum-format: - type: string - default: pkgname-and-test-fails - description: gotestsum format. https://github.com/gotestyourself/gotestsum#format - coverage: - type: string - default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/... - description: Coverage flag. Set to the empty string to disable. - codecov-upload: - type: boolean - default: false - description: | - Upload coverage report to https://codecov.io/. Requires the codecov API token to be - set as an environment variable for private projects. - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: - command: make deps lotus - no_output_timeout: 30m - - download-params - - go/install-gotestsum: - gobin: $HOME/.local/bin - version: 0.5.2 - - run: - name: go test - environment: - LOTUS_TEST_WINDOW_POST: << parameters.winpost-test >> - LOTUS_TEST_DEADLINE_TOGGLING: << parameters.deadline-test >> - TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> - SKIP_CONFORMANCE: "1" - command: | - mkdir -p /tmp/test-reports/<< parameters.test-suite-name >> - mkdir -p /tmp/test-artifacts - gotestsum \ - --format << parameters.gotestsum-format >> \ - --junitfile /tmp/test-reports/<< parameters.test-suite-name >>/junit.xml \ - --jsonfile /tmp/test-artifacts/<< parameters.test-suite-name >>.json \ - -- \ - << parameters.coverage >> \ - << parameters.go-test-flags >> \ - << parameters.packages >> - no_output_timeout: 30m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-artifacts/<< parameters.test-suite-name >>.json - - when: - condition: << parameters.codecov-upload >> - steps: - - go/install: {package: bash} - - go/install: {package: curl} - - run: - shell: /bin/bash -eo pipefail - command: | - bash <(curl -s https://codecov.io/bash) - - test-chain: - <<: *test - test-node: - <<: *test - test-storage: - <<: *test - test-cli: - <<: *test - test-short: - <<: *test - test-window-post: - <<: *test - test-window-post-dispute: - <<: *test - test-deadline-toggling: - <<: *test - test-terminate: - <<: *test - check-proofs-multicore-sdr: - <<: *test - test-conformance: - description: | - Run tests using a corpus of interoperable test vectors for Filecoin - implementations to test their correctness and compliance with the Filecoin - specifications. - parameters: - <<: *test-params - vectors-branch: - type: string - default: "" - description: | - Branch on github.com/filecoin-project/test-vectors to checkout and - test with. If empty (the default) the commit defined by the git - submodule is used. - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: - command: make deps lotus - no_output_timeout: 30m - - download-params - - when: - condition: - not: - equal: [ "", << parameters.vectors-branch >> ] - steps: - - run: - name: checkout vectors branch - command: | - cd extern/test-vectors - git fetch - git checkout origin/<< parameters.vectors-branch >> - - go/install-gotestsum: - gobin: $HOME/.local/bin - version: 0.5.2 - - run: - name: install statediff globally - command: | - ## statediff is optional; we succeed even if compilation fails. - mkdir -p /tmp/statediff - git clone https://github.com/filecoin-project/statediff.git /tmp/statediff - cd /tmp/statediff - go install ./cmd/statediff || exit 0 - - run: - name: go test - environment: - SKIP_CONFORMANCE: "0" - command: | - mkdir -p /tmp/test-reports - mkdir -p /tmp/test-artifacts - gotestsum \ - --format pkgname-and-test-fails \ - --junitfile /tmp/test-reports/junit.xml \ - -- \ - -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ - go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html - no_output_timeout: 30m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-artifacts/conformance-coverage.html - build-ntwk-calibration: - description: | - Compile lotus binaries for the calibration network - parameters: - <<: *test-params - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: make calibnet - - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet - - persist_to_workspace: - root: "." - paths: - - linux-calibrationnet - build-ntwk-butterfly: - description: | - Compile lotus binaries for the butterfly network - parameters: - <<: *test-params - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: make butterflynet - - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet - - persist_to_workspace: - root: "." - paths: - - linux-butterflynet - build-ntwk-nerpa: - description: | - Compile lotus binaries for the nerpa network - parameters: - <<: *test-params - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: make nerpanet - - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet - - persist_to_workspace: - root: "." - paths: - - linux-nerpanet - build-lotus-soup: - description: | - Compile `lotus-soup` Testground test plan - parameters: - <<: *test-params - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: cd extern/filecoin-ffi && make - - run: - name: "go get lotus@master" - command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy - - run: - name: "build lotus-soup testplan" - command: pushd testplans/lotus-soup && go build -tags=testground . - trigger-testplans: - description: | - Trigger `lotus-soup` test cases on TaaS - parameters: - <<: *test-params - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: - name: "download testground" - command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli - - run: - name: "prepare .env.toml" - command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml - - run: - name: "prepare testground home dir and link test plans" - command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync - - run: - name: "go get lotus@master" - command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master - - run: - name: "trigger deals baseline testplan on taas" - command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - - run: - name: "trigger payment channel stress testplan on taas" - command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - - run: - name: "trigger graphsync testplan on taas" - command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - - - build-macos: - description: build darwin lotus binary - macos: - xcode: "10.0.0" - working_directory: ~/go/src/github.com/filecoin-project/lotus - steps: - - prepare: - linux: false - darwin: true - - run: - name: Install go - command: | - curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \ - sudo installer -pkg go1.16.4.darwin-amd64.pkg -target / - - run: - name: Install pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config - - run: go version - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: - name: Install jq - command: | - curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq - chmod +x /usr/local/bin/jq - - run: - name: Install hwloc - command: | - mkdir ~/hwloc - curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz - cd ~/hwloc - tar -xvzpf hwloc-2.4.1.tar.gz - cd hwloc-2.4.1 - ./configure && make && sudo make install - - restore_cache: - name: restore cargo cache - key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - - install-deps - - run: - command: make build - no_output_timeout: 30m - - store_artifacts: - path: lotus - - store_artifacts: - path: lotus-miner - - store_artifacts: - path: lotus-worker - - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ - - persist_to_workspace: - root: "." - paths: - - darwin - - save_cache: - name: save cargo cache - key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - paths: - - "~/.rustup" - - "~/.cargo" - - build-appimage: - machine: - image: ubuntu-2004:202104-01 - steps: - - checkout - - attach_workspace: - at: "." - - run: - name: install appimage-builder - command: | - # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html - sudo apt update - sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace - sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage - sudo chmod +x /usr/local/bin/appimagetool - sudo pip3 install appimage-builder - - run: - name: install lotus dependencies - command: sudo apt install ocl-icd-opencl-dev libhwloc-dev - - run: - name: build appimage - command: | - sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml - make appimage - - run: - name: prepare workspace - command: | - mkdir appimage - mv Lotus-latest-x86_64.AppImage appimage - - persist_to_workspace: - root: "." - paths: - - appimage - - - gofmt: - executor: golang - steps: - - install-deps - - prepare - - run: - command: "! go fmt ./... 2>&1 | read" - - gen-check: - executor: golang - steps: - - install-deps - - prepare - - run: make deps - - run: go install golang.org/x/tools/cmd/goimports - - run: go install github.com/hannahhoward/cbor-gen-for - - run: make gen - - run: git --no-pager diff - - run: git --no-pager diff --quiet - - run: make docsgen-cli - - run: git --no-pager diff - - run: git --no-pager diff --quiet - - docs-check: - executor: golang - steps: - - install-deps - - prepare - - run: go install golang.org/x/tools/cmd/goimports - - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full - - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner - - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker - - run: make deps - - run: make docsgen - - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full - - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner - - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker - - run: git --no-pager diff - - run: diff ../pre-openrpc-full ../post-openrpc-full - - run: diff ../pre-openrpc-miner ../post-openrpc-miner - - run: diff ../pre-openrpc-worker ../post-openrpc-worker - - run: git --no-pager diff --quiet - - lint: &lint - description: | - Run golangci-lint. - parameters: - executor: - type: executor - default: golang - golangci-lint-version: - type: string - default: 1.27.0 - concurrency: - type: string - default: '2' - description: | - Concurrency used to run linters. Defaults to 2 because NumCPU is not - aware of container CPU limits. - args: - type: string - default: '' - description: | - Arguments to pass to golangci-lint - executor: << parameters.executor >> - steps: - - install-deps - - prepare - - run: - command: make deps - no_output_timeout: 30m - - go/install-golangci-lint: - gobin: $HOME/.local/bin - version: << parameters.golangci-lint-version >> - - run: - name: Lint - command: | - $HOME/.local/bin/golangci-lint run -v --timeout 2m \ - --concurrency << parameters.concurrency >> << parameters.args >> - lint-all: - <<: *lint - - publish: - description: publish binary artifacts - executor: ubuntu - steps: - - run: - name: Install git jq curl - command: apt update && apt install -y git jq curl - - checkout - - git_fetch_all_tags - - checkout - - install_ipfs - - attach_workspace: - at: "." - - run: - name: Create bundles - command: ./scripts/build-bundle.sh - - run: - name: Publish release - command: ./scripts/publish-release.sh - - publish-snapcraft: - description: build and push snapcraft - machine: - image: ubuntu-2004:202104-01 - resource_class: 2xlarge - parameters: - channel: - type: string - default: "edge" - description: snapcraft channel - steps: + - go/install: { package: git } - checkout - run: - name: install snapcraft - command: sudo snap install snapcraft --classic - - run: - name: create snapcraft config file + name: generate pipeline command: | - mkdir -p ~/.config/snapcraft - echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg - - run: - name: build snap - command: snapcraft --use-lxd - - run: - name: publish snap - command: snapcraft push *.snap --release << parameters.channel >> - - - build-and-push-image: - description: build and push docker images to public AWS ECR registry - executor: aws-cli/default - parameters: - profile-name: - type: string - default: "default" - description: AWS profile name to be configured. - - aws-access-key-id: - type: env_var_name - default: AWS_ACCESS_KEY_ID - description: > - AWS access key id for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_ACCESS_KEY. - - aws-secret-access-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - description: > - AWS secret key for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_SECRET_ACCESS_KEY. - - region: - type: env_var_name - default: AWS_REGION - description: > - Name of env var storing your AWS region information, - defaults to AWS_REGION - - account-url: - type: env_var_name - default: AWS_ECR_ACCOUNT_URL - description: > - Env var storing Amazon ECR account URL that maps to an AWS account, - e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com - defaults to AWS_ECR_ACCOUNT_URL - - dockerfile: - type: string - default: Dockerfile - description: Name of dockerfile to use. Defaults to Dockerfile. - - path: - type: string - default: . - description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). - - extra-build-args: - type: string - default: "" - description: > - Extra flags to pass to docker build. For examples, see - https://docs.docker.com/engine/reference/commandline/build - - repo: - type: string - description: Name of an Amazon ECR repository - - tag: - type: string - default: "latest" - description: A comma-separated string containing docker image tags to build and push (default = latest) - - steps: - - run: - name: Confirm that environment variables are set - command: | - if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..." - circleci-agent step halt - fi - - - aws-cli/setup: - profile-name: <> - aws-access-key-id: <> - aws-secret-access-key: <> - aws-region: <> - - - run: - name: Log into Amazon ECR - command: | - aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - - - checkout - - - setup_remote_docker: - version: 19.03.13 - docker_layer_caching: false - - - run: - name: Build docker image - command: | - registry_id=$(echo $<> | sed "s;\..*;;g") - - docker_tag_args="" - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker_tag_args="$docker_tag_args -t $<>/<>:$tag" - done - - docker build \ - <<#parameters.extra-build-args>><><> \ - -f <>/<> \ - $docker_tag_args \ - <> - - - run: - name: Push image to Amazon ECR - command: | - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker push $<>/<>:${tag} - done - - publish-packer-mainnet: - description: build and push AWS IAM and DigitalOcean droplet. - executor: - name: packer/default - packer-version: 1.6.6 - steps: - - checkout - - attach_workspace: - at: "." - - packer/build: - template: tools/packer/lotus.pkr.hcl - args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" - publish-packer-calibrationnet: - description: build and push AWS IAM and DigitalOcean droplet. - executor: - name: packer/default - packer-version: 1.6.6 - steps: - - checkout - - attach_workspace: - at: "." - - packer/build: - template: tools/packer/lotus.pkr.hcl - args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" - publish-packer-butterflynet: - description: build and push AWS IAM and DigitalOcean droplet. - executor: - name: packer/default - packer-version: 1.6.6 - steps: - - checkout - - attach_workspace: - at: "." - - packer/build: - template: tools/packer/lotus.pkr.hcl - args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" - publish-packer-nerpanet: - description: build and push AWS IAM and DigitalOcean droplet. - executor: - name: packer/default - packer-version: 1.6.6 - steps: - - checkout - - attach_workspace: - at: "." - - packer/build: - template: tools/packer/lotus.pkr.hcl - args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" - + cd .circleci + go run ./gen.go $(pwd) > generated_config.yml + - continuation/continue: + parameters: '{}' + configuration_path: .circleci/generated_config.yml workflows: - version: 2.1 - ci: + setup-workflow: jobs: - - lint-all: - concurrency: "16" # expend all docker 2xlarge CPUs. - - mod-tidy-check - - gofmt - - gen-check - - docs-check - - test: - codecov-upload: true - test-suite-name: full - - test-chain: - codecov-upload: true - test-suite-name: chain - packages: "./chain/..." - - test-node: - codecov-upload: true - test-suite-name: node - packages: "./node/..." - - test-storage: - codecov-upload: true - test-suite-name: storage - packages: "./storage/... ./extern/..." - - test-cli: - codecov-upload: true - test-suite-name: cli - packages: "./cli/... ./cmd/... ./api/..." - - test-window-post: - codecov-upload: true - go-test-flags: "-run=TestWindowedPost" - winpost-test: "1" - test-suite-name: window-post - - test-window-post-dispute: - codecov-upload: true - go-test-flags: "-run=TestWindowPostDispute" - winpost-test: "1" - test-suite-name: window-post-dispute - - test-terminate: - codecov-upload: true - go-test-flags: "-run=TestTerminate" - winpost-test: "1" - test-suite-name: terminate - - test-deadline-toggling: - codecov-upload: true - go-test-flags: "-run=TestDeadlineToggling" - deadline-test: "1" - test-suite-name: deadline-toggling - - test-short: - go-test-flags: "--timeout 10m --short" - test-suite-name: short - filters: - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - check-proofs-multicore-sdr: - codecov-upload: true - go-test-flags: "-run=TestMulticoreSDR" - test-suite-name: multicore-sdr-check - packages: "./extern/sector-storage/ffiwrapper" - proofs-log-test: "1" - - test-conformance: - test-suite-name: conformance - packages: "./conformance" - - test-conformance: - name: test-conformance-bleeding-edge - test-suite-name: conformance-bleeding-edge - packages: "./conformance" - vectors-branch: master - - trigger-testplans: - filters: - branches: - only: - - master - - build-debug - - build-all: - requires: - - test-short - filters: - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-ntwk-calibration: - requires: - - test-short - filters: - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-ntwk-butterfly: - requires: - - test-short - filters: - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-ntwk-nerpa: - requires: - - test-short - filters: - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-lotus-soup - - build-macos: - requires: - - test-short - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-appimage: - requires: - - test-short - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish: - requires: - - build-all - - build-macos - - build-appimage - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-and-push-image: - dockerfile: Dockerfile.lotus - path: . - repo: lotus-dev - tag: '${CIRCLE_SHA1:0:8}' - - publish-packer-mainnet: - requires: - - build-all - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-packer-calibrationnet: - requires: - - build-ntwk-calibration - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-packer-butterflynet: - requires: - - build-ntwk-butterfly - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-packer-nerpanet: - requires: - - build-ntwk-nerpa - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish-snapcraft: - name: publish-snapcraft-stable - channel: stable - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - publish-snapcraft: - name: publish-snapcraft-nightly - channel: edge + - generate-config \ No newline at end of file diff --git a/.circleci/gen.go b/.circleci/gen.go new file mode 100644 index 000000000..db806b003 --- /dev/null +++ b/.circleci/gen.go @@ -0,0 +1,124 @@ +package main + +import ( + "embed" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" +) + +//go:embed template.yml +var templateFile embed.FS + +type ( + dirs = []string + suite = string +) + +// groupedUnitTests maps suite names to top-level directories that should be +// included in that suite. The program adds an implicit group "rest" that +// includes all other top-level directories. +var groupedUnitTests = map[suite]dirs{ + "unit-node": {"node"}, + "unit-storage": {"storage", "extern"}, + "unit-cli": {"cli", "cmd", "api"}, +} + +func main() { + if len(os.Args) != 2 { + panic("expected path to repo as argument") + } + + repo := os.Args[1] + + tmpl := template.New("template.yml") + tmpl.Delims("[[", "]]") + tmpl.Funcs(template.FuncMap{ + "stripSuffix": func(in string) string { + return strings.TrimSuffix(in, "_test.go") + }, + }) + tmpl = template.Must(tmpl.ParseFS(templateFile, "*")) + + // list all itests. + itests, err := filepath.Glob("./itests/*_test.go") + if err != nil { + panic(err) + } + + // strip the dir from all entries. + for i, f := range itests { + itests[i] = filepath.Base(f) + } + + // calculate the exclusion set of unit test directories to exclude because + // they are already included in a grouped suite. + var excluded = map[string]struct{}{} + for _, ss := range groupedUnitTests { + for _, s := range ss { + e, err := filepath.Abs(filepath.Join(repo, s)) + if err != nil { + panic(err) + } + excluded[e] = struct{}{} + } + } + + // all unit tests top-level dirs that are not itests, nor included in other suites. + var rest = map[string]struct{}{} + err = filepath.Walk(repo, func(path string, f os.FileInfo, err error) error { + // include all tests that aren't in the itests directory. + if strings.Contains(path, "itests") { + return filepath.SkipDir + } + // exclude all tests included in other suites + if f.IsDir() { + if _, ok := excluded[path]; ok { + return filepath.SkipDir + } + } + if strings.HasSuffix(path, "_test.go") { + rel, err := filepath.Rel(repo, path) + if err != nil { + panic(err) + } + // take the first directory + rest[strings.Split(rel, string(os.PathSeparator))[0]] = struct{}{} + } + return err + }) + if err != nil { + panic(err) + } + + // add other directories to a 'rest' suite. + for k := range rest { + groupedUnitTests["unit-rest"] = append(groupedUnitTests["unit-rest"], k) + } + + // form the input data. + type data struct { + ItestFiles []string + UnitSuites map[string]string + } + in := data{ + ItestFiles: itests, + UnitSuites: func() map[string]string { + ret := make(map[string]string) + for name, dirs := range groupedUnitTests { + for i, d := range dirs { + dirs[i] = fmt.Sprintf("./%s/...", d) // turn into package + } + ret[name] = strings.Join(dirs, " ") + } + return ret + }(), + } + + // execute the template. + if err := tmpl.Execute(os.Stdout, in); err != nil { + panic(err) + } +} diff --git a/.circleci/go.mod b/.circleci/go.mod new file mode 100644 index 000000000..9b92b8c78 --- /dev/null +++ b/.circleci/go.mod @@ -0,0 +1 @@ +module ".circleci" \ No newline at end of file diff --git a/.circleci/template.yml b/.circleci/template.yml new file mode 100644 index 000000000..fad4cdee5 --- /dev/null +++ b/.circleci/template.yml @@ -0,0 +1,915 @@ +version: 2.1 +orbs: + go: gotest/tools@0.0.13 + aws-cli: circleci/aws-cli@1.3.2 + packer: salaxander/packer@0.0.3 + +executors: + golang: + docker: + - image: circleci/golang:1.16.4 + resource_class: 2xlarge + ubuntu: + docker: + - image: ubuntu:20.04 + +commands: + install-deps: + steps: + - go/install-ssh + - go/install: {package: git} + prepare: + parameters: + linux: + default: true + description: is a linux build environment? + type: boolean + darwin: + default: false + description: is a darwin build environment? + type: boolean + steps: + - checkout + - git_fetch_all_tags + - checkout + - when: + condition: << parameters.linux >> + steps: + - run: sudo apt-get update + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - run: git submodule sync + - run: git submodule update --init + download-params: + steps: + - restore_cache: + name: Restore parameters cache + keys: + - 'v25-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + - run: ./lotus fetch-params 2048 + - save_cache: + name: Save parameters cache + key: 'v25-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + install_ipfs: + steps: + - run: | + apt update + apt install -y wget + wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz + wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512 + if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ] + then + echo "ipfs failed checksum check" + exit 1 + fi + tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz + mv go-ipfs/ipfs /usr/local/bin/ipfs + chmod +x /usr/local/bin/ipfs + git_fetch_all_tags: + steps: + - run: + name: fetch all tags + command: | + git fetch --all + +jobs: + mod-tidy-check: + executor: golang + steps: + - install-deps + - prepare + - go/mod-tidy-check + + build-all: + executor: golang + steps: + - install-deps + - prepare + - run: sudo apt-get update + - run: sudo apt-get install npm + - run: + command: make buildall + - store_artifacts: + path: lotus + - store_artifacts: + path: lotus-miner + - store_artifacts: + path: lotus-worker + - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/ + - persist_to_workspace: + root: "." + paths: + - linux + + build-debug: + executor: golang + steps: + - install-deps + - prepare + - run: + command: make debug + + test: + description: | + Run tests with gotestsum. + parameters: &test-params + executor: + type: executor + default: golang + go-test-flags: + type: string + default: "-timeout 30m" + description: Flags passed to go test. + target: + type: string + default: "./..." + description: Import paths of packages to be tested. + proofs-log-test: + type: string + default: "0" + suite: + type: string + default: unit + description: Test suite name to report to CircleCI. + gotestsum-format: + type: string + default: pkgname-and-test-fails + description: gotestsum format. https://github.com/gotestyourself/gotestsum#format + coverage: + type: string + default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/... + description: Coverage flag. Set to the empty string to disable. + codecov-upload: + type: boolean + default: true + description: | + Upload coverage report to https://codecov.io/. Requires the codecov API token to be + set as an environment variable for private projects. + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + command: make deps lotus + no_output_timeout: 30m + - download-params + - go/install-gotestsum: + gobin: $HOME/.local/bin + version: 0.5.2 + - run: + name: go test + environment: + TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> + SKIP_CONFORMANCE: "1" + command: | + mkdir -p /tmp/test-reports/<< parameters.suite >> + mkdir -p /tmp/test-artifacts + gotestsum \ + --format << parameters.gotestsum-format >> \ + --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ + --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ + -- \ + << parameters.coverage >> \ + << parameters.go-test-flags >> \ + << parameters.packages >> + no_output_timeout: 30m + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-artifacts/<< parameters.suite >>.json + - when: + condition: << parameters.codecov-upload >> + steps: + - go/install: {package: bash} + - go/install: {package: curl} + - run: + shell: /bin/bash -eo pipefail + command: | + bash <(curl -s https://codecov.io/bash) + + test-conformance: + description: | + Run tests using a corpus of interoperable test vectors for Filecoin + implementations to test their correctness and compliance with the Filecoin + specifications. + parameters: + <<: *test-params + vectors-branch: + type: string + default: "" + description: | + Branch on github.com/filecoin-project/test-vectors to checkout and + test with. If empty (the default) the commit defined by the git + submodule is used. + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + command: make deps lotus + no_output_timeout: 30m + - download-params + - when: + condition: + not: + equal: [ "", << parameters.vectors-branch >> ] + steps: + - run: + name: checkout vectors branch + command: | + cd extern/test-vectors + git fetch + git checkout origin/<< parameters.vectors-branch >> + - go/install-gotestsum: + gobin: $HOME/.local/bin + version: 0.5.2 + - run: + name: install statediff globally + command: | + ## statediff is optional; we succeed even if compilation fails. + mkdir -p /tmp/statediff + git clone https://github.com/filecoin-project/statediff.git /tmp/statediff + cd /tmp/statediff + go install ./cmd/statediff || exit 0 + - run: + name: go test + environment: + SKIP_CONFORMANCE: "0" + command: | + mkdir -p /tmp/test-reports + mkdir -p /tmp/test-artifacts + gotestsum \ + --format pkgname-and-test-fails \ + --junitfile /tmp/test-reports/junit.xml \ + -- \ + -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ + go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html + no_output_timeout: 30m + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-artifacts/conformance-coverage.html + build-ntwk-calibration: + description: | + Compile lotus binaries for the calibration network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make calibnet + - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet + - persist_to_workspace: + root: "." + paths: + - linux-calibrationnet + build-ntwk-butterfly: + description: | + Compile lotus binaries for the butterfly network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make butterflynet + - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet + - persist_to_workspace: + root: "." + paths: + - linux-butterflynet + build-ntwk-nerpa: + description: | + Compile lotus binaries for the nerpa network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make nerpanet + - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet + - persist_to_workspace: + root: "." + paths: + - linux-nerpanet + build-lotus-soup: + description: | + Compile `lotus-soup` Testground test plan + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: cd extern/filecoin-ffi && make + - run: + name: "go get lotus@master" + command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy + - run: + name: "build lotus-soup testplan" + command: pushd testplans/lotus-soup && go build -tags=testground . + trigger-testplans: + description: | + Trigger `lotus-soup` test cases on TaaS + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + name: "download testground" + command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli + - run: + name: "prepare .env.toml" + command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml + - run: + name: "prepare testground home dir and link test plans" + command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync + - run: + name: "go get lotus@master" + command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master + - run: + name: "trigger deals baseline testplan on taas" + command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH + - run: + name: "trigger payment channel stress testplan on taas" + command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH + - run: + name: "trigger graphsync testplan on taas" + command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH + + + build-macos: + description: build darwin lotus binary + macos: + xcode: "10.0.0" + working_directory: ~/go/src/github.com/filecoin-project/lotus + steps: + - prepare: + linux: false + darwin: true + - run: + name: Install go + command: | + curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \ + sudo installer -pkg go1.16.4.darwin-amd64.pkg -target / + - run: + name: Install pkg-config + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config + - run: go version + - run: + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + - run: + name: Install jq + command: | + curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq + chmod +x /usr/local/bin/jq + - run: + name: Install hwloc + command: | + mkdir ~/hwloc + curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz + cd ~/hwloc + tar -xvzpf hwloc-2.4.1.tar.gz + cd hwloc-2.4.1 + ./configure && make && sudo make install + - restore_cache: + name: restore cargo cache + key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} + - install-deps + - run: + command: make build + no_output_timeout: 30m + - store_artifacts: + path: lotus + - store_artifacts: + path: lotus-miner + - store_artifacts: + path: lotus-worker + - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ + - persist_to_workspace: + root: "." + paths: + - darwin + - save_cache: + name: save cargo cache + key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} + paths: + - "~/.rustup" + - "~/.cargo" + + build-appimage: + machine: + image: ubuntu-2004:202104-01 + steps: + - checkout + - attach_workspace: + at: "." + - run: + name: install appimage-builder + command: | + # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html + sudo apt update + sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace + sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage + sudo chmod +x /usr/local/bin/appimagetool + sudo pip3 install appimage-builder + - run: + name: install lotus dependencies + command: sudo apt install ocl-icd-opencl-dev libhwloc-dev + - run: + name: build appimage + command: | + sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml + make appimage + - run: + name: prepare workspace + command: | + mkdir appimage + mv Lotus-latest-x86_64.AppImage appimage + - persist_to_workspace: + root: "." + paths: + - appimage + + + gofmt: + executor: golang + steps: + - install-deps + - prepare + - run: + command: "! go fmt ./... 2>&1 | read" + + gen-check: + executor: golang + steps: + - install-deps + - prepare + - run: make deps + - run: go install golang.org/x/tools/cmd/goimports + - run: go install github.com/hannahhoward/cbor-gen-for + - run: make gen + - run: git --no-pager diff + - run: git --no-pager diff --quiet + - run: make docsgen-cli + - run: git --no-pager diff + - run: git --no-pager diff --quiet + + docs-check: + executor: golang + steps: + - install-deps + - prepare + - run: go install golang.org/x/tools/cmd/goimports + - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full + - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner + - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker + - run: make deps + - run: make docsgen + - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full + - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner + - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker + - run: git --no-pager diff + - run: diff ../pre-openrpc-full ../post-openrpc-full + - run: diff ../pre-openrpc-miner ../post-openrpc-miner + - run: diff ../pre-openrpc-worker ../post-openrpc-worker + - run: git --no-pager diff --quiet + + lint: &lint + description: | + Run golangci-lint. + parameters: + executor: + type: executor + default: golang + golangci-lint-version: + type: string + default: 1.27.0 + concurrency: + type: string + default: '2' + description: | + Concurrency used to run linters. Defaults to 2 because NumCPU is not + aware of container CPU limits. + args: + type: string + default: '' + description: | + Arguments to pass to golangci-lint + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + command: make deps + no_output_timeout: 30m + - go/install-golangci-lint: + gobin: $HOME/.local/bin + version: << parameters.golangci-lint-version >> + - run: + name: Lint + command: | + $HOME/.local/bin/golangci-lint run -v --timeout 2m \ + --concurrency << parameters.concurrency >> << parameters.args >> + lint-all: + <<: *lint + + publish: + description: publish binary artifacts + executor: ubuntu + steps: + - run: + name: Install git jq curl + command: apt update && apt install -y git jq curl + - checkout + - git_fetch_all_tags + - checkout + - install_ipfs + - attach_workspace: + at: "." + - run: + name: Create bundles + command: ./scripts/build-bundle.sh + - run: + name: Publish release + command: ./scripts/publish-release.sh + + publish-snapcraft: + description: build and push snapcraft + machine: + image: ubuntu-2004:202104-01 + resource_class: 2xlarge + parameters: + channel: + type: string + default: "edge" + description: snapcraft channel + steps: + - checkout + - run: + name: install snapcraft + command: sudo snap install snapcraft --classic + - run: + name: create snapcraft config file + command: | + mkdir -p ~/.config/snapcraft + echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg + - run: + name: build snap + command: snapcraft --use-lxd + - run: + name: publish snap + command: snapcraft push *.snap --release << parameters.channel >> + + build-and-push-image: + description: build and push docker images to public AWS ECR registry + executor: aws-cli/default + parameters: + profile-name: + type: string + default: "default" + description: AWS profile name to be configured. + + aws-access-key-id: + type: env_var_name + default: AWS_ACCESS_KEY_ID + description: > + AWS access key id for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_ACCESS_KEY. + + aws-secret-access-key: + type: env_var_name + default: AWS_SECRET_ACCESS_KEY + description: > + AWS secret key for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_SECRET_ACCESS_KEY. + + region: + type: env_var_name + default: AWS_REGION + description: > + Name of env var storing your AWS region information, + defaults to AWS_REGION + + account-url: + type: env_var_name + default: AWS_ECR_ACCOUNT_URL + description: > + Env var storing Amazon ECR account URL that maps to an AWS account, + e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com + defaults to AWS_ECR_ACCOUNT_URL + + dockerfile: + type: string + default: Dockerfile + description: Name of dockerfile to use. Defaults to Dockerfile. + + path: + type: string + default: . + description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). + + extra-build-args: + type: string + default: "" + description: > + Extra flags to pass to docker build. For examples, see + https://docs.docker.com/engine/reference/commandline/build + + repo: + type: string + description: Name of an Amazon ECR repository + + tag: + type: string + default: "latest" + description: A comma-separated string containing docker image tags to build and push (default = latest) + + steps: + - run: + name: Confirm that environment variables are set + command: | + if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..." + circleci-agent step halt + fi + + - aws-cli/setup: + profile-name: <> + aws-access-key-id: <> + aws-secret-access-key: <> + aws-region: <> + + - run: + name: Log into Amazon ECR + command: | + aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> + + - checkout + + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: false + + - run: + name: Build docker image + command: | + registry_id=$(echo $<> | sed "s;\..*;;g") + + docker_tag_args="" + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker_tag_args="$docker_tag_args -t $<>/<>:$tag" + done + + docker build \ + <<#parameters.extra-build-args>><><> \ + -f <>/<> \ + $docker_tag_args \ + <> + + - run: + name: Push image to Amazon ECR + command: | + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker push $<>/<>:${tag} + done + + publish-packer-mainnet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" + publish-packer-calibrationnet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" + publish-packer-butterflynet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" + publish-packer-nerpanet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" + +workflows: + version: 2.1 + ci: + jobs: + - lint-all: + concurrency: "16" # expend all docker 2xlarge CPUs. + - mod-tidy-check + - gofmt + - gen-check + - docs-check + + [[- range $file := .ItestFiles -]] + [[ with $name := $file | stripSuffix ]] + - test: + name: test-itest-[[ $name ]] + suite: itest-[[ $name ]] + target: "./itests/[[ $file ]]" + [[ end ]] + [[- end -]] + + [[range $suite, $pkgs := .UnitSuites]] + - test: + name: test-[[ $suite ]] + suite: utest-[[ $suite ]] + target: "[[ $pkgs ]]" + [[- end]] + + - check-proofs-multicore-sdr: + go-test-flags: "-run=TestMulticoreSDR" + suite: multicore-sdr-check + target: "./extern/sector-storage/ffiwrapper" + proofs-log-test: "1" + - test-conformance: + suite: conformance + codecov-upload: false + target: "./conformance" + - test-conformance: + name: test-conformance-bleeding-edge + codecov-upload: false + suite: conformance-bleeding-edge + target: "./conformance" + vectors-branch: master + - trigger-testplans: + filters: + branches: + only: + - master + - build-debug + - build-all: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-ntwk-calibration: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-ntwk-butterfly: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-ntwk-nerpa: + requires: + - test-short + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-lotus-soup + - build-macos: + requires: + - test-short + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-appimage: + requires: + - test-short + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish: + requires: + - build-all + - build-macos + - build-appimage + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-dev + tag: '${CIRCLE_SHA1:0:8}' + - publish-packer-mainnet: + requires: + - build-all + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-calibrationnet: + requires: + - build-ntwk-calibration + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-butterflynet: + requires: + - build-ntwk-butterfly + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-nerpanet: + requires: + - build-ntwk-nerpa + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-snapcraft: + name: publish-snapcraft-stable + channel: stable + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + + nightly: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + jobs: + - publish-snapcraft: + name: publish-snapcraft-nightly + channel: edge From ec6c394de76489e08bcabe3e567a6d8c446a5e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 22 Jun 2021 21:03:05 +0100 Subject: [PATCH 02/11] remove dynamic circleci. --- .circleci/config.yml | 987 ++++++++++++++++++++++++++++++++++++++++- .circleci/gen.go | 12 +- .circleci/go.mod | 1 - .circleci/template.yml | 17 +- 4 files changed, 985 insertions(+), 32 deletions(-) delete mode 100644 .circleci/go.mod diff --git a/.circleci/config.yml b/.circleci/config.yml index b2de04b79..f41a8c513 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,28 +1,987 @@ version: 2.1 -setup: true orbs: - continuation: circleci/continuation@0.2.0 go: gotest/tools@0.0.13 + aws-cli: circleci/aws-cli@1.3.2 + packer: salaxander/packer@0.0.3 + executors: golang: docker: - image: circleci/golang:1.16.4 -jobs: - generate-config: - executor: golang + resource_class: 2xlarge + ubuntu: + docker: + - image: ubuntu:20.04 + +commands: + install-deps: steps: - go/install-ssh - - go/install: { package: git } + - go/install: {package: git} + prepare: + parameters: + linux: + default: true + description: is a linux build environment? + type: boolean + darwin: + default: false + description: is a darwin build environment? + type: boolean + steps: + - checkout + - git_fetch_all_tags + - checkout + - when: + condition: << parameters.linux >> + steps: + - run: sudo apt-get update + - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - run: git submodule sync + - run: git submodule update --init + download-params: + steps: + - restore_cache: + name: Restore parameters cache + keys: + - 'v25-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + - run: ./lotus fetch-params 2048 + - save_cache: + name: Save parameters cache + key: 'v25-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + install_ipfs: + steps: + - run: | + apt update + apt install -y wget + wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz + wget https://github.com/ipfs/go-ipfs/releases/download/v0.4.22/go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512 + if [ "$(sha512sum go-ipfs_v0.4.22_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.4.22_linux-amd64.tar.gz.sha512)" ] + then + echo "ipfs failed checksum check" + exit 1 + fi + tar -xf go-ipfs_v0.4.22_linux-amd64.tar.gz + mv go-ipfs/ipfs /usr/local/bin/ipfs + chmod +x /usr/local/bin/ipfs + git_fetch_all_tags: + steps: + - run: + name: fetch all tags + command: | + git fetch --all + +jobs: + mod-tidy-check: + executor: golang + steps: + - install-deps + - prepare + - go/mod-tidy-check + + build-all: + executor: golang + steps: + - install-deps + - prepare + - run: sudo apt-get update + - run: sudo apt-get install npm + - run: + command: make buildall + - store_artifacts: + path: lotus + - store_artifacts: + path: lotus-miner + - store_artifacts: + path: lotus-worker + - run: mkdir linux && mv lotus lotus-miner lotus-worker linux/ + - persist_to_workspace: + root: "." + paths: + - linux + + build-debug: + executor: golang + steps: + - install-deps + - prepare + - run: + command: make debug + + test: + description: | + Run tests with gotestsum. + parameters: &test-params + executor: + type: executor + default: golang + go-test-flags: + type: string + default: "-timeout 30m" + description: Flags passed to go test. + target: + type: string + default: "./..." + description: Import paths of packages to be tested. + proofs-log-test: + type: string + default: "0" + suite: + type: string + default: unit + description: Test suite name to report to CircleCI. + gotestsum-format: + type: string + default: pkgname-and-test-fails + description: gotestsum format. https://github.com/gotestyourself/gotestsum#format + coverage: + type: string + default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/... + description: Coverage flag. Set to the empty string to disable. + codecov-upload: + type: boolean + default: true + description: | + Upload coverage report to https://codecov.io/. Requires the codecov API token to be + set as an environment variable for private projects. + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + command: make deps lotus + no_output_timeout: 30m + - download-params + - go/install-gotestsum: + gobin: $HOME/.local/bin + version: 0.5.2 + - run: + name: go test + environment: + TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >> + SKIP_CONFORMANCE: "1" + command: | + mkdir -p /tmp/test-reports/<< parameters.suite >> + mkdir -p /tmp/test-artifacts + gotestsum \ + --format << parameters.gotestsum-format >> \ + --junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \ + --jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \ + -- \ + << parameters.coverage >> \ + << parameters.go-test-flags >> \ + << parameters.target >> + no_output_timeout: 30m + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-artifacts/<< parameters.suite >>.json + - when: + condition: << parameters.codecov-upload >> + steps: + - go/install: {package: bash} + - go/install: {package: curl} + - run: + shell: /bin/bash -eo pipefail + command: | + bash <(curl -s https://codecov.io/bash) + + test-conformance: + description: | + Run tests using a corpus of interoperable test vectors for Filecoin + implementations to test their correctness and compliance with the Filecoin + specifications. + parameters: + <<: *test-params + vectors-branch: + type: string + default: "" + description: | + Branch on github.com/filecoin-project/test-vectors to checkout and + test with. If empty (the default) the commit defined by the git + submodule is used. + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + command: make deps lotus + no_output_timeout: 30m + - download-params + - when: + condition: + not: + equal: [ "", << parameters.vectors-branch >> ] + steps: + - run: + name: checkout vectors branch + command: | + cd extern/test-vectors + git fetch + git checkout origin/<< parameters.vectors-branch >> + - go/install-gotestsum: + gobin: $HOME/.local/bin + version: 0.5.2 + - run: + name: install statediff globally + command: | + ## statediff is optional; we succeed even if compilation fails. + mkdir -p /tmp/statediff + git clone https://github.com/filecoin-project/statediff.git /tmp/statediff + cd /tmp/statediff + go install ./cmd/statediff || exit 0 + - run: + name: go test + environment: + SKIP_CONFORMANCE: "0" + command: | + mkdir -p /tmp/test-reports + mkdir -p /tmp/test-artifacts + gotestsum \ + --format pkgname-and-test-fails \ + --junitfile /tmp/test-reports/junit.xml \ + -- \ + -v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/ + go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html + no_output_timeout: 30m + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-artifacts/conformance-coverage.html + build-ntwk-calibration: + description: | + Compile lotus binaries for the calibration network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make calibnet + - run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet + - persist_to_workspace: + root: "." + paths: + - linux-calibrationnet + build-ntwk-butterfly: + description: | + Compile lotus binaries for the butterfly network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make butterflynet + - run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet + - persist_to_workspace: + root: "." + paths: + - linux-butterflynet + build-ntwk-nerpa: + description: | + Compile lotus binaries for the nerpa network + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: make nerpanet + - run: mkdir linux-nerpanet && mv lotus lotus-miner lotus-worker linux-nerpanet + - persist_to_workspace: + root: "." + paths: + - linux-nerpanet + build-lotus-soup: + description: | + Compile `lotus-soup` Testground test plan + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: cd extern/filecoin-ffi && make + - run: + name: "go get lotus@master" + command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy + - run: + name: "build lotus-soup testplan" + command: pushd testplans/lotus-soup && go build -tags=testground . + trigger-testplans: + description: | + Trigger `lotus-soup` test cases on TaaS + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + name: "download testground" + command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli + - run: + name: "prepare .env.toml" + command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml + - run: + name: "prepare testground home dir and link test plans" + command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup && ln -s $(pwd)/testplans/graphsync $HOME/testground/plans/graphsync + - run: + name: "go get lotus@master" + command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master + - run: + name: "trigger deals baseline testplan on taas" + command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH + - run: + name: "trigger payment channel stress testplan on taas" + command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH + - run: + name: "trigger graphsync testplan on taas" + command: ~/testground-cli run composition -f $HOME/testground/plans/graphsync/_compositions/stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH + + + build-macos: + description: build darwin lotus binary + macos: + xcode: "10.0.0" + working_directory: ~/go/src/github.com/filecoin-project/lotus + steps: + - prepare: + linux: false + darwin: true + - run: + name: Install go + command: | + curl -O https://dl.google.com/go/go1.16.4.darwin-amd64.pkg && \ + sudo installer -pkg go1.16.4.darwin-amd64.pkg -target / + - run: + name: Install pkg-config + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config + - run: go version + - run: + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + - run: + name: Install jq + command: | + curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output /usr/local/bin/jq + chmod +x /usr/local/bin/jq + - run: + name: Install hwloc + command: | + mkdir ~/hwloc + curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz + cd ~/hwloc + tar -xvzpf hwloc-2.4.1.tar.gz + cd hwloc-2.4.1 + ./configure && make && sudo make install + - restore_cache: + name: restore cargo cache + key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} + - install-deps + - run: + command: make build + no_output_timeout: 30m + - store_artifacts: + path: lotus + - store_artifacts: + path: lotus-miner + - store_artifacts: + path: lotus-worker + - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ + - persist_to_workspace: + root: "." + paths: + - darwin + - save_cache: + name: save cargo cache + key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} + paths: + - "~/.rustup" + - "~/.cargo" + + build-appimage: + machine: + image: ubuntu-2004:202104-01 + steps: + - checkout + - attach_workspace: + at: "." + - run: + name: install appimage-builder + command: | + # docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html + sudo apt update + sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace + sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage + sudo chmod +x /usr/local/bin/appimagetool + sudo pip3 install appimage-builder + - run: + name: install lotus dependencies + command: sudo apt install ocl-icd-opencl-dev libhwloc-dev + - run: + name: build appimage + command: | + sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml + make appimage + - run: + name: prepare workspace + command: | + mkdir appimage + mv Lotus-latest-x86_64.AppImage appimage + - persist_to_workspace: + root: "." + paths: + - appimage + + + gofmt: + executor: golang + steps: + - install-deps + - prepare + - run: + command: "! go fmt ./... 2>&1 | read" + + gen-check: + executor: golang + steps: + - install-deps + - prepare + - run: make deps + - run: go install golang.org/x/tools/cmd/goimports + - run: go install github.com/hannahhoward/cbor-gen-for + - run: make gen + - run: git --no-pager diff + - run: git --no-pager diff --quiet + - run: make docsgen-cli + - run: git --no-pager diff + - run: git --no-pager diff --quiet + + docs-check: + executor: golang + steps: + - install-deps + - prepare + - run: go install golang.org/x/tools/cmd/goimports + - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full + - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner + - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker + - run: make deps + - run: make docsgen + - run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full + - run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner + - run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker + - run: git --no-pager diff + - run: diff ../pre-openrpc-full ../post-openrpc-full + - run: diff ../pre-openrpc-miner ../post-openrpc-miner + - run: diff ../pre-openrpc-worker ../post-openrpc-worker + - run: git --no-pager diff --quiet + + lint: &lint + description: | + Run golangci-lint. + parameters: + executor: + type: executor + default: golang + golangci-lint-version: + type: string + default: 1.27.0 + concurrency: + type: string + default: '2' + description: | + Concurrency used to run linters. Defaults to 2 because NumCPU is not + aware of container CPU limits. + args: + type: string + default: '' + description: | + Arguments to pass to golangci-lint + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: + command: make deps + no_output_timeout: 30m + - go/install-golangci-lint: + gobin: $HOME/.local/bin + version: << parameters.golangci-lint-version >> + - run: + name: Lint + command: | + $HOME/.local/bin/golangci-lint run -v --timeout 2m \ + --concurrency << parameters.concurrency >> << parameters.args >> + lint-all: + <<: *lint + + publish: + description: publish binary artifacts + executor: ubuntu + steps: + - run: + name: Install git jq curl + command: apt update && apt install -y git jq curl + - checkout + - git_fetch_all_tags + - checkout + - install_ipfs + - attach_workspace: + at: "." + - run: + name: Create bundles + command: ./scripts/build-bundle.sh + - run: + name: Publish release + command: ./scripts/publish-release.sh + + publish-snapcraft: + description: build and push snapcraft + machine: + image: ubuntu-2004:202104-01 + resource_class: 2xlarge + parameters: + channel: + type: string + default: "edge" + description: snapcraft channel + steps: - checkout - run: - name: generate pipeline + name: install snapcraft + command: sudo snap install snapcraft --classic + - run: + name: create snapcraft config file command: | - cd .circleci - go run ./gen.go $(pwd) > generated_config.yml - - continuation/continue: - parameters: '{}' - configuration_path: .circleci/generated_config.yml + mkdir -p ~/.config/snapcraft + echo "$SNAPCRAFT_LOGIN_FILE" | base64 -d > ~/.config/snapcraft/snapcraft.cfg + - run: + name: build snap + command: snapcraft --use-lxd + - run: + name: publish snap + command: snapcraft push *.snap --release << parameters.channel >> + + build-and-push-image: + description: build and push docker images to public AWS ECR registry + executor: aws-cli/default + parameters: + profile-name: + type: string + default: "default" + description: AWS profile name to be configured. + + aws-access-key-id: + type: env_var_name + default: AWS_ACCESS_KEY_ID + description: > + AWS access key id for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_ACCESS_KEY. + + aws-secret-access-key: + type: env_var_name + default: AWS_SECRET_ACCESS_KEY + description: > + AWS secret key for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_SECRET_ACCESS_KEY. + + region: + type: env_var_name + default: AWS_REGION + description: > + Name of env var storing your AWS region information, + defaults to AWS_REGION + + account-url: + type: env_var_name + default: AWS_ECR_ACCOUNT_URL + description: > + Env var storing Amazon ECR account URL that maps to an AWS account, + e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com + defaults to AWS_ECR_ACCOUNT_URL + + dockerfile: + type: string + default: Dockerfile + description: Name of dockerfile to use. Defaults to Dockerfile. + + path: + type: string + default: . + description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). + + extra-build-args: + type: string + default: "" + description: > + Extra flags to pass to docker build. For examples, see + https://docs.docker.com/engine/reference/commandline/build + + repo: + type: string + description: Name of an Amazon ECR repository + + tag: + type: string + default: "latest" + description: A comma-separated string containing docker image tags to build and push (default = latest) + + steps: + - run: + name: Confirm that environment variables are set + command: | + if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..." + circleci-agent step halt + fi + + - aws-cli/setup: + profile-name: <> + aws-access-key-id: <> + aws-secret-access-key: <> + aws-region: <> + + - run: + name: Log into Amazon ECR + command: | + aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> + + - checkout + + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: false + + - run: + name: Build docker image + command: | + registry_id=$(echo $<> | sed "s;\..*;;g") + + docker_tag_args="" + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker_tag_args="$docker_tag_args -t $<>/<>:$tag" + done + + docker build \ + <<#parameters.extra-build-args>><><> \ + -f <>/<> \ + $docker_tag_args \ + <> + + - run: + name: Push image to Amazon ECR + command: | + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker push $<>/<>:${tag} + done + + publish-packer-mainnet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux -var lotus_network=mainnet -var git_tag=$CIRCLE_TAG" + publish-packer-calibrationnet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-calibrationnet -var lotus_network=calibrationnet -var git_tag=$CIRCLE_TAG" + publish-packer-butterflynet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-butterflynet -var lotus_network=butterflynet -var git_tag=$CIRCLE_TAG" + publish-packer-nerpanet: + description: build and push AWS IAM and DigitalOcean droplet. + executor: + name: packer/default + packer-version: 1.6.6 + steps: + - checkout + - attach_workspace: + at: "." + - packer/build: + template: tools/packer/lotus.pkr.hcl + args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" + workflows: - setup-workflow: + version: 2.1 + ci: jobs: - - generate-config \ No newline at end of file + - lint-all: + concurrency: "16" # expend all docker 2xlarge CPUs. + - mod-tidy-check + - gofmt + - gen-check + - docs-check + - test: + name: test-itest-api + suite: itest-api + target: "./itests/api_test.go" + + - test: + name: test-itest-batch_deal + suite: itest-batch_deal + target: "./itests/batch_deal_test.go" + + - test: + name: test-itest-ccupgrade + suite: itest-ccupgrade + target: "./itests/ccupgrade_test.go" + + - test: + name: test-itest-cli + suite: itest-cli + target: "./itests/cli_test.go" + + - test: + name: test-itest-deadlines + suite: itest-deadlines + target: "./itests/deadlines_test.go" + + - test: + name: test-itest-deals + suite: itest-deals + target: "./itests/deals_test.go" + + - test: + name: test-itest-gateway + suite: itest-gateway + target: "./itests/gateway_test.go" + + - test: + name: test-itest-multisig + suite: itest-multisig + target: "./itests/multisig_test.go" + + - test: + name: test-itest-paych_api + suite: itest-paych_api + target: "./itests/paych_api_test.go" + + - test: + name: test-itest-paych_cli + suite: itest-paych_cli + target: "./itests/paych_cli_test.go" + + - test: + name: test-itest-sdr_upgrade + suite: itest-sdr_upgrade + target: "./itests/sdr_upgrade_test.go" + + - test: + name: test-itest-sector_pledge + suite: itest-sector_pledge + target: "./itests/sector_pledge_test.go" + + - test: + name: test-itest-sector_terminate + suite: itest-sector_terminate + target: "./itests/sector_terminate_test.go" + + - test: + name: test-itest-tape + suite: itest-tape + target: "./itests/tape_test.go" + + - test: + name: test-itest-verifreg + suite: itest-verifreg + target: "./itests/verifreg_test.go" + + - test: + name: test-itest-wdpost_dispute + suite: itest-wdpost_dispute + target: "./itests/wdpost_dispute_test.go" + + - test: + name: test-itest-wdpost + suite: itest-wdpost + target: "./itests/wdpost_test.go" + + - test: + name: test-unit-cli + suite: utest-unit-cli + target: "./cli/... ./cmd/... ./api/..." + - test: + name: test-unit-node + suite: utest-unit-node + target: "./node/..." + - test: + name: test-unit-rest + suite: utest-unit-rest + target: "./blockstore/... ./build/... ./cmd/... ./extern/... ./paychmgr/... ./storage/... ./api/... ./conformance/... ./journal/... ./chain/... ./markets/... ./tools/... ./cli/... ./gateway/... ./lib/... ./node/..." + - test: + name: test-unit-storage + suite: utest-unit-storage + target: "./storage/... ./extern/..." + - test: + go-test-flags: "-run=TestMulticoreSDR" + suite: multicore-sdr-check + target: "./extern/sector-storage/ffiwrapper" + proofs-log-test: "1" + - test-conformance: + suite: conformance + codecov-upload: false + target: "./conformance" + - test-conformance: + name: test-conformance-bleeding-edge + codecov-upload: false + suite: conformance-bleeding-edge + target: "./conformance" + vectors-branch: master + - trigger-testplans: + filters: + branches: + only: + - master + - build-debug + - build-all: + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-ntwk-calibration: + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-ntwk-butterfly: + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-ntwk-nerpa: + filters: + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-lotus-soup + - build-macos: + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-appimage: + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish: + requires: + - build-all + - build-macos + - build-appimage + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-dev + tag: '${CIRCLE_SHA1:0:8}' + - publish-packer-mainnet: + requires: + - build-all + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-calibrationnet: + requires: + - build-ntwk-calibration + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-butterflynet: + requires: + - build-ntwk-butterfly + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-packer-nerpanet: + requires: + - build-ntwk-nerpa + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-snapcraft: + name: publish-snapcraft-stable + channel: stable + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + + nightly: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + jobs: + - publish-snapcraft: + name: publish-snapcraft-nightly + channel: edge diff --git a/.circleci/gen.go b/.circleci/gen.go index db806b003..f5c56c6e3 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -9,6 +9,8 @@ import ( "text/template" ) +//go:generate go run ./gen.go .. + //go:embed template.yml var templateFile embed.FS @@ -43,7 +45,7 @@ func main() { tmpl = template.Must(tmpl.ParseFS(templateFile, "*")) // list all itests. - itests, err := filepath.Glob("./itests/*_test.go") + itests, err := filepath.Glob(filepath.Join(repo, "./itests/*_test.go")) if err != nil { panic(err) } @@ -117,8 +119,14 @@ func main() { }(), } + out, err := os.Create("./config.yml") + if err != nil { + panic(err) + } + defer out.Close() + // execute the template. - if err := tmpl.Execute(os.Stdout, in); err != nil { + if err := tmpl.Execute(out, in); err != nil { panic(err) } } diff --git a/.circleci/go.mod b/.circleci/go.mod deleted file mode 100644 index 9b92b8c78..000000000 --- a/.circleci/go.mod +++ /dev/null @@ -1 +0,0 @@ -module ".circleci" \ No newline at end of file diff --git a/.circleci/template.yml b/.circleci/template.yml index fad4cdee5..e2227f2fe 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -174,7 +174,7 @@ jobs: -- \ << parameters.coverage >> \ << parameters.go-test-flags >> \ - << parameters.packages >> + << parameters.target >> no_output_timeout: 30m - store_test_results: path: /tmp/test-reports @@ -762,8 +762,7 @@ workflows: suite: utest-[[ $suite ]] target: "[[ $pkgs ]]" [[- end]] - - - check-proofs-multicore-sdr: + - test: go-test-flags: "-run=TestMulticoreSDR" suite: multicore-sdr-check target: "./extern/sector-storage/ffiwrapper" @@ -785,37 +784,27 @@ workflows: - master - build-debug - build-all: - requires: - - test-short filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-ntwk-calibration: - requires: - - test-short filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-ntwk-butterfly: - requires: - - test-short filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-ntwk-nerpa: - requires: - - test-short filters: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - build-macos: - requires: - - test-short filters: branches: ignore: @@ -824,8 +813,6 @@ workflows: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-appimage: - requires: - - test-short filters: branches: ignore: From 539f8c1f61c688768260656441a19643990313f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 22 Jun 2021 21:25:04 +0100 Subject: [PATCH 03/11] fix test that references private symbols; avoid hacky test selection flags. --- itests/deadlines_test.go | 4 -- itests/gateway_test.go | 3 +- itests/multisig/suite.go | 96 +++++++++++++++++++++++++++++++++ itests/multisig_test.go | 91 +------------------------------ itests/sector_terminate_test.go | 4 -- itests/wdpost_dispute_test.go | 8 --- itests/wdpost_test.go | 13 ----- 7 files changed, 100 insertions(+), 119 deletions(-) create mode 100644 itests/multisig/suite.go diff --git a/itests/deadlines_test.go b/itests/deadlines_test.go index 9551465a5..bef72297f 100644 --- a/itests/deadlines_test.go +++ b/itests/deadlines_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "os" "testing" "time" @@ -55,9 +54,6 @@ import ( // * asserts that miner B loses power // * asserts that miner D loses power, is inactive func TestDeadlineToggling(t *testing.T) { - if os.Getenv("LOTUS_TEST_DEADLINE_TOGGLING") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_DEADLINE_TOGGLING=1 to run") - } _ = logging.SetLogLevel("miner", "ERROR") _ = logging.SetLogLevel("chainstore", "ERROR") _ = logging.SetLogLevel("chain", "ERROR") diff --git a/itests/gateway_test.go b/itests/gateway_test.go index f5eb4e363..c7fe59444 100644 --- a/itests/gateway_test.go +++ b/itests/gateway_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/itests/multisig" "github.com/stretchr/testify/require" "golang.org/x/xerrors" @@ -188,7 +189,7 @@ func TestGatewayMsigCLI(t *testing.T) { defer nodes.closer() lite := nodes.lite - runMultisigTests(t, lite) + multisig.RunMultisigTests(t, lite) } func TestGatewayDealFlow(t *testing.T) { diff --git a/itests/multisig/suite.go b/itests/multisig/suite.go new file mode 100644 index 000000000..31113cb10 --- /dev/null +++ b/itests/multisig/suite.go @@ -0,0 +1,96 @@ +package multisig + +import ( + "context" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/stretchr/testify/require" +) + +func RunMultisigTests(t *testing.T, clientNode kit.TestFullNode) { + // Create mock CLI + ctx := context.Background() + mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) + clientCLI := mockCLI.Client(clientNode.ListenAddr) + + // Create some wallets on the node to use for testing multisig + var walletAddrs []address.Address + for i := 0; i < 4; i++ { + addr, err := clientNode.WalletNew(ctx, types.KTSecp256k1) + require.NoError(t, err) + + walletAddrs = append(walletAddrs, addr) + + kit.SendFunds(ctx, t, clientNode, addr, types.NewInt(1e15)) + } + + // Create an msig with three of the addresses and threshold of two sigs + // msig create --required=2 --duration=50 --value=1000attofil + amtAtto := types.NewInt(1000) + threshold := 2 + paramDuration := "--duration=50" + paramRequired := fmt.Sprintf("--required=%d", threshold) + paramValue := fmt.Sprintf("--value=%dattofil", amtAtto) + out := clientCLI.RunCmd( + "msig", "create", + paramRequired, + paramDuration, + paramValue, + walletAddrs[0].String(), + walletAddrs[1].String(), + walletAddrs[2].String(), + ) + fmt.Println(out) + + // Extract msig robust address from output + expCreateOutPrefix := "Created new multisig:" + require.Regexp(t, regexp.MustCompile(expCreateOutPrefix), out) + parts := strings.Split(strings.TrimSpace(strings.Replace(out, expCreateOutPrefix, "", -1)), " ") + require.Len(t, parts, 2) + msigRobustAddr := parts[1] + fmt.Println("msig robust address:", msigRobustAddr) + + // Propose to add a new address to the msig + // msig add-propose --from= + paramFrom := fmt.Sprintf("--from=%s", walletAddrs[0]) + out = clientCLI.RunCmd( + "msig", "add-propose", + paramFrom, + msigRobustAddr, + walletAddrs[3].String(), + ) + fmt.Println(out) + + // msig inspect + out = clientCLI.RunCmd("msig", "inspect", "--vesting", "--decode-params", msigRobustAddr) + fmt.Println(out) + + // Expect correct balance + require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) + // Expect 1 transaction + require.Regexp(t, regexp.MustCompile(`Transactions:\s*1`), out) + // Expect transaction to be "AddSigner" + require.Regexp(t, regexp.MustCompile(`AddSigner`), out) + + // Approve adding the new address + // msig add-approve --from= 0 false + txnID := "0" + paramFrom = fmt.Sprintf("--from=%s", walletAddrs[1]) + out = clientCLI.RunCmd( + "msig", "add-approve", + paramFrom, + msigRobustAddr, + walletAddrs[0].String(), + txnID, + walletAddrs[3].String(), + "false", + ) + fmt.Println(out) +} diff --git a/itests/multisig_test.go b/itests/multisig_test.go index 4c513640d..f30b26641 100644 --- a/itests/multisig_test.go +++ b/itests/multisig_test.go @@ -2,18 +2,12 @@ package itests import ( "context" - "fmt" "os" - "regexp" - "strings" "testing" "time" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/itests/kit" - "github.com/stretchr/testify/require" + "github.com/filecoin-project/lotus/itests/multisig" ) // TestMultisig does a basic test to exercise the multisig CLI commands @@ -25,86 +19,5 @@ func TestMultisig(t *testing.T) { ctx := context.Background() clientNode, _ := kit.StartOneNodeOneMiner(ctx, t, blocktime) - runMultisigTests(t, clientNode) -} - -func runMultisigTests(t *testing.T, clientNode kit.TestFullNode) { - // Create mock CLI - ctx := context.Background() - mockCLI := kit.NewMockCLI(ctx, t, cli.Commands) - clientCLI := mockCLI.Client(clientNode.ListenAddr) - - // Create some wallets on the node to use for testing multisig - var walletAddrs []address.Address - for i := 0; i < 4; i++ { - addr, err := clientNode.WalletNew(ctx, types.KTSecp256k1) - require.NoError(t, err) - - walletAddrs = append(walletAddrs, addr) - - kit.SendFunds(ctx, t, clientNode, addr, types.NewInt(1e15)) - } - - // Create an msig with three of the addresses and threshold of two sigs - // msig create --required=2 --duration=50 --value=1000attofil - amtAtto := types.NewInt(1000) - threshold := 2 - paramDuration := "--duration=50" - paramRequired := fmt.Sprintf("--required=%d", threshold) - paramValue := fmt.Sprintf("--value=%dattofil", amtAtto) - out := clientCLI.RunCmd( - "msig", "create", - paramRequired, - paramDuration, - paramValue, - walletAddrs[0].String(), - walletAddrs[1].String(), - walletAddrs[2].String(), - ) - fmt.Println(out) - - // Extract msig robust address from output - expCreateOutPrefix := "Created new multisig:" - require.Regexp(t, regexp.MustCompile(expCreateOutPrefix), out) - parts := strings.Split(strings.TrimSpace(strings.Replace(out, expCreateOutPrefix, "", -1)), " ") - require.Len(t, parts, 2) - msigRobustAddr := parts[1] - fmt.Println("msig robust address:", msigRobustAddr) - - // Propose to add a new address to the msig - // msig add-propose --from= - paramFrom := fmt.Sprintf("--from=%s", walletAddrs[0]) - out = clientCLI.RunCmd( - "msig", "add-propose", - paramFrom, - msigRobustAddr, - walletAddrs[3].String(), - ) - fmt.Println(out) - - // msig inspect - out = clientCLI.RunCmd("msig", "inspect", "--vesting", "--decode-params", msigRobustAddr) - fmt.Println(out) - - // Expect correct balance - require.Regexp(t, regexp.MustCompile("Balance: 0.000000000000001 FIL"), out) - // Expect 1 transaction - require.Regexp(t, regexp.MustCompile(`Transactions:\s*1`), out) - // Expect transaction to be "AddSigner" - require.Regexp(t, regexp.MustCompile(`AddSigner`), out) - - // Approve adding the new address - // msig add-approve --from= 0 false - txnID := "0" - paramFrom = fmt.Sprintf("--from=%s", walletAddrs[1]) - out = clientCLI.RunCmd( - "msig", "add-approve", - paramFrom, - msigRobustAddr, - walletAddrs[0].String(), - txnID, - walletAddrs[3].String(), - "false", - ) - fmt.Println(out) + multisig.RunMultisigTests(t, clientNode) } diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index b00337c7e..84260fd9d 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -18,10 +18,6 @@ import ( ) func TestTerminate(t *testing.T) { - if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") - } - kit.QuietMiningLogs() const blocktime = 2 * time.Millisecond diff --git a/itests/wdpost_dispute_test.go b/itests/wdpost_dispute_test.go index 6c7302af3..b453fee2e 100644 --- a/itests/wdpost_dispute_test.go +++ b/itests/wdpost_dispute_test.go @@ -23,10 +23,6 @@ import ( ) func TestWindowPostDispute(t *testing.T) { - if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") - } - kit.QuietMiningLogs() b := kit.MockMinerBuilder @@ -254,10 +250,6 @@ func TestWindowPostDispute(t *testing.T) { } func TestWindowPostDisputeFails(t *testing.T) { - if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") - } - kit.QuietMiningLogs() b := kit.MockMinerBuilder diff --git a/itests/wdpost_test.go b/itests/wdpost_test.go index f59465f05..70f526e2b 100644 --- a/itests/wdpost_test.go +++ b/itests/wdpost_test.go @@ -3,7 +3,6 @@ package itests import ( "context" "fmt" - "os" "testing" "time" @@ -23,10 +22,6 @@ import ( ) func TestWindowedPost(t *testing.T) { - if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") - } - kit.QuietMiningLogs() var ( @@ -264,10 +259,6 @@ func testWindowPostUpgrade(t *testing.T, b kit.APIBuilder, blocktime time.Durati } func TestWindowPostBaseFeeNoBurn(t *testing.T) { - if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") - } - kit.QuietMiningLogs() var ( @@ -323,10 +314,6 @@ waitForProof: } func TestWindowPostBaseFeeBurn(t *testing.T) { - if os.Getenv("LOTUS_TEST_WINDOW_POST") != "1" { - t.Skip("this takes a few minutes, set LOTUS_TEST_WINDOW_POST=1 to run") - } - kit.QuietMiningLogs() ctx, cancel := context.WithCancel(context.Background()) From c6c202d7d403fb01d320067659fa34dcac89b71f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 22 Jun 2021 21:25:36 +0100 Subject: [PATCH 04/11] add makefile target. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index e6c5f1b54..5821922ea 100644 --- a/Makefile +++ b/Makefile @@ -379,3 +379,6 @@ docsgen-cli: lotus lotus-miner lotus-worker print-%: @echo $*=$($*) + +circleci: + go generate -x ./.circleci \ No newline at end of file From b74ad796ce6224b48465c4cbb39b3cab1761b034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 22 Jun 2021 21:29:07 +0100 Subject: [PATCH 05/11] fix dangling import. --- itests/sector_terminate_test.go | 1 - itests/wdpost_dispute_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index 84260fd9d..74dbc5fa0 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -3,7 +3,6 @@ package itests import ( "context" "fmt" - "os" "testing" "time" diff --git a/itests/wdpost_dispute_test.go b/itests/wdpost_dispute_test.go index b453fee2e..ad7024823 100644 --- a/itests/wdpost_dispute_test.go +++ b/itests/wdpost_dispute_test.go @@ -3,7 +3,6 @@ package itests import ( "context" "fmt" - "os" "testing" "time" From da96414bf80d847ed396fbca04c5d4c54141eed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 22 Jun 2021 21:32:10 +0100 Subject: [PATCH 06/11] switch test output format. --- .circleci/config.yml | 4 ++-- .circleci/template.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f41a8c513..4f6036f45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,7 @@ jobs: description: Test suite name to report to CircleCI. gotestsum-format: type: string - default: pkgname-and-test-fails + default: standard-verbose description: gotestsum format. https://github.com/gotestyourself/gotestsum#format coverage: type: string @@ -842,7 +842,7 @@ workflows: - test: name: test-unit-rest suite: utest-unit-rest - target: "./blockstore/... ./build/... ./cmd/... ./extern/... ./paychmgr/... ./storage/... ./api/... ./conformance/... ./journal/... ./chain/... ./markets/... ./tools/... ./cli/... ./gateway/... ./lib/... ./node/..." + target: "./blockstore/... ./build/... ./journal/... ./cli/... ./conformance/... ./node/... ./paychmgr/... ./api/... ./chain/... ./lib/... ./cmd/... ./extern/... ./gateway/... ./markets/... ./storage/... ./tools/..." - test: name: test-unit-storage suite: utest-unit-storage diff --git a/.circleci/template.yml b/.circleci/template.yml index e2227f2fe..75fbdfc76 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -136,7 +136,7 @@ jobs: description: Test suite name to report to CircleCI. gotestsum-format: type: string - default: pkgname-and-test-fails + default: standard-verbose description: gotestsum format. https://github.com/gotestyourself/gotestsum#format coverage: type: string From 211751f8b8fe73c9f54d2bae3e0ef5239a181c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 23 Jun 2021 16:37:43 +0100 Subject: [PATCH 07/11] declare some tests as expensive. --- itests/deadlines_test.go | 2 ++ itests/kit/run.go | 20 ++++++++++++++++++++ itests/sector_terminate_test.go | 2 ++ itests/wdpost_dispute_test.go | 4 ++++ itests/wdpost_test.go | 6 ++++++ 5 files changed, 34 insertions(+) create mode 100644 itests/kit/run.go diff --git a/itests/deadlines_test.go b/itests/deadlines_test.go index bef72297f..193cdf40b 100644 --- a/itests/deadlines_test.go +++ b/itests/deadlines_test.go @@ -54,6 +54,8 @@ import ( // * asserts that miner B loses power // * asserts that miner D loses power, is inactive func TestDeadlineToggling(t *testing.T) { + kit.Expensive(t) + _ = logging.SetLogLevel("miner", "ERROR") _ = logging.SetLogLevel("chainstore", "ERROR") _ = logging.SetLogLevel("chain", "ERROR") diff --git a/itests/kit/run.go b/itests/kit/run.go new file mode 100644 index 000000000..713efa3b8 --- /dev/null +++ b/itests/kit/run.go @@ -0,0 +1,20 @@ +package kit + +import ( + "os" + "testing" +) + +// EnvRunExpensiveTests is the environment variable that needs to be present +// and set to value "1" to enable running expensive tests outside of CI. +const EnvRunExpensiveTests = "LOTUS_RUN_EXPENSIVE_TESTS" + +// Expensive marks a test as expensive, skipping it immediately if not running an +func Expensive(t *testing.T) { + switch { + case os.Getenv("CI") == "true": + return + case os.Getenv(EnvRunExpensiveTests) != "1": + t.Skipf("skipping expensive test outside of CI; enable by setting env var %s=1", EnvRunExpensiveTests) + } +} diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index 74dbc5fa0..73b4e443c 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -17,6 +17,8 @@ import ( ) func TestTerminate(t *testing.T) { + kit.Expensive(t) + kit.QuietMiningLogs() const blocktime = 2 * time.Millisecond diff --git a/itests/wdpost_dispute_test.go b/itests/wdpost_dispute_test.go index ad7024823..b665667a3 100644 --- a/itests/wdpost_dispute_test.go +++ b/itests/wdpost_dispute_test.go @@ -22,6 +22,8 @@ import ( ) func TestWindowPostDispute(t *testing.T) { + kit.Expensive(t) + kit.QuietMiningLogs() b := kit.MockMinerBuilder @@ -249,6 +251,8 @@ func TestWindowPostDispute(t *testing.T) { } func TestWindowPostDisputeFails(t *testing.T) { + kit.Expensive(t) + kit.QuietMiningLogs() b := kit.MockMinerBuilder diff --git a/itests/wdpost_test.go b/itests/wdpost_test.go index 70f526e2b..81f938379 100644 --- a/itests/wdpost_test.go +++ b/itests/wdpost_test.go @@ -22,6 +22,8 @@ import ( ) func TestWindowedPost(t *testing.T) { + kit.Expensive(t) + kit.QuietMiningLogs() var ( @@ -259,6 +261,8 @@ func testWindowPostUpgrade(t *testing.T, b kit.APIBuilder, blocktime time.Durati } func TestWindowPostBaseFeeNoBurn(t *testing.T) { + kit.Expensive(t) + kit.QuietMiningLogs() var ( @@ -314,6 +318,8 @@ waitForProof: } func TestWindowPostBaseFeeBurn(t *testing.T) { + kit.Expensive(t) + kit.QuietMiningLogs() ctx, cancel := context.WithCancel(context.Background()) From 6017715fd7269379ea69ee1a2104c51ea042f4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 23 Jun 2021 16:40:20 +0100 Subject: [PATCH 08/11] (test) deliberately check in an outdated .circleci/config.yml to test the gen-check CI target. --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f6036f45..e73d42988 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -830,7 +830,6 @@ workflows: name: test-itest-wdpost suite: itest-wdpost target: "./itests/wdpost_test.go" - - test: name: test-unit-cli suite: utest-unit-cli From 0c31676bf8911882e540711eb57188f5f74d2256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 23 Jun 2021 16:46:13 +0100 Subject: [PATCH 09/11] Makefile: add circleci target as dependency of gen target. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5821922ea..1e7cc4cc0 100644 --- a/Makefile +++ b/Makefile @@ -364,7 +364,7 @@ docsgen-openrpc-worker: docsgen-openrpc-bin .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin -gen: actors-gen type-gen method-gen docsgen api-gen +gen: actors-gen type-gen method-gen docsgen api-gen circleci @echo ">>> IF YOU'VE MODIFIED THE CLI, REMEMBER TO ALSO MAKE docsgen-cli" .PHONY: gen From 2c605e08f1ccec9008c22d1e56c45cf1f1bfb26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 23 Jun 2021 16:50:11 +0100 Subject: [PATCH 10/11] Revert "(test) deliberately check in an outdated .circleci/config.yml to test the gen-check CI target." This reverts commit 6017715fd7269379ea69ee1a2104c51ea042f4e0. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e73d42988..4f6036f45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -830,6 +830,7 @@ workflows: name: test-itest-wdpost suite: itest-wdpost target: "./itests/wdpost_test.go" + - test: name: test-unit-cli suite: utest-unit-cli From f4341409a1a8d26769d774979468fe7418d593d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 23 Jun 2021 17:08:46 +0100 Subject: [PATCH 11/11] deterministic order for rest unit tests. --- .circleci/config.yml | 2 +- .circleci/gen.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f6036f45..133fb4d96 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -842,7 +842,7 @@ workflows: - test: name: test-unit-rest suite: utest-unit-rest - target: "./blockstore/... ./build/... ./journal/... ./cli/... ./conformance/... ./node/... ./paychmgr/... ./api/... ./chain/... ./lib/... ./cmd/... ./extern/... ./gateway/... ./markets/... ./storage/... ./tools/..." + target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..." - test: name: test-unit-storage suite: utest-unit-storage diff --git a/.circleci/gen.go b/.circleci/gen.go index f5c56c6e3..844348e29 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "text/template" ) @@ -100,6 +101,9 @@ func main() { groupedUnitTests["unit-rest"] = append(groupedUnitTests["unit-rest"], k) } + // map iteration guarantees no order, so sort the array in-place. + sort.Strings(groupedUnitTests["unit-rest"]) + // form the input data. type data struct { ItestFiles []string