7f1b4dc438
This is a small refactor of our workflow to test out goreleaser, a yaml based tool for building, packaging, and releasing go binaries on multiple platforms. It supports building binaries for to most of the platforms we care about, including linux and macos, and also supports publishing those binaries automatically as releases in Github, homebrew, snap, and even apt / deb. If this trial goes well, I think we should eventually replace the entire release workflow with goreleaser. For now, this test is more tightly scoped to only automated the MacOS release process, since that is the one we have the most issues with. This PRi / commit: - Builds darwin-amd64 and darwin-arm64 binaries of lotus, lotus-miner, and lotus-worker - Packages them into a universal darwin binary - Publishes those to a release in Github based on the current tag - Uses the binaries in the release to auto-publish and updated homebrew configuration to filecoin-project/homebrew-lotus - Does a `dry-run` build to produce a snapshot on release branches with no tag - Manually generate and upload checksums after goreleaser
1188 lines
37 KiB
YAML
1188 lines
37 KiB
YAML
version: 2.1
|
|
orbs:
|
|
aws-cli: circleci/aws-cli@1.3.2
|
|
|
|
executors:
|
|
golang:
|
|
docker:
|
|
- image: cimg/go:1.17.9
|
|
resource_class: 2xlarge
|
|
ubuntu:
|
|
docker:
|
|
- image: ubuntu:20.04
|
|
packer:
|
|
description: |
|
|
The HashiCorp provided Packer container
|
|
parameters:
|
|
packer-version:
|
|
type: string
|
|
default: "1.8"
|
|
docker:
|
|
- image: hashicorp/packer:<< parameters.packer-version >>
|
|
|
|
|
|
commands:
|
|
install-deps:
|
|
steps:
|
|
- run: |
|
|
sudo apt update
|
|
sudo apt install python-is-python3
|
|
prepare:
|
|
parameters:
|
|
linux:
|
|
default: true
|
|
description: is a linux build environment?
|
|
type: boolean
|
|
darwin:
|
|
default: false
|
|
description: is a darwin build environment?
|
|
type: boolean
|
|
steps:
|
|
- checkout
|
|
- git_fetch_all_tags
|
|
- checkout
|
|
- when:
|
|
condition: << parameters.linux >>
|
|
steps:
|
|
- run: sudo apt-get update
|
|
- run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev
|
|
- when:
|
|
condition: <<parameters.darwin>>
|
|
steps:
|
|
- run:
|
|
name: Install Go
|
|
command: |
|
|
curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \
|
|
sudo installer -pkg /tmp/go.pkg -target /
|
|
- run:
|
|
name: Export Go
|
|
command: |
|
|
echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
|
|
- run: go version
|
|
- run:
|
|
name: Install pkg-config, goreleaser, and sha512sum
|
|
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils
|
|
- run:
|
|
name: Install Rust
|
|
command: |
|
|
curl https://sh.rustup.rs -sSf | sh -s -- -y
|
|
- run:
|
|
name: Install hwloc
|
|
command: |
|
|
mkdir ~/hwloc
|
|
curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz
|
|
cd ~/hwloc
|
|
tar -xvzpf hwloc-2.4.1.tar.gz
|
|
cd hwloc-2.4.1
|
|
./configure && make && sudo make install
|
|
- run: git submodule sync
|
|
- run: git submodule update --init
|
|
download-params:
|
|
steps:
|
|
- restore_cache:
|
|
name: Restore parameters cache
|
|
keys:
|
|
- 'v26-2k-lotus-params'
|
|
paths:
|
|
- /var/tmp/filecoin-proof-parameters/
|
|
- run: ./lotus fetch-params 2048
|
|
- save_cache:
|
|
name: Save parameters cache
|
|
key: 'v26-2k-lotus-params'
|
|
paths:
|
|
- /var/tmp/filecoin-proof-parameters/
|
|
install_ipfs:
|
|
steps:
|
|
- run: |
|
|
apt update
|
|
apt install -y wget
|
|
wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz
|
|
wget https://github.com/ipfs/go-ipfs/releases/download/v0.12.2/go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512
|
|
if [ "$(sha512sum go-ipfs_v0.12.2_linux-amd64.tar.gz)" != "$(cat go-ipfs_v0.12.2_linux-amd64.tar.gz.sha512)" ]
|
|
then
|
|
echo "ipfs failed checksum check"
|
|
exit 1
|
|
fi
|
|
tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz
|
|
mv go-ipfs/ipfs /usr/local/bin/ipfs
|
|
chmod +x /usr/local/bin/ipfs
|
|
install_ipfs_macos:
|
|
steps:
|
|
- run: |
|
|
curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz
|
|
tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz
|
|
pushd kubo
|
|
sudo bash install.sh
|
|
popd
|
|
rm -rf kubo/
|
|
rm kubo_v0.14.0_darwin-amd64.tar.gz
|
|
git_fetch_all_tags:
|
|
steps:
|
|
- run:
|
|
name: fetch all tags
|
|
command: |
|
|
git fetch --all
|
|
packer_build:
|
|
description: "Run a packer build"
|
|
parameters:
|
|
template:
|
|
description: |
|
|
The name of the packer template file
|
|
type: string
|
|
default: packer.json
|
|
args:
|
|
description: |
|
|
Arguments to pass to the packer build command
|
|
type: string
|
|
default: ""
|
|
|
|
steps:
|
|
- run:
|
|
name: "Run a packer build"
|
|
command: packer build << parameters.args >> << parameters.template >>
|
|
no_output_timeout: 30m
|
|
|
|
jobs:
|
|
mod-tidy-check:
|
|
executor: golang
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: go mod tidy -v
|
|
- run:
|
|
name: Check git diff
|
|
command: |
|
|
git --no-pager diff go.mod go.sum
|
|
git --no-pager diff --quiet go.mod go.sum
|
|
build-linux:
|
|
executor: golang
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: sudo apt-get update
|
|
- run: sudo apt-get install npm
|
|
- run:
|
|
command: make buildall
|
|
- run:
|
|
name: check tag and version output match
|
|
command: ./scripts/version-check.sh ./lotus
|
|
- store_artifacts:
|
|
path: lotus
|
|
- store_artifacts:
|
|
path: lotus-miner
|
|
- store_artifacts:
|
|
path: lotus-worker
|
|
- run: mkdir linux && mv lotus lotus-miner lotus-worker linux/
|
|
- persist_to_workspace:
|
|
root: "."
|
|
paths:
|
|
- linux
|
|
|
|
build-debug:
|
|
executor: golang
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run:
|
|
command: make debug
|
|
|
|
test:
|
|
description: |
|
|
Run tests with gotestsum.
|
|
parameters: &test-params
|
|
executor:
|
|
type: executor
|
|
default: golang
|
|
go-test-flags:
|
|
type: string
|
|
default: "-timeout 30m"
|
|
description: Flags passed to go test.
|
|
target:
|
|
type: string
|
|
default: "./..."
|
|
description: Import paths of packages to be tested.
|
|
proofs-log-test:
|
|
type: string
|
|
default: "0"
|
|
suite:
|
|
type: string
|
|
default: unit
|
|
description: Test suite name to report to CircleCI.
|
|
gotestsum-format:
|
|
type: string
|
|
default: standard-verbose
|
|
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run:
|
|
command: make deps lotus
|
|
no_output_timeout: 30m
|
|
- download-params
|
|
- run:
|
|
name: go test
|
|
environment:
|
|
TEST_RUSTPROOFS_LOGS: << parameters.proofs-log-test >>
|
|
SKIP_CONFORMANCE: "1"
|
|
LOTUS_SRC_DIR: /home/circleci/project
|
|
command: |
|
|
mkdir -p /tmp/test-reports/<< parameters.suite >>
|
|
mkdir -p /tmp/test-artifacts
|
|
gotestsum \
|
|
--format << parameters.gotestsum-format >> \
|
|
--junitfile /tmp/test-reports/<< parameters.suite >>/junit.xml \
|
|
--jsonfile /tmp/test-artifacts/<< parameters.suite >>.json \
|
|
-- \
|
|
<< parameters.go-test-flags >> \
|
|
<< parameters.target >>
|
|
no_output_timeout: 30m
|
|
- store_test_results:
|
|
path: /tmp/test-reports
|
|
- store_artifacts:
|
|
path: /tmp/test-artifacts/<< parameters.suite >>.json
|
|
|
|
test-conformance:
|
|
description: |
|
|
Run tests using a corpus of interoperable test vectors for Filecoin
|
|
implementations to test their correctness and compliance with the Filecoin
|
|
specifications.
|
|
parameters:
|
|
<<: *test-params
|
|
vectors-branch:
|
|
type: string
|
|
default: ""
|
|
description: |
|
|
Branch on github.com/filecoin-project/test-vectors to checkout and
|
|
test with. If empty (the default) the commit defined by the git
|
|
submodule is used.
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run:
|
|
command: make deps lotus
|
|
no_output_timeout: 30m
|
|
- download-params
|
|
- when:
|
|
condition:
|
|
not:
|
|
equal: [ "", << parameters.vectors-branch >> ]
|
|
steps:
|
|
- run:
|
|
name: checkout vectors branch
|
|
command: |
|
|
cd extern/test-vectors
|
|
git fetch
|
|
git checkout origin/<< parameters.vectors-branch >>
|
|
- run:
|
|
name: install statediff globally
|
|
command: |
|
|
## statediff is optional; we succeed even if compilation fails.
|
|
mkdir -p /tmp/statediff
|
|
git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
|
|
cd /tmp/statediff
|
|
go install ./cmd/statediff || exit 0
|
|
- run:
|
|
name: go test
|
|
environment:
|
|
SKIP_CONFORMANCE: "0"
|
|
command: |
|
|
mkdir -p /tmp/test-reports
|
|
mkdir -p /tmp/test-artifacts
|
|
gotestsum \
|
|
--format pkgname-and-test-fails \
|
|
--junitfile /tmp/test-reports/junit.xml \
|
|
-- \
|
|
-v -coverpkg ./chain/vm/,github.com/filecoin-project/specs-actors/... -coverprofile=/tmp/conformance.out ./conformance/
|
|
go tool cover -html=/tmp/conformance.out -o /tmp/test-artifacts/conformance-coverage.html
|
|
no_output_timeout: 30m
|
|
- store_test_results:
|
|
path: /tmp/test-reports
|
|
- store_artifacts:
|
|
path: /tmp/test-artifacts/conformance-coverage.html
|
|
build-ntwk-calibration:
|
|
description: |
|
|
Compile lotus binaries for the calibration network
|
|
parameters:
|
|
<<: *test-params
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: make calibnet
|
|
- run: mkdir linux-calibrationnet && mv lotus lotus-miner lotus-worker linux-calibrationnet
|
|
- persist_to_workspace:
|
|
root: "."
|
|
paths:
|
|
- linux-calibrationnet
|
|
build-ntwk-butterfly:
|
|
description: |
|
|
Compile lotus binaries for the butterfly network
|
|
parameters:
|
|
<<: *test-params
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: make butterflynet
|
|
- run: mkdir linux-butterflynet && mv lotus lotus-miner lotus-worker linux-butterflynet
|
|
- persist_to_workspace:
|
|
root: "."
|
|
paths:
|
|
- linux-butterflynet
|
|
build-lotus-soup:
|
|
description: |
|
|
Compile `lotus-soup` Testground test plan
|
|
parameters:
|
|
<<: *test-params
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: cd extern/filecoin-ffi && make
|
|
- run:
|
|
name: "go get lotus@master"
|
|
command: cd testplans/lotus-soup && go mod edit -replace=github.com/filecoin-project/lotus=../.. && go mod tidy
|
|
- run:
|
|
name: "build lotus-soup testplan"
|
|
command: pushd testplans/lotus-soup && go build -tags=testground .
|
|
trigger-testplans:
|
|
description: |
|
|
Trigger `lotus-soup` test cases on TaaS
|
|
parameters:
|
|
<<: *test-params
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run:
|
|
name: "download testground"
|
|
command: wget https://gist.github.com/nonsense/5fbf3167cac79945f658771aed32fc44/raw/2e17eb0debf7ec6bdf027c1bdafc2c92dd97273b/testground-d3e9603 -O ~/testground-cli && chmod +x ~/testground-cli
|
|
- run:
|
|
name: "prepare .env.toml"
|
|
command: pushd testplans/lotus-soup && mkdir -p $HOME/testground && cp env-ci.toml $HOME/testground/.env.toml && echo 'endpoint="https://ci.testground.ipfs.team"' >> $HOME/testground/.env.toml && echo 'user="circleci"' >> $HOME/testground/.env.toml
|
|
- run:
|
|
name: "prepare testground home dir and link test plans"
|
|
command: mkdir -p $HOME/testground/plans && ln -s $(pwd)/testplans/lotus-soup $HOME/testground/plans/lotus-soup
|
|
- run:
|
|
name: "go get lotus@master"
|
|
command: cd testplans/lotus-soup && go get github.com/filecoin-project/lotus@master
|
|
- run:
|
|
name: "trigger deals baseline testplan on taas"
|
|
command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/baseline-k8s-3-1.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
|
|
- run:
|
|
name: "trigger payment channel stress testplan on taas"
|
|
command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH
|
|
build-macos:
|
|
description: build darwin lotus binary
|
|
parameters:
|
|
publish:
|
|
default: false
|
|
description: publish github release and homebrew?
|
|
type: boolean
|
|
macos:
|
|
xcode: "12.5.0"
|
|
working_directory: ~/go/src/github.com/filecoin-project/lotus
|
|
steps:
|
|
- prepare:
|
|
linux: false
|
|
darwin: true
|
|
- install_ipfs_macos
|
|
- restore_cache:
|
|
name: restore cargo cache
|
|
key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
|
|
- when:
|
|
condition: << parameters.publish >>
|
|
steps:
|
|
- run: goreleaser release --rm-dist
|
|
- run: ./scripts/generate-checksums.sh
|
|
- run: ./scripts/publish-checksums.sh
|
|
- when:
|
|
condition:
|
|
not: << parameters.publish >>
|
|
steps:
|
|
- run: goreleaser release --rm-dist --snapshot
|
|
- run: ./scripts/generate-checksums.sh
|
|
- store_artifacts:
|
|
path: dist
|
|
- persist_to_workspace:
|
|
root: "."
|
|
paths:
|
|
- dist
|
|
- save_cache:
|
|
name: save cargo cache
|
|
key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }}
|
|
paths:
|
|
- "~/.rustup"
|
|
- "~/.cargo"
|
|
|
|
build-appimage:
|
|
machine:
|
|
image: ubuntu-2004:202111-02
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: "."
|
|
- run:
|
|
name: install appimage-builder
|
|
command: |
|
|
# appimage-builder requires /dev/snd to exist. It creates containers during the testing phase
|
|
# that pass sound devices from the host to the testing container. (hard coded!)
|
|
# https://github.com/AppImageCrafters/appimage-builder/blob/master/appimagebuilder/modules/test/execution_test.py#L54
|
|
# Circleci doesn't provide a working sound device; this is enough to fake it.
|
|
if [ ! -e /dev/snd ]
|
|
then
|
|
sudo mkdir /dev/snd
|
|
sudo mknod /dev/snd/ControlC0 c 1 2
|
|
fi
|
|
|
|
# docs: https://appimage-builder.readthedocs.io/en/latest/intro/install.html
|
|
sudo apt update
|
|
sudo apt install -y python3-pip python3-setuptools patchelf desktop-file-utils libgdk-pixbuf2.0-dev fakeroot strace
|
|
sudo curl -Lo /usr/local/bin/appimagetool https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
|
|
sudo chmod +x /usr/local/bin/appimagetool
|
|
sudo pip3 install appimage-builder
|
|
- run:
|
|
name: install lotus dependencies
|
|
command: sudo apt install ocl-icd-opencl-dev libhwloc-dev
|
|
- run:
|
|
name: build appimage
|
|
command: |
|
|
sed -i "s/version: latest/version: ${CIRCLE_TAG:-latest}/" AppImageBuilder.yml
|
|
make appimage
|
|
- run:
|
|
name: prepare workspace
|
|
command: |
|
|
mkdir appimage
|
|
mv Lotus-*.AppImage appimage
|
|
- persist_to_workspace:
|
|
root: "."
|
|
paths:
|
|
- appimage
|
|
|
|
|
|
gofmt:
|
|
executor: golang
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run:
|
|
command: "! go fmt ./... 2>&1 | read"
|
|
|
|
gen-check:
|
|
executor: golang
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: make deps
|
|
- run: go install golang.org/x/tools/cmd/goimports
|
|
- run: go install github.com/hannahhoward/cbor-gen-for
|
|
- run: make gen
|
|
- run: git --no-pager diff
|
|
- run: git --no-pager diff --quiet
|
|
- run: make docsgen-cli
|
|
- run: git --no-pager diff
|
|
- run: git --no-pager diff --quiet
|
|
|
|
docs-check:
|
|
executor: golang
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run: go install golang.org/x/tools/cmd/goimports
|
|
- run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
|
|
- run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
|
|
- run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker
|
|
- run: make deps
|
|
- run: make docsgen
|
|
- run: zcat build/openrpc/full.json.gz | jq > ../post-openrpc-full
|
|
- run: zcat build/openrpc/miner.json.gz | jq > ../post-openrpc-miner
|
|
- run: zcat build/openrpc/worker.json.gz | jq > ../post-openrpc-worker
|
|
- run: git --no-pager diff
|
|
- run: diff ../pre-openrpc-full ../post-openrpc-full
|
|
- run: diff ../pre-openrpc-miner ../post-openrpc-miner
|
|
- run: diff ../pre-openrpc-worker ../post-openrpc-worker
|
|
- run: git --no-pager diff --quiet
|
|
|
|
lint: &lint
|
|
description: |
|
|
Run golangci-lint.
|
|
parameters:
|
|
executor:
|
|
type: executor
|
|
default: golang
|
|
concurrency:
|
|
type: string
|
|
default: '2'
|
|
description: |
|
|
Concurrency used to run linters. Defaults to 2 because NumCPU is not
|
|
aware of container CPU limits.
|
|
args:
|
|
type: string
|
|
default: ''
|
|
description: |
|
|
Arguments to pass to golangci-lint
|
|
executor: << parameters.executor >>
|
|
steps:
|
|
- install-deps
|
|
- prepare
|
|
- run:
|
|
command: make deps
|
|
no_output_timeout: 30m
|
|
- run:
|
|
name: Lint
|
|
command: |
|
|
golangci-lint run -v --timeout 2m \
|
|
--concurrency << parameters.concurrency >> << parameters.args >>
|
|
lint-all:
|
|
<<: *lint
|
|
|
|
publish:
|
|
description: publish binary artifacts
|
|
executor: ubuntu
|
|
parameters:
|
|
linux:
|
|
default: false
|
|
description: publish linux binaries?
|
|
type: boolean
|
|
appimage:
|
|
default: false
|
|
description: publish appimage binaries?
|
|
type: boolean
|
|
steps:
|
|
- run:
|
|
name: Install git jq curl
|
|
command: apt update && apt install -y git jq curl
|
|
- checkout
|
|
- git_fetch_all_tags
|
|
- checkout
|
|
- install_ipfs
|
|
- attach_workspace:
|
|
at: "."
|
|
- when:
|
|
condition: << parameters.linux >>
|
|
steps:
|
|
- run: ./scripts/build-arch-bundle.sh linux
|
|
- run: ./scripts/publish-arch-release.sh linux
|
|
- when:
|
|
condition: << parameters.appimage >>
|
|
steps:
|
|
- run: ./scripts/build-appimage-bundle.sh
|
|
- run: ./scripts/publish-arch-release.sh appimage
|
|
|
|
publish-snapcraft:
|
|
description: build and push snapcraft
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
resource_class: 2xlarge
|
|
parameters:
|
|
channel:
|
|
type: string
|
|
default: "edge"
|
|
description: snapcraft channel
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: install snapcraft
|
|
command: sudo snap install snapcraft --classic
|
|
- run:
|
|
name: build snap
|
|
command: snapcraft --use-lxd --debug
|
|
- run:
|
|
name: publish snap
|
|
command: |
|
|
pwd
|
|
ls *.snap
|
|
snapcraft upload lotus-filecoin_latest_amd64.snap --release << parameters.channel >>
|
|
|
|
build-and-push-image:
|
|
description: build and push docker images to public AWS ECR registry
|
|
executor: aws-cli/default
|
|
parameters:
|
|
profile-name:
|
|
type: string
|
|
default: "default"
|
|
description: AWS profile name to be configured.
|
|
|
|
aws-access-key-id:
|
|
type: env_var_name
|
|
default: AWS_ACCESS_KEY_ID
|
|
description: >
|
|
AWS access key id for IAM role. Set this to the name of
|
|
the environment variable you will set to hold this
|
|
value, i.e. AWS_ACCESS_KEY.
|
|
|
|
aws-secret-access-key:
|
|
type: env_var_name
|
|
default: AWS_SECRET_ACCESS_KEY
|
|
description: >
|
|
AWS secret key for IAM role. Set this to the name of
|
|
the environment variable you will set to hold this
|
|
value, i.e. AWS_SECRET_ACCESS_KEY.
|
|
|
|
region:
|
|
type: env_var_name
|
|
default: AWS_REGION
|
|
description: >
|
|
Name of env var storing your AWS region information,
|
|
defaults to AWS_REGION
|
|
|
|
account-url:
|
|
type: env_var_name
|
|
default: AWS_ECR_ACCOUNT_URL
|
|
description: >
|
|
Env var storing Amazon ECR account URL that maps to an AWS account,
|
|
e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
|
|
defaults to AWS_ECR_ACCOUNT_URL
|
|
|
|
dockerfile:
|
|
type: string
|
|
default: Dockerfile
|
|
description: Name of dockerfile to use. Defaults to Dockerfile.
|
|
|
|
path:
|
|
type: string
|
|
default: .
|
|
description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
|
|
|
|
extra-build-args:
|
|
type: string
|
|
default: ""
|
|
description: >
|
|
Extra flags to pass to docker build. For examples, see
|
|
https://docs.docker.com/engine/reference/commandline/build
|
|
|
|
repo:
|
|
type: string
|
|
description: Name of an Amazon ECR repository
|
|
|
|
tag:
|
|
type: string
|
|
default: "latest"
|
|
description: A comma-separated string containing docker image tags to build and push (default = latest)
|
|
|
|
target:
|
|
type: string
|
|
default: "lotus-all-in-one"
|
|
description: Docker target to build
|
|
|
|
steps:
|
|
- run:
|
|
name: Confirm that environment variables are set
|
|
command: |
|
|
if [ -z "$AWS_ACCESS_KEY_ID" ]; then
|
|
echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
|
|
circleci-agent step halt
|
|
fi
|
|
|
|
- aws-cli/setup:
|
|
profile-name: <<parameters.profile-name>>
|
|
aws-access-key-id: <<parameters.aws-access-key-id>>
|
|
aws-secret-access-key: <<parameters.aws-secret-access-key>>
|
|
aws-region: <<parameters.region>>
|
|
|
|
- run:
|
|
name: Log into Amazon ECR
|
|
command: |
|
|
aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
|
|
|
|
- checkout
|
|
|
|
- setup_remote_docker:
|
|
version: 19.03.13
|
|
docker_layer_caching: false
|
|
|
|
- run:
|
|
name: Build docker image
|
|
command: |
|
|
registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
|
|
|
|
docker_tag_args=""
|
|
IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
|
|
for tag in "${DOCKER_TAGS[@]}"; do
|
|
docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
|
|
done
|
|
|
|
docker build \
|
|
<<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
|
|
--target <<parameters.target>> \
|
|
-f <<parameters.path>>/<<parameters.dockerfile>> \
|
|
$docker_tag_args \
|
|
<<parameters.path>>
|
|
|
|
- run:
|
|
name: Push image to Amazon ECR
|
|
command: |
|
|
IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
|
|
for tag in "${DOCKER_TAGS[@]}"; do
|
|
docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
|
|
done
|
|
|
|
publish-packer-snap:
|
|
description: build packer image with snap. mainnet only.
|
|
executor:
|
|
name: packer
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: "."
|
|
- packer_build:
|
|
template: tools/packer/lotus-snap.pkr.hcl
|
|
publish-dockerhub:
|
|
description: publish to dockerhub
|
|
machine:
|
|
image: ubuntu-2004:202010-01
|
|
parameters:
|
|
tag:
|
|
type: string
|
|
default: latest
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: dockerhub login
|
|
command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
|
|
- run:
|
|
name: docker build
|
|
command: |
|
|
docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
|
|
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
|
|
if [[ ! -z $CIRCLE_SHA1 ]]; then
|
|
docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
|
|
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
|
|
fi
|
|
if [[ ! -z $CIRCLE_TAG ]]; then
|
|
docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
|
|
docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
|
|
fi
|
|
- run:
|
|
name: docker push
|
|
command: |
|
|
docker push filecoin/lotus:<< parameters.tag >>
|
|
docker push filecoin/lotus-all-in-one:<< parameters.tag >>
|
|
if [[ ! -z $CIRCLE_SHA1 ]]; then
|
|
docker push filecoin/lotus:$CIRCLE_SHA1
|
|
docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
|
|
fi
|
|
if [[ ! -z $CIRCLE_TAG ]]; then
|
|
docker push filecoin/lotus:$CIRCLE_TAG
|
|
docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
|
|
fi
|
|
|
|
workflows:
|
|
version: 2.1
|
|
ci:
|
|
jobs:
|
|
- lint-all:
|
|
concurrency: "16" # expend all docker 2xlarge CPUs.
|
|
- mod-tidy-check
|
|
- gofmt
|
|
- gen-check
|
|
- docs-check
|
|
- test:
|
|
name: test-itest-api
|
|
suite: itest-api
|
|
target: "./itests/api_test.go"
|
|
|
|
- test:
|
|
name: test-itest-batch_deal
|
|
suite: itest-batch_deal
|
|
target: "./itests/batch_deal_test.go"
|
|
|
|
- test:
|
|
name: test-itest-ccupgrade
|
|
suite: itest-ccupgrade
|
|
target: "./itests/ccupgrade_test.go"
|
|
|
|
- test:
|
|
name: test-itest-cli
|
|
suite: itest-cli
|
|
target: "./itests/cli_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deadlines
|
|
suite: itest-deadlines
|
|
target: "./itests/deadlines_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_512mb
|
|
suite: itest-deals_512mb
|
|
target: "./itests/deals_512mb_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_anycid
|
|
suite: itest-deals_anycid
|
|
target: "./itests/deals_anycid_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_concurrent
|
|
suite: itest-deals_concurrent
|
|
target: "./itests/deals_concurrent_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_invalid_utf8_label
|
|
suite: itest-deals_invalid_utf8_label
|
|
target: "./itests/deals_invalid_utf8_label_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_max_staging_deals
|
|
suite: itest-deals_max_staging_deals
|
|
target: "./itests/deals_max_staging_deals_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_offline
|
|
suite: itest-deals_offline
|
|
target: "./itests/deals_offline_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_padding
|
|
suite: itest-deals_padding
|
|
target: "./itests/deals_padding_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_partial_retrieval_dm-level
|
|
suite: itest-deals_partial_retrieval_dm-level
|
|
target: "./itests/deals_partial_retrieval_dm-level_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_partial_retrieval
|
|
suite: itest-deals_partial_retrieval
|
|
target: "./itests/deals_partial_retrieval_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_power
|
|
suite: itest-deals_power
|
|
target: "./itests/deals_power_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_pricing
|
|
suite: itest-deals_pricing
|
|
target: "./itests/deals_pricing_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_publish
|
|
suite: itest-deals_publish
|
|
target: "./itests/deals_publish_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals_retry_deal_no_funds
|
|
suite: itest-deals_retry_deal_no_funds
|
|
target: "./itests/deals_retry_deal_no_funds_test.go"
|
|
|
|
- test:
|
|
name: test-itest-deals
|
|
suite: itest-deals
|
|
target: "./itests/deals_test.go"
|
|
|
|
- test:
|
|
name: test-itest-gas_estimation
|
|
suite: itest-gas_estimation
|
|
target: "./itests/gas_estimation_test.go"
|
|
|
|
- test:
|
|
name: test-itest-gateway
|
|
suite: itest-gateway
|
|
target: "./itests/gateway_test.go"
|
|
|
|
- test:
|
|
name: test-itest-get_messages_in_ts
|
|
suite: itest-get_messages_in_ts
|
|
target: "./itests/get_messages_in_ts_test.go"
|
|
|
|
- test:
|
|
name: test-itest-lite_migration
|
|
suite: itest-lite_migration
|
|
target: "./itests/lite_migration_test.go"
|
|
|
|
- test:
|
|
name: test-itest-lookup_robust_address
|
|
suite: itest-lookup_robust_address
|
|
target: "./itests/lookup_robust_address_test.go"
|
|
|
|
- test:
|
|
name: test-itest-mempool
|
|
suite: itest-mempool
|
|
target: "./itests/mempool_test.go"
|
|
|
|
- test:
|
|
name: test-itest-multisig
|
|
suite: itest-multisig
|
|
target: "./itests/multisig_test.go"
|
|
|
|
- test:
|
|
name: test-itest-net
|
|
suite: itest-net
|
|
target: "./itests/net_test.go"
|
|
|
|
- test:
|
|
name: test-itest-nonce
|
|
suite: itest-nonce
|
|
target: "./itests/nonce_test.go"
|
|
|
|
- test:
|
|
name: test-itest-path_type_filters
|
|
suite: itest-path_type_filters
|
|
target: "./itests/path_type_filters_test.go"
|
|
|
|
- test:
|
|
name: test-itest-paych_api
|
|
suite: itest-paych_api
|
|
target: "./itests/paych_api_test.go"
|
|
|
|
- test:
|
|
name: test-itest-paych_cli
|
|
suite: itest-paych_cli
|
|
target: "./itests/paych_cli_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sdr_upgrade
|
|
suite: itest-sdr_upgrade
|
|
target: "./itests/sdr_upgrade_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_finalize_early
|
|
suite: itest-sector_finalize_early
|
|
target: "./itests/sector_finalize_early_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_make_cc_avail
|
|
suite: itest-sector_make_cc_avail
|
|
target: "./itests/sector_make_cc_avail_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_miner_collateral
|
|
suite: itest-sector_miner_collateral
|
|
target: "./itests/sector_miner_collateral_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_pledge
|
|
suite: itest-sector_pledge
|
|
target: "./itests/sector_pledge_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_prefer_no_upgrade
|
|
suite: itest-sector_prefer_no_upgrade
|
|
target: "./itests/sector_prefer_no_upgrade_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_revert_available
|
|
suite: itest-sector_revert_available
|
|
target: "./itests/sector_revert_available_test.go"
|
|
|
|
- test:
|
|
name: test-itest-sector_terminate
|
|
suite: itest-sector_terminate
|
|
target: "./itests/sector_terminate_test.go"
|
|
|
|
- test:
|
|
name: test-itest-self_sent_txn
|
|
suite: itest-self_sent_txn
|
|
target: "./itests/self_sent_txn_test.go"
|
|
|
|
- test:
|
|
name: test-itest-tape
|
|
suite: itest-tape
|
|
target: "./itests/tape_test.go"
|
|
|
|
- test:
|
|
name: test-itest-verifreg
|
|
suite: itest-verifreg
|
|
target: "./itests/verifreg_test.go"
|
|
|
|
- test:
|
|
name: test-itest-wdpost_check_config
|
|
suite: itest-wdpost_check_config
|
|
target: "./itests/wdpost_check_config_test.go"
|
|
|
|
- test:
|
|
name: test-itest-wdpost_dispute
|
|
suite: itest-wdpost_dispute
|
|
target: "./itests/wdpost_dispute_test.go"
|
|
|
|
- test:
|
|
name: test-itest-wdpost_no_miner_storage
|
|
suite: itest-wdpost_no_miner_storage
|
|
target: "./itests/wdpost_no_miner_storage_test.go"
|
|
|
|
- test:
|
|
name: test-itest-wdpost
|
|
suite: itest-wdpost
|
|
target: "./itests/wdpost_test.go"
|
|
|
|
- test:
|
|
name: test-itest-wdpost_worker_config
|
|
suite: itest-wdpost_worker_config
|
|
target: "./itests/wdpost_worker_config_test.go"
|
|
|
|
- test:
|
|
name: test-itest-worker
|
|
suite: itest-worker
|
|
target: "./itests/worker_test.go"
|
|
|
|
- test:
|
|
name: test-unit-cli
|
|
suite: utest-unit-cli
|
|
target: "./cli/... ./cmd/... ./api/..."
|
|
- test:
|
|
name: test-unit-node
|
|
suite: utest-unit-node
|
|
target: "./node/..."
|
|
- test:
|
|
name: test-unit-rest
|
|
suite: utest-unit-rest
|
|
target: "./api/... ./blockstore/... ./build/... ./chain/... ./cli/... ./cmd/... ./conformance/... ./extern/... ./gateway/... ./journal/... ./lib/... ./markets/... ./node/... ./paychmgr/... ./storage/... ./tools/..."
|
|
- test:
|
|
name: test-unit-storage
|
|
suite: utest-unit-storage
|
|
target: "./storage/... ./extern/..."
|
|
- test:
|
|
go-test-flags: "-run=TestMulticoreSDR"
|
|
suite: multicore-sdr-check
|
|
target: "./storage/sealer/ffiwrapper"
|
|
proofs-log-test: "1"
|
|
- test-conformance:
|
|
suite: conformance
|
|
target: "./conformance"
|
|
- test-conformance:
|
|
name: test-conformance-bleeding-edge
|
|
suite: conformance-bleeding-edge
|
|
target: "./conformance"
|
|
vectors-branch: specs-actors-v7
|
|
- trigger-testplans:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- master
|
|
- build-debug
|
|
- build-linux:
|
|
filters:
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- build-ntwk-calibration:
|
|
filters:
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- build-ntwk-butterfly:
|
|
filters:
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- build-lotus-soup
|
|
- build-macos:
|
|
name: publish-macos
|
|
publish: true
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- /.*/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- build-macos:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- build-appimage:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- publish:
|
|
name: publish-linux
|
|
linux: true
|
|
requires:
|
|
- build-linux
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- /.*/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- publish:
|
|
name: publish-appimage
|
|
appimage: true
|
|
requires:
|
|
- build-appimage
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- /.*/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
- build-and-push-image:
|
|
dockerfile: Dockerfile.lotus
|
|
path: .
|
|
repo: lotus-dev
|
|
tag: '${CIRCLE_SHA1:0:8}'
|
|
target: lotus-all-in-one
|
|
- build-and-push-image:
|
|
dockerfile: Dockerfile.lotus
|
|
path: .
|
|
repo: lotus-test
|
|
tag: '${CIRCLE_SHA1:0:8}'
|
|
target: lotus-test
|
|
- publish-snapcraft:
|
|
name: publish-snapcraft-stable
|
|
channel: stable
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- /.*/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+$/
|
|
- publish-snapcraft:
|
|
name: publish-snapcraft-candidate
|
|
channel: candidate
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- /.*/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+-rc\d+$/
|
|
- publish-dockerhub:
|
|
name: publish-dockerhub
|
|
tag: stable
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- /.*/
|
|
tags:
|
|
only:
|
|
- /^v\d+\.\d+\.\d+(-rc\d+)?$/
|
|
|
|
nightly:
|
|
triggers:
|
|
- schedule:
|
|
cron: "0 0 * * *"
|
|
filters:
|
|
branches:
|
|
only:
|
|
- master
|
|
jobs:
|
|
- publish-snapcraft:
|
|
name: publish-snapcraft-nightly
|
|
channel: edge
|
|
- publish-dockerhub:
|
|
name: publish-dockerhub-nightly
|
|
tag: nightly
|
|
monthly:
|
|
triggers:
|
|
- schedule:
|
|
cron: "0 0 1 * *"
|
|
filters:
|
|
branches:
|
|
only:
|
|
- master
|
|
jobs:
|
|
- publish-packer-snap
|