From b94fd23da25d3fe1a436d5100c8656b720fe8c26 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Thu, 20 Oct 2022 21:15:13 +0100 Subject: [PATCH 1/5] Refactor Lotus Dockerfile This is a major refactor of our dockerfile to support the following - The lotus image will remain as is. - The lotus-test image will be deprecated. - The lotus-all-in-one image will also ship with the lotus-seed and lotus-fountain binaries, which it currently does not. - The lotus-all-in-one image will be built in debug, calibnet, and butterflynet modes in addition to the (current) mainnet mode. - The lotus-all-in-one image will now be published regularly using the following tags: - 1.18.0-rc1 , 1.18.0-rc1-debug, 1.18.0-rc1-calibnet, 1.18.0-rc1-butterflynet . This pattern will be used for all lotus releases, including RC releases. - nightly, nightly-debug, nightly-calibnet, nightly-butterflynet - stable, stable-debug, stable-calibnet, stable-butterflynet --- .circleci/config.yml | 464 ++++++++++++++++++++++------------------- .circleci/gen.go | 4 + .circleci/template.yml | 323 ++++++++++------------------ Dockerfile | 118 +++++++++++ Dockerfile.lotus | 11 +- 5 files changed, 497 insertions(+), 423 deletions(-) create mode 100644 Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c58c1859..44ce76cf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 orbs: aws-cli: circleci/aws-cli@1.3.2 + docker: circleci/docker@2.1.4 executors: golang: @@ -582,129 +583,92 @@ jobs: shell: /bin/bash -o pipefail command: | snapcraft upload *.snap --release << parameters.channel >> - - build-and-push-image: - description: build and push docker images to public AWS ECR registry - executor: aws-cli/default + build-docker: + description: > + Publish to Dockerhub + executor: docker/docker parameters: - profile-name: + image: type: string - default: "default" - description: AWS profile name to be configured. - - aws-access-key-id: - type: env_var_name - default: AWS_ACCESS_KEY_ID + default: lotus description: > - AWS access key id for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_ACCESS_KEY. - - aws-secret-access-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - description: > - AWS secret key for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_SECRET_ACCESS_KEY. - - region: - type: env_var_name - default: AWS_REGION - description: > - Name of env var storing your AWS region information, - defaults to AWS_REGION - - account-url: - type: env_var_name - default: AWS_ECR_ACCOUNT_URL - description: > - Env var storing Amazon ECR account URL that maps to an AWS account, - e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com - defaults to AWS_ECR_ACCOUNT_URL - - dockerfile: + Passed to the docker build process to determine which image in the + Dockerfile should be built. Expected values are `lotus`, + `lotus-all-in-one` + network: type: string - default: Dockerfile - description: Name of dockerfile to use. Defaults to Dockerfile. - - path: - type: string - default: . - description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). - - extra-build-args: + default: "mainnet" + description: > + Passed to the docker build process using GOFLAGS+=-tags=<>. + Expected values are `debug`, `2k`, `calibnet`, `butterflynet`, + `interopnet`. + channel: type: string default: "" description: > - Extra flags to pass to docker build. For examples, see - https://docs.docker.com/engine/reference/commandline/build - - repo: - type: string - description: Name of an Amazon ECR repository - - tag: - type: string - default: "latest" - description: A comma-separated string containing docker image tags to build and push (default = latest) - - target: - type: string - default: "lotus-all-in-one" - description: Docker target to build - + The release channel to use for this image. + push: + type: boolean + default: false + description: > + When true, pushes the image to Dockerhub steps: - - run: - name: Confirm that environment variables are set - command: | - if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..." - circleci-agent step halt - fi - - - aws-cli/setup: - profile-name: <> - aws-access-key-id: <> - aws-secret-access-key: <> - aws-region: <> - - - run: - name: Log into Amazon ECR - command: | - aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - + - setup_remote_docker - checkout - - - setup_remote_docker: - version: 19.03.13 - docker_layer_caching: false - - - run: - name: Build docker image - command: | - registry_id=$(echo $<> | sed "s;\..*;;g") - - docker_tag_args="" - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker_tag_args="$docker_tag_args -t $<>/<>:$tag" - done - - docker build \ - <<#parameters.extra-build-args>><><> \ - --target <> \ - -f <>/<> \ - $docker_tag_args \ - <> - - - run: - name: Push image to Amazon ECR - command: | - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker push $<>/<>:${tag} - done + - docker/check: + docker-username: DOCKERHUB_USERNAME + docker-password: DOCKERHUB_PASSWORD + - when: + condition: + equal: [ mainnet, <> ] + steps: + - when: + condition: > + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> + tag: <> + - run: + name: Docker push + command: | + echo docker push filecoin/<>:<> + if [[ ! -z $CIRCLE_TAG ]]; then + docker image tag filecoin/<>:<> filecoin/<>:"${CIRCLE_TAG}" + echo docker push filecoin/<>:"${CIRCLE_TAG}" + fi + - unless: + condition: <> + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> + - when: + condition: + not: + equal: [ mainnet, <> ] + steps: + - when: + condition: <> + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> + tag: <>-<> + - run: + name: Docker push + command: | + echo docker push filecoin/<>:<>-<> + if [[ ! -z $CIRCLE_TAG ]]; then + docker image tag filecoin/<>:<>-<> filecoin/<>:"${CIRCLE_TAG}"-<> + echo docker push filecoin/<>:"${CIRCLE_TAG}"-<> + fi + - unless: + condition: <> + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> publish-packer-snap: description: build packer image with snap. mainnet only. @@ -714,51 +678,6 @@ jobs: - checkout - packer_build: template: tools/packer/lotus-snap.pkr.hcl - publish-dockerhub: - description: publish to dockerhub - machine: - image: ubuntu-2004:202010-01 - parameters: - tag: - type: string - default: latest - steps: - - checkout - - run: - name: dockerhub login - command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin - - run: - name: docker build - command: | - docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus . - docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus . - docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus . - if [[ ! -z $CIRCLE_SHA1 ]]; then - docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus . - docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus . - docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus . - fi - if [[ ! -z $CIRCLE_TAG ]]; then - docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus . - docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus . - docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus . - fi - - run: - name: docker push - command: | - docker push filecoin/lotus:<< parameters.tag >> - docker push filecoin/lotus-gateway:<< parameters.tag >> - docker push filecoin/lotus-all-in-one:<< parameters.tag >> - if [[ ! -z $CIRCLE_SHA1 ]]; then - docker push filecoin/lotus:$CIRCLE_SHA1 - docker push filecoin/lotus-gateway:$CIRCLE_SHA1 - docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1 - fi - if [[ ! -z $CIRCLE_TAG ]]; then - docker push filecoin/lotus:$CIRCLE_TAG - docker push filecoin/lotus-gateway:$CIRCLE_TAG - docker push filecoin/lotus-all-in-one:$CIRCLE_TAG - fi workflows: version: 2.1 @@ -1176,42 +1095,6 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-and-push-image: - name: "Publish ECR (lotus-all-in-one)" - dockerfile: Dockerfile.lotus - path: . - repo: lotus-dev - tag: '${CIRCLE_SHA1:0:8}' - target: lotus-all-in-one - - build-and-push-image: - name: "Publish ECR (lotus-test)" - dockerfile: Dockerfile.lotus - path: . - repo: lotus-test - tag: '${CIRCLE_SHA1:0:8}' - target: lotus-test - - publish-snapcraft: - name: "Publish Snapcraft (lotus-filecoin / candidate)" - channel: stable - snap-name: lotus-filecoin - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+$/ - - publish-snapcraft: - name: "Publish Snapcraft (lotus-filecoin / candidate)" - channel: candidate - snap-name: lotus-filecoin - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ - publish-snapcraft: name: "Publish Snapcraft (lotus / stable)" channel: stable @@ -1234,9 +1117,10 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+-rc\d+$/ - - publish-dockerhub: - name: "Publish Dockerhub (stable)" - tag: stable + - publish-snapcraft: + name: "Publish Snapcraft (lotus-filecoin / stable)" + channel: stable + snap-name: lotus-filecoin filters: branches: ignore: @@ -1244,9 +1128,10 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+$/ - - publish-dockerhub: - name: "Publish Dockerhub (candidate)" - tag: candidate + - publish-snapcraft: + name: "Publish Snapcraft (lotus-filecoin / candidate)" + channel: candidate + snap-name: lotus-filecoin filters: branches: ignore: @@ -1254,6 +1139,146 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+-rc\d+$/ + - build-docker: + name: "Docker push (lotus-all-in-one / stable / mainnet)" + image: lotus-all-in-one + channel: stable + network: mainnet + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-docker: + name: "Docker push (lotus-all-in-one / candidate / mainnet)" + image: lotus-all-in-one + channel: candidate + network: mainnet + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+-rc\d+$/ + - build-docker: + name: "Docker build (lotus-all-in-one / mainnet)" + image: lotus-all-in-one + network: mainnet + push: false + filters: + branches: + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-docker: + name: "Docker push (lotus-all-in-one / stable / butterflynet)" + image: lotus-all-in-one + channel: stable + network: butterflynet + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-docker: + name: "Docker push (lotus-all-in-one / candidate / butterflynet)" + image: lotus-all-in-one + channel: candidate + network: butterflynet + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+-rc\d+$/ + - build-docker: + name: "Docker build (lotus-all-in-one / butterflynet)" + image: lotus-all-in-one + network: butterflynet + push: false + filters: + branches: + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-docker: + name: "Docker push (lotus-all-in-one / stable / calibnet)" + image: lotus-all-in-one + channel: stable + network: calibnet + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-docker: + name: "Docker push (lotus-all-in-one / candidate / calibnet)" + image: lotus-all-in-one + channel: candidate + network: calibnet + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+-rc\d+$/ + - build-docker: + name: "Docker build (lotus-all-in-one / calibnet)" + image: lotus-all-in-one + network: calibnet + push: false + filters: + branches: + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-docker: + name: "Docker push (lotus-all-in-one / stable / debug)" + image: lotus-all-in-one + channel: stable + network: debug + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+$/ + - build-docker: + name: "Docker push (lotus-all-in-one / candidate / debug)" + image: lotus-all-in-one + channel: candidate + network: debug + push: true + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+-rc\d+$/ + - build-docker: + name: "Docker build (lotus-all-in-one / debug)" + image: lotus-all-in-one + network: debug + push: false + filters: + branches: + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ nightly: triggers: @@ -1265,16 +1290,37 @@ workflows: - master jobs: - publish-snapcraft: - name: "Publish Snapcraft Nightly (lotus-filecoin / edge)" - channel: edge - snap-name: lotus-filecoin - - publish-snapcraft: - name: "Publish Snapcraft Nightly (lotus / edge)" + name: "Publish Snapcraft (lotus / edge)" channel: edge snap-name: lotus - - publish-dockerhub: - name: publish-dockerhub-nightly - tag: nightly + - publish-snapcraft: + name: "Publish Snapcraft (lotus-filecoin / edge)" + channel: edge + snap-name: lotus-filecoin + - build-docker: + name: "Docker (lotus-all-in-one / nightly / mainnet)" + image: lotus-all-in-one + channel: nightly + network: mainnet + push: true + - build-docker: + name: "Docker (lotus-all-in-one / nightly / butterflynet)" + image: lotus-all-in-one + channel: nightly + network: butterflynet + push: true + - build-docker: + name: "Docker (lotus-all-in-one / nightly / calibnet)" + image: lotus-all-in-one + channel: nightly + network: calibnet + push: true + - build-docker: + name: "Docker (lotus-all-in-one / nightly / debug)" + image: lotus-all-in-one + channel: nightly + network: debug + push: true biweekly: triggers: - schedule: diff --git a/.circleci/gen.go b/.circleci/gen.go index 844348e29..def011410 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -106,10 +106,14 @@ func main() { // form the input data. type data struct { + Networks []string + SnapNames []string ItestFiles []string UnitSuites map[string]string } in := data{ + Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, + SnapNames: []string{"lotus", "lotus-filecoin"}, ItestFiles: itests, UnitSuites: func() map[string]string { ret := make(map[string]string) diff --git a/.circleci/template.yml b/.circleci/template.yml index 48542b2b9..0d889a118 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -1,6 +1,7 @@ version: 2.1 orbs: aws-cli: circleci/aws-cli@1.3.2 + docker: circleci/docker@2.1.4 executors: golang: @@ -582,129 +583,92 @@ jobs: shell: /bin/bash -o pipefail command: | snapcraft upload *.snap --release << parameters.channel >> - - build-and-push-image: - description: build and push docker images to public AWS ECR registry - executor: aws-cli/default + build-docker: + description: > + Publish to Dockerhub + executor: docker/docker parameters: - profile-name: + image: type: string - default: "default" - description: AWS profile name to be configured. - - aws-access-key-id: - type: env_var_name - default: AWS_ACCESS_KEY_ID + default: lotus description: > - AWS access key id for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_ACCESS_KEY. - - aws-secret-access-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - description: > - AWS secret key for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_SECRET_ACCESS_KEY. - - region: - type: env_var_name - default: AWS_REGION - description: > - Name of env var storing your AWS region information, - defaults to AWS_REGION - - account-url: - type: env_var_name - default: AWS_ECR_ACCOUNT_URL - description: > - Env var storing Amazon ECR account URL that maps to an AWS account, - e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com - defaults to AWS_ECR_ACCOUNT_URL - - dockerfile: + Passed to the docker build process to determine which image in the + Dockerfile should be built. Expected values are `lotus`, + `lotus-all-in-one` + network: type: string - default: Dockerfile - description: Name of dockerfile to use. Defaults to Dockerfile. - - path: - type: string - default: . - description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). - - extra-build-args: + default: "mainnet" + description: > + Passed to the docker build process using GOFLAGS+=-tags=<>. + Expected values are `debug`, `2k`, `calibnet`, `butterflynet`, + `interopnet`. + channel: type: string default: "" description: > - Extra flags to pass to docker build. For examples, see - https://docs.docker.com/engine/reference/commandline/build - - repo: - type: string - description: Name of an Amazon ECR repository - - tag: - type: string - default: "latest" - description: A comma-separated string containing docker image tags to build and push (default = latest) - - target: - type: string - default: "lotus-all-in-one" - description: Docker target to build - + The release channel to use for this image. + push: + type: boolean + default: false + description: > + When true, pushes the image to Dockerhub steps: - - run: - name: Confirm that environment variables are set - command: | - if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..." - circleci-agent step halt - fi - - - aws-cli/setup: - profile-name: <> - aws-access-key-id: <> - aws-secret-access-key: <> - aws-region: <> - - - run: - name: Log into Amazon ECR - command: | - aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - + - setup_remote_docker - checkout - - - setup_remote_docker: - version: 19.03.13 - docker_layer_caching: false - - - run: - name: Build docker image - command: | - registry_id=$(echo $<> | sed "s;\..*;;g") - - docker_tag_args="" - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker_tag_args="$docker_tag_args -t $<>/<>:$tag" - done - - docker build \ - <<#parameters.extra-build-args>><><> \ - --target <> \ - -f <>/<> \ - $docker_tag_args \ - <> - - - run: - name: Push image to Amazon ECR - command: | - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker push $<>/<>:${tag} - done + - docker/check: + docker-username: DOCKERHUB_USERNAME + docker-password: DOCKERHUB_PASSWORD + - when: + condition: + equal: [ mainnet, <> ] + steps: + - when: + condition: > + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> + tag: <> + - run: + name: Docker push + command: | + echo docker push filecoin/<>:<> + if [["[[ ! -z $CIRCLE_TAG ]]"]]; then + docker image tag filecoin/<>:<> filecoin/<>:"${CIRCLE_TAG}" + echo docker push filecoin/<>:"${CIRCLE_TAG}" + fi + - unless: + condition: <> + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> + - when: + condition: + not: + equal: [ mainnet, <> ] + steps: + - when: + condition: <> + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> + tag: <>-<> + - run: + name: Docker push + command: | + echo docker push filecoin/<>:<>-<> + if [["[[ ! -z $CIRCLE_TAG ]]"]]; then + docker image tag filecoin/<>:<>-<> filecoin/<>:"${CIRCLE_TAG}"-<> + echo docker push filecoin/<>:"${CIRCLE_TAG}"-<> + fi + - unless: + condition: <> + steps: + - docker/build: + image: filecoin/<> + extra_build_args: --target <> --build-arg GOFLAGS=-tags=<> publish-packer-snap: description: build packer image with snap. mainnet only. @@ -714,51 +678,6 @@ jobs: - checkout - packer_build: template: tools/packer/lotus-snap.pkr.hcl - publish-dockerhub: - description: publish to dockerhub - machine: - image: ubuntu-2004:202010-01 - parameters: - tag: - type: string - default: latest - steps: - - checkout - - run: - name: dockerhub login - command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin - - run: - name: docker build - command: | - docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus . - docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus . - docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus . - if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then - docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus . - docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus . - docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus . - fi - if [["[[ ! -z $CIRCLE_TAG ]]"]]; then - docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus . - docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus . - docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus . - fi - - run: - name: docker push - command: | - docker push filecoin/lotus:<< parameters.tag >> - docker push filecoin/lotus-gateway:<< parameters.tag >> - docker push filecoin/lotus-all-in-one:<< parameters.tag >> - if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then - docker push filecoin/lotus:$CIRCLE_SHA1 - docker push filecoin/lotus-gateway:$CIRCLE_SHA1 - docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1 - fi - if [["[[ ! -z $CIRCLE_TAG ]]"]]; then - docker push filecoin/lotus:$CIRCLE_TAG - docker push filecoin/lotus-gateway:$CIRCLE_TAG - docker push filecoin/lotus-all-in-one:$CIRCLE_TAG - fi workflows: version: 2.1 @@ -881,24 +800,11 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-and-push-image: - name: "Publish ECR (lotus-all-in-one)" - dockerfile: Dockerfile.lotus - path: . - repo: lotus-dev - tag: '${CIRCLE_SHA1:0:8}' - target: lotus-all-in-one - - build-and-push-image: - name: "Publish ECR (lotus-test)" - dockerfile: Dockerfile.lotus - path: . - repo: lotus-test - tag: '${CIRCLE_SHA1:0:8}' - target: lotus-test + [[- range .SnapNames]] - publish-snapcraft: - name: "Publish Snapcraft (lotus-filecoin / candidate)" + name: "Publish Snapcraft ([[.]] / stable)" channel: stable - snap-name: lotus-filecoin + snap-name: [[.]] filters: branches: ignore: @@ -907,9 +813,9 @@ workflows: only: - /^v\d+\.\d+\.\d+$/ - publish-snapcraft: - name: "Publish Snapcraft (lotus-filecoin / candidate)" + name: "Publish Snapcraft ([[.]] / candidate)" channel: candidate - snap-name: lotus-filecoin + snap-name: [[.]] filters: branches: ignore: @@ -917,10 +823,14 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+-rc\d+$/ - - publish-snapcraft: - name: "Publish Snapcraft (lotus / stable)" + [[- end]] + [[- range .Networks]] + - build-docker: + name: "Docker push (lotus-all-in-one / stable / [[.]])" + image: lotus-all-in-one channel: stable - snap-name: lotus + network: [[.]] + push: true filters: branches: ignore: @@ -928,10 +838,12 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+$/ - - publish-snapcraft: - name: "Publish Snapcraft (lotus / candidate)" + - build-docker: + name: "Docker push (lotus-all-in-one / candidate / [[.]])" + image: lotus-all-in-one channel: candidate - snap-name: lotus + network: [[.]] + push: true filters: branches: ignore: @@ -939,26 +851,16 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+-rc\d+$/ - - publish-dockerhub: - name: "Publish Dockerhub (stable)" - tag: stable + - build-docker: + name: "Docker build (lotus-all-in-one / [[.]])" + image: lotus-all-in-one + network: [[.]] + push: false filters: branches: - ignore: - - /.*/ - tags: only: - - /^v\d+\.\d+\.\d+$/ - - publish-dockerhub: - name: "Publish Dockerhub (candidate)" - tag: candidate - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+-rc\d+$/ + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ + [[- end]] nightly: triggers: @@ -969,17 +871,20 @@ workflows: only: - master jobs: + [[- range .SnapNames]] - publish-snapcraft: - name: "Publish Snapcraft Nightly (lotus-filecoin / edge)" + name: "Publish Snapcraft ([[.]] / edge)" channel: edge - snap-name: lotus-filecoin - - publish-snapcraft: - name: "Publish Snapcraft Nightly (lotus / edge)" - channel: edge - snap-name: lotus - - publish-dockerhub: - name: publish-dockerhub-nightly - tag: nightly + snap-name: [[.]] + [[- end]] + [[- range .Networks]] + - build-docker: + name: "Docker (lotus-all-in-one / nightly / [[.]])" + image: lotus-all-in-one + channel: nightly + network: [[.]] + push: true + [[- end]] biweekly: triggers: - schedule: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f1c69e03b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,118 @@ +##################################### +FROM golang:1.18.1-buster AS lotus-builder +MAINTAINER Lotus Development Team + +RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev + +ARG RUST_VERSION=nightly +ENV XDG_CACHE_HOME="/tmp" + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \ + chmod +x rustup-init; \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \ + rm rustup-init; \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ + rustup --version; \ + cargo --version; \ + rustc --version; + +COPY ./ /opt/filecoin +WORKDIR /opt/filecoin +RUN make clean deps + +ARG RUSTFLAGS="" +ARG GOFLAGS="" + +RUN make buildall + +##################################### +FROM ubuntu:20.04 AS lotus-base +MAINTAINER Lotus Development Team + +# Base resources +COPY --from=lotus-builder /etc/ssl/certs /etc/ssl/certs +COPY --from=lotus-builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/ +COPY --from=lotus-builder /lib/x86_64-linux-gnu/librt.so.1 /lib/ +COPY --from=lotus-builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/ +COPY --from=lotus-builder /lib/x86_64-linux-gnu/libutil.so.1 /lib/ +COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libltdl.so.7 /lib/ +COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libnuma.so.1 /lib/ +COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5 /lib/ +COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/ + +RUN useradd -r -u 532 -U fc \ + && mkdir -p /etc/OpenCL/vendors \ + && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +##################################### +FROM lotus-base AS lotus +MAINTAINER Lotus Development Team + +COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/ +COPY scripts/docker-lotus-entrypoint.sh / + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_PATH /var/lib/lotus +ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car +ENV DOCKER_LOTUS_IMPORT_WALLET "" + +RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters + +VOLUME /var/lib/lotus +VOLUME /var/tmp/filecoin-proof-parameters + +USER fc + +EXPOSE 1234 + +ENTRYPOINT ["/docker-lotus-entrypoint.sh"] + +CMD ["-help"] + +##################################### +FROM lotus-base AS lotus-all-in-one + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_MINER_PATH /var/lib/lotus-miner +ENV LOTUS_PATH /var/lib/lotus +ENV LOTUS_WORKER_PATH /var/lib/lotus-worker +ENV WALLET_PATH /var/lib/lotus-wallet + +COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-seed /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/ + +RUN mkdir /var/tmp/filecoin-proof-parameters +RUN mkdir /var/lib/lotus +RUN mkdir /var/lib/lotus-miner +RUN mkdir /var/lib/lotus-worker +RUN mkdir /var/lib/lotus-wallet +RUN chown fc: /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus +RUN chown fc: /var/lib/lotus-miner +RUN chown fc: /var/lib/lotus-worker +RUN chown fc: /var/lib/lotus-wallet + + +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/lotus +VOLUME /var/lib/lotus-miner +VOLUME /var/lib/lotus-worker +VOLUME /var/lib/lotus-wallet + +EXPOSE 1234 +EXPOSE 2345 +EXPOSE 3456 +EXPOSE 1777 diff --git a/Dockerfile.lotus b/Dockerfile.lotus index 7037a5813..47e70d4c1 100644 --- a/Dockerfile.lotus +++ b/Dockerfile.lotus @@ -1,3 +1,5 @@ +##### DEPRECATED + FROM golang:1.18.1-buster AS builder-deps MAINTAINER Lotus Development Team @@ -43,7 +45,7 @@ ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE} RUN make clean deps -FROM builder-local AS builder-test +FROM builder-local AS builder-debug MAINTAINER Lotus Development Team WORKDIR /opt/filecoin @@ -59,7 +61,7 @@ WORKDIR /opt/filecoin ARG RUSTFLAGS="" ARG GOFLAGS="" -RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats +RUN make build-devnets FROM ubuntu:20.04 AS base @@ -192,14 +194,13 @@ CMD ["-help"] ### -from base as lotus-all-in-one +FROM base as lotus-all-in-one ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters ENV LOTUS_MINER_PATH /var/lib/lotus-miner ENV LOTUS_PATH /var/lib/lotus ENV LOTUS_WORKER_PATH /var/lib/lotus-worker ENV WALLET_PATH /var/lib/lotus-wallet -ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car COPY --from=builder /opt/filecoin/lotus /usr/local/bin/ COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/ @@ -233,7 +234,7 @@ EXPOSE 3456 EXPOSE 1777 ### -from base as lotus-test +FROM base as lotus-test ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters ENV LOTUS_MINER_PATH /var/lib/lotus-miner From de0c592a9582cf991a68fc258d4d4db79aba9b6c Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Tue, 8 Nov 2022 17:58:35 +0000 Subject: [PATCH 2/5] Port changes from https://github.com/filecoin-project/lotus/pull/9363 --- Dockerfile | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1c69e03b..f4874ed36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,16 +4,26 @@ MAINTAINER Lotus Development Team RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev -ARG RUST_VERSION=nightly ENV XDG_CACHE_HOME="/tmp" +### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.63.0 -RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \ +RUN set -eux; \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ + amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \ + arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac; \ + url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \ + wget "$url"; \ + echo "${rustupSha256} *rustup-init" | sha256sum -c -; \ chmod +x rustup-init; \ - ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \ rm rustup-init; \ chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ rustup --version; \ @@ -22,6 +32,11 @@ RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rust COPY ./ /opt/filecoin WORKDIR /opt/filecoin + +### make configurable filecoin-ffi build +ARG FFI_BUILD_FROM_SOURCE=0 +ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE} + RUN make clean deps ARG RUSTFLAGS="" @@ -35,14 +50,14 @@ MAINTAINER Lotus Development Team # Base resources COPY --from=lotus-builder /etc/ssl/certs /etc/ssl/certs -COPY --from=lotus-builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/ -COPY --from=lotus-builder /lib/x86_64-linux-gnu/librt.so.1 /lib/ -COPY --from=lotus-builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/ -COPY --from=lotus-builder /lib/x86_64-linux-gnu/libutil.so.1 /lib/ -COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libltdl.so.7 /lib/ -COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libnuma.so.1 /lib/ -COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5 /lib/ -COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/ +COPY --from=lotus-builder /lib/*/libdl.so.2 /lib/ +COPY --from=lotus-builder /lib/*/librt.so.1 /lib/ +COPY --from=lotus-builder /lib/*/libgcc_s.so.1 /lib/ +COPY --from=lotus-builder /lib/*/libutil.so.1 /lib/ +COPY --from=lotus-builder /usr/lib/*/libltdl.so.7 /lib/ +COPY --from=lotus-builder /usr/lib/*/libnuma.so.1 /lib/ +COPY --from=lotus-builder /usr/lib/*/libhwloc.so.5 /lib/ +COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/ RUN useradd -r -u 532 -U fc \ && mkdir -p /etc/OpenCL/vendors \ From 75a026827c8ba000c7e89cce96e294c59d4b8e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Nov 2022 11:13:06 +0100 Subject: [PATCH 3/5] circle gen: gofmt --- .circleci/gen.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/gen.go b/.circleci/gen.go index def011410..6cc9cedb1 100644 --- a/.circleci/gen.go +++ b/.circleci/gen.go @@ -106,14 +106,14 @@ func main() { // form the input data. type data struct { - Networks []string - SnapNames []string + Networks []string + SnapNames []string ItestFiles []string UnitSuites map[string]string } in := data{ - Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, - SnapNames: []string{"lotus", "lotus-filecoin"}, + Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"}, + SnapNames: []string{"lotus", "lotus-filecoin"}, ItestFiles: itests, UnitSuites: func() map[string]string { ret := make(map[string]string) From fc1b57da6e486331c3ffa3badf97c8b6aed07e58 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Fri, 11 Nov 2022 15:15:07 +0000 Subject: [PATCH 4/5] Revert accidental changes to Dockerfile.lotus --- Dockerfile.lotus | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile.lotus b/Dockerfile.lotus index 47e70d4c1..e6ae6afe9 100644 --- a/Dockerfile.lotus +++ b/Dockerfile.lotus @@ -45,7 +45,7 @@ ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE} RUN make clean deps -FROM builder-local AS builder-debug +FROM builder-local AS builder-test MAINTAINER Lotus Development Team WORKDIR /opt/filecoin @@ -61,7 +61,7 @@ WORKDIR /opt/filecoin ARG RUSTFLAGS="" ARG GOFLAGS="" -RUN make build-devnets +RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats FROM ubuntu:20.04 AS base @@ -194,13 +194,14 @@ CMD ["-help"] ### -FROM base as lotus-all-in-one +from base as lotus-all-in-one ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters ENV LOTUS_MINER_PATH /var/lib/lotus-miner ENV LOTUS_PATH /var/lib/lotus ENV LOTUS_WORKER_PATH /var/lib/lotus-worker ENV WALLET_PATH /var/lib/lotus-wallet +ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car COPY --from=builder /opt/filecoin/lotus /usr/local/bin/ COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/ @@ -234,7 +235,7 @@ EXPOSE 3456 EXPOSE 1777 ### -FROM base as lotus-test +from base as lotus-test ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters ENV LOTUS_MINER_PATH /var/lib/lotus-miner From 62ee9b55e1471b99d4a73b9e863bcbd8a7abbf53 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Mon, 14 Nov 2022 14:26:57 +0000 Subject: [PATCH 5/5] Set better default DOCKER_LOTUS_IMPORT_SNAPSHOT --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f4874ed36..f3638cc46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,9 +71,10 @@ COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/ COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/ COPY scripts/docker-lotus-entrypoint.sh / +ARG DOCKER_LOTUS_IMPORT_SNAPSHOT https://snapshots.mainnet.filops.net/minimal/latest +ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT} ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters ENV LOTUS_PATH /var/lib/lotus -ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car ENV DOCKER_LOTUS_IMPORT_WALLET "" RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters