Refactor Lotus Dockerfile

This is a major refactor of our dockerfile to support the following
- The lotus image will remain as is.
- The lotus-test image will be deprecated.
- The lotus-all-in-one image will also ship with the lotus-seed and lotus-fountain binaries, which it currently does not.
- The lotus-all-in-one image will be built in debug, calibnet, and butterflynet modes in addition to the (current) mainnet mode.
- The lotus-all-in-one image will now be published regularly using the following tags:
  - 1.18.0-rc1 , 1.18.0-rc1-debug, 1.18.0-rc1-calibnet, 1.18.0-rc1-butterflynet . This pattern will be used for all lotus releases, including RC releases.
  - nightly, nightly-debug, nightly-calibnet, nightly-butterflynet
  - stable, stable-debug, stable-calibnet, stable-butterflynet

.circleci/config.yml

@@ -1,6 +1,7 @@
 version: 2.1
 orbs:
   aws-cli: circleci/aws-cli@1.3.2
+  docker: circleci/docker@2.1.4
 
 executors:
   golang:
@@ -582,129 +583,92 @@ jobs:
           shell: /bin/bash -o pipefail
           command: |
             snapcraft upload *.snap --release << parameters.channel >>
-
+  build-docker:
-  build-and-push-image:
+    description: >
-    description: build and push docker images to public AWS ECR registry
+      Publish to Dockerhub
-    executor: aws-cli/default
+    executor: docker/docker
     parameters:
-      profile-name:
+      image:
         type: string
-        default: "default"
+        default: lotus
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
         description: >
-          AWS access key id for IAM role. Set this to the name of
+          Passed to the docker build process to determine which image in the
-          the environment variable you will set to hold this
+          Dockerfile should be built. Expected values are `lotus`,
-          value, i.e. AWS_ACCESS_KEY.
+          `lotus-all-in-one`
-
+      network:
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
         type: string
-        default: Dockerfile
+        default: "mainnet"
-        description: Name of dockerfile to use. Defaults to Dockerfile.
+        description: >
-
+          Passed to the docker build process using GOFLAGS+=-tags=<<network>>.
-      path:
+          Expected values are `debug`, `2k`, `calibnet`, `butterflynet`,
-        type: string
+          `interopnet`.
-        default: .
+      channel:
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
         type: string
         default: ""
         description: >
-          Extra flags to pass to docker build. For examples, see
+          The release channel to use for this image.
-          https://docs.docker.com/engine/reference/commandline/build
+      push:
-
+        type: boolean
-      repo:
+        default: false
-        type: string
+        description: >
-        description: Name of an Amazon ECR repository
+          When true, pushes the image to Dockerhub
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-      target:
-        type: string
-        default: "lotus-all-in-one"
-        description: Docker target to build
-
     steps:
-      - run:
+      - setup_remote_docker
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
       - checkout
-
+      - docker/check:
-      - setup_remote_docker:
+          docker-username: DOCKERHUB_USERNAME
-          version: 19.03.13
+          docker-password: DOCKERHUB_PASSWORD
-          docker_layer_caching: false
+      - when:
-
+          condition:
+            equal: [ mainnet, <<parameters.network>> ]
+          steps:
+            - when:
+                condition: <parameters.push>>
+                steps:
+                  - docker/build:
+                      image: filecoin/<<parameters.image>>
+                      extra_build_args: --target <<parameters.image>>
+                      tag: <<parameters.channel>>
                   - run:
-          name: Build docker image
+                      name: Docker push
                       command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
+                        echo docker push filecoin/<<parameters.image>>:<<parameters.channel>>
-
+                        if [[ ! -z $CIRCLE_TAG ]]; then
-            docker_tag_args=""
+                          docker image tag filecoin/<<parameters.image>>:<<parameters.channel>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
+                          echo docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"
-            for tag in "${DOCKER_TAGS[@]}"; do
+                        fi
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
+            - unless:
-            done
+                condition: <<parameters.push>>
-
+                steps:
-            docker build \
+                  - docker/build:
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+                      image: filecoin/<<parameters.image>>
-              --target <<parameters.target>> \
+                      extra_build_args: --target <<parameters.image>>
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
+      - when:
-              $docker_tag_args \
+          condition:
-              <<parameters.path>>
+            not:
-
+              equal: [ mainnet, <<parameters.network>> ]
+          steps:
+            - when:
+                condition: <<parameters.push>>
+                steps:
+                  - docker/build:
+                      image: filecoin/<<parameters.image>>
+                      extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
+                      tag: <<parameters.channel>>-<<parameters.network>>
                   - run:
-          name: Push image to Amazon ECR
+                      name: Docker push
                       command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
+                        echo docker push filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>>
-            for tag in "${DOCKER_TAGS[@]}"; do
+                        if [[ ! -z $CIRCLE_TAG ]]; then
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
+                          docker image tag filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>>
-            done
+                          echo docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>>
+                        fi
+            - unless:
+                condition: <<parameters.push>>
+                steps:
+                  - docker/build:
+                      image: filecoin/<<parameters.image>>
+                      extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
 
   publish-packer-snap:
     description: build packer image with snap. mainnet only.
@@ -714,51 +678,6 @@ jobs:
       - checkout
       - packer_build:
           template: tools/packer/lotus-snap.pkr.hcl
-  publish-dockerhub:
-    description: publish to dockerhub
-    machine:
-      image: ubuntu-2004:202010-01
-    parameters:
-      tag:
-        type: string
-        default: latest
-    steps:
-      - checkout
-      - run:
-          name: dockerhub login
-          command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
-      - run:
-          name: docker build
-          command: |
-            docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
-            docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus .
-            docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
-            if [[ ! -z $CIRCLE_SHA1 ]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
-              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
-            fi
-            if [[ ! -z $CIRCLE_TAG ]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
-              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
-            fi
-      - run:
-          name: docker push
-          command: |
-            docker push filecoin/lotus:<< parameters.tag >>
-            docker push filecoin/lotus-gateway:<< parameters.tag >>
-            docker push filecoin/lotus-all-in-one:<< parameters.tag >>
-            if [[ ! -z $CIRCLE_SHA1 ]]; then
-              docker push filecoin/lotus:$CIRCLE_SHA1
-              docker push filecoin/lotus-gateway:$CIRCLE_SHA1
-              docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
-            fi
-            if [[ ! -z $CIRCLE_TAG ]]; then
-              docker push filecoin/lotus:$CIRCLE_TAG
-              docker push filecoin/lotus-gateway:$CIRCLE_TAG
-              docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
-            fi
 
 workflows:
   version: 2.1
@@ -1176,42 +1095,6 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-and-push-image:
-          name: "Publish ECR (lotus-all-in-one)"
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-          target: lotus-all-in-one
-      - build-and-push-image:
-          name: "Publish ECR (lotus-test)"
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-test
-          tag: '${CIRCLE_SHA1:0:8}'
-          target: lotus-test
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / candidate)"
-          channel: stable
-          snap-name: lotus-filecoin
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+$/
-      - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / candidate)"
-          channel: candidate
-          snap-name: lotus-filecoin
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
       - publish-snapcraft:
           name: "Publish Snapcraft (lotus / stable)"
           channel: stable
@@ -1234,9 +1117,10 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+-rc\d+$/
-      - publish-dockerhub:
+      - publish-snapcraft:
-          name: "Publish Dockerhub (stable)"
+          name: "Publish Snapcraft (lotus-filecoin / stable)"
-          tag: stable
+          channel: stable
+          snap-name: lotus-filecoin
           filters:
             branches:
               ignore:
@@ -1244,9 +1128,10 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
-      - publish-dockerhub:
+      - publish-snapcraft:
-          name: "Publish Dockerhub (candidate)"
+          name: "Publish Snapcraft (lotus-filecoin / candidate)"
-          tag: candidate
+          channel: candidate
+          snap-name: lotus-filecoin
           filters:
             branches:
               ignore:
@@ -1254,6 +1139,146 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / stable / mainnet)"
+          image: lotus-all-in-one
+          channel: stable
+          network: mainnet
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / candidate / mainnet)"
+          image: lotus-all-in-one
+          channel: candidate
+          network: mainnet
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-docker:
+          name: "Docker build (lotus-all-in-one / mainnet)"
+          image: lotus-all-in-one
+          network: mainnet
+          push: false
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / stable / butterflynet)"
+          image: lotus-all-in-one
+          channel: stable
+          network: butterflynet
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / candidate / butterflynet)"
+          image: lotus-all-in-one
+          channel: candidate
+          network: butterflynet
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-docker:
+          name: "Docker build (lotus-all-in-one / butterflynet)"
+          image: lotus-all-in-one
+          network: butterflynet
+          push: false
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / stable / calibnet)"
+          image: lotus-all-in-one
+          channel: stable
+          network: calibnet
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / candidate / calibnet)"
+          image: lotus-all-in-one
+          channel: candidate
+          network: calibnet
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-docker:
+          name: "Docker build (lotus-all-in-one / calibnet)"
+          image: lotus-all-in-one
+          network: calibnet
+          push: false
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / stable / debug)"
+          image: lotus-all-in-one
+          channel: stable
+          network: debug
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+$/
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / candidate / debug)"
+          image: lotus-all-in-one
+          channel: candidate
+          network: debug
+          push: true
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v\d+\.\d+\.\d+-rc\d+$/
+      - build-docker:
+          name: "Docker build (lotus-all-in-one / debug)"
+          image: lotus-all-in-one
+          network: debug
+          push: false
+          filters:
+            branches:
+              only:
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
 
   nightly:
     triggers:
@@ -1265,16 +1290,37 @@ workflows:
                 - master
     jobs:
       - publish-snapcraft:
-          name: "Publish Snapcraft Nightly (lotus-filecoin / edge)"
+          name: "Publish Snapcraft (lotus / edge)"
-          channel: edge
-          snap-name: lotus-filecoin
-      - publish-snapcraft:
-          name: "Publish Snapcraft Nightly (lotus / edge)"
           channel: edge
           snap-name: lotus
-      - publish-dockerhub:
+      - publish-snapcraft:
-          name: publish-dockerhub-nightly
+          name: "Publish Snapcraft (lotus-filecoin / edge)"
-          tag: nightly
+          channel: edge
+          snap-name: lotus-filecoin
+      - build-docker:
+          name: "Docker (lotus-all-in-one / nightly / mainnet)"
+          image: lotus-all-in-one
+          channel: nightly
+          network: mainnet
+          push: true
+      - build-docker:
+          name: "Docker (lotus-all-in-one / nightly / butterflynet)"
+          image: lotus-all-in-one
+          channel: nightly
+          network: butterflynet
+          push: true
+      - build-docker:
+          name: "Docker (lotus-all-in-one / nightly / calibnet)"
+          image: lotus-all-in-one
+          channel: nightly
+          network: calibnet
+          push: true
+      - build-docker:
+          name: "Docker (lotus-all-in-one / nightly / debug)"
+          image: lotus-all-in-one
+          channel: nightly
+          network: debug
+          push: true
   biweekly:
     triggers:
       - schedule:

.circleci/gen.go

@@ -106,10 +106,14 @@ func main() {
 
 	// form the input data.
 	type data struct {
+    Networks []string
+    SnapNames []string
 		ItestFiles []string
 		UnitSuites map[string]string
 	}
 	in := data{
+    Networks: []string{"mainnet", "butterflynet", "calibnet", "debug"},
+    SnapNames: []string{"lotus", "lotus-filecoin"},
 		ItestFiles: itests,
 		UnitSuites: func() map[string]string {
 			ret := make(map[string]string)

.circleci/template.yml

@@ -1,6 +1,7 @@
 version: 2.1
 orbs:
   aws-cli: circleci/aws-cli@1.3.2
+  docker: circleci/docker@2.1.4
 
 executors:
   golang:
@@ -582,129 +583,92 @@ jobs:
           shell: /bin/bash -o pipefail
           command: |
             snapcraft upload *.snap --release << parameters.channel >>
-
+  build-docker:
-  build-and-push-image:
+    description: >
-    description: build and push docker images to public AWS ECR registry
+      Publish to Dockerhub
-    executor: aws-cli/default
+    executor: docker/docker
     parameters:
-      profile-name:
+      image:
         type: string
-        default: "default"
+        default: lotus
-        description: AWS profile name to be configured.
-
-      aws-access-key-id:
-        type: env_var_name
-        default: AWS_ACCESS_KEY_ID
         description: >
-          AWS access key id for IAM role. Set this to the name of
+          Passed to the docker build process to determine which image in the
-          the environment variable you will set to hold this
+          Dockerfile should be built. Expected values are `lotus`,
-          value, i.e. AWS_ACCESS_KEY.
+          `lotus-all-in-one`
-
+      network:
-      aws-secret-access-key:
-        type: env_var_name
-        default: AWS_SECRET_ACCESS_KEY
-        description: >
-          AWS secret key for IAM role. Set this to the name of
-          the environment variable you will set to hold this
-          value, i.e. AWS_SECRET_ACCESS_KEY.
-
-      region:
-        type: env_var_name
-        default: AWS_REGION
-        description: >
-          Name of env var storing your AWS region information,
-          defaults to AWS_REGION
-
-      account-url:
-        type: env_var_name
-        default: AWS_ECR_ACCOUNT_URL
-        description: >
-          Env var storing Amazon ECR account URL that maps to an AWS account,
-          e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com
-          defaults to AWS_ECR_ACCOUNT_URL
-
-      dockerfile:
         type: string
-        default: Dockerfile
+        default: "mainnet"
-        description: Name of dockerfile to use. Defaults to Dockerfile.
+        description: >
-
+          Passed to the docker build process using GOFLAGS+=-tags=<<network>>.
-      path:
+          Expected values are `debug`, `2k`, `calibnet`, `butterflynet`,
-        type: string
+          `interopnet`.
-        default: .
+      channel:
-        description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory).
-
-      extra-build-args:
         type: string
         default: ""
         description: >
-          Extra flags to pass to docker build. For examples, see
+          The release channel to use for this image.
-          https://docs.docker.com/engine/reference/commandline/build
+      push:
-
+        type: boolean
-      repo:
+        default: false
-        type: string
+        description: >
-        description: Name of an Amazon ECR repository
+          When true, pushes the image to Dockerhub
-
-      tag:
-        type: string
-        default: "latest"
-        description: A comma-separated string containing docker image tags to build and push (default = latest)
-
-      target:
-        type: string
-        default: "lotus-all-in-one"
-        description: Docker target to build
-
     steps:
-      - run:
+      - setup_remote_docker
-          name: Confirm that environment variables are set
-          command: |
-            if [ -z "$AWS_ACCESS_KEY_ID" ]; then
-              echo "No AWS_ACCESS_KEY_ID is set. Skipping build-and-push job ..."
-              circleci-agent step halt
-            fi
-
-      - aws-cli/setup:
-          profile-name: <<parameters.profile-name>>
-          aws-access-key-id: <<parameters.aws-access-key-id>>
-          aws-secret-access-key: <<parameters.aws-secret-access-key>>
-          aws-region: <<parameters.region>>
-
-      - run:
-          name: Log into Amazon ECR
-          command: |
-            aws ecr-public get-login-password --region $<<parameters.region>> --profile <<parameters.profile-name>> | docker login --username AWS --password-stdin $<<parameters.account-url>>
-
       - checkout
-
+      - docker/check:
-      - setup_remote_docker:
+          docker-username: DOCKERHUB_USERNAME
-          version: 19.03.13
+          docker-password: DOCKERHUB_PASSWORD
-          docker_layer_caching: false
+      - when:
-
+          condition:
+            equal: [ mainnet, <<parameters.network>> ]
+          steps:
+            - when:
+                condition: <parameters.push>>
+                steps:
+                  - docker/build:
+                      image: filecoin/<<parameters.image>>
+                      extra_build_args: --target <<parameters.image>>
+                      tag: <<parameters.channel>>
                   - run:
-          name: Build docker image
+                      name: Docker push
                       command: |
-            registry_id=$(echo $<<parameters.account-url>> | sed "s;\..*;;g")
+                        echo docker push filecoin/<<parameters.image>>:<<parameters.channel>>
-
+                        if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
-            docker_tag_args=""
+                          docker image tag filecoin/<<parameters.image>>:<<parameters.channel>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
+                          echo docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"
-            for tag in "${DOCKER_TAGS[@]}"; do
+                        fi
-              docker_tag_args="$docker_tag_args -t $<<parameters.account-url>>/<<parameters.repo>>:$tag"
+            - unless:
-            done
+                condition: <<parameters.push>>
-
+                steps:
-            docker build \
+                  - docker/build:
-              <<#parameters.extra-build-args>><<parameters.extra-build-args>><</parameters.extra-build-args>> \
+                      image: filecoin/<<parameters.image>>
-              --target <<parameters.target>> \
+                      extra_build_args: --target <<parameters.image>>
-              -f <<parameters.path>>/<<parameters.dockerfile>> \
+      - when:
-              $docker_tag_args \
+          condition:
-              <<parameters.path>>
+            not:
-
+              equal: [ mainnet, <<parameters.network>> ]
+          steps:
+            - when:
+                condition: <<parameters.push>>
+                steps:
+                  - docker/build:
+                      image: filecoin/<<parameters.image>>
+                      extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
+                      tag: <<parameters.channel>>-<<parameters.network>>
                   - run:
-          name: Push image to Amazon ECR
+                      name: Docker push
                       command: |
-            IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>"
+                        echo docker push filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>>
-            for tag in "${DOCKER_TAGS[@]}"; do
+                        if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
-              docker push $<<parameters.account-url>>/<<parameters.repo>>:${tag}
+                          docker image tag filecoin/<<parameters.image>>:<<parameters.channel>>-<<parameters.network>> filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>>
-            done
+                          echo docker push filecoin/<<parameters.image>>:"${CIRCLE_TAG}"-<<parameters.network>>
+                        fi
+            - unless:
+                condition: <<parameters.push>>
+                steps:
+                  - docker/build:
+                      image: filecoin/<<parameters.image>>
+                      extra_build_args: --target <<parameters.image>> --build-arg GOFLAGS=-tags=<<parameters.network>>
 
   publish-packer-snap:
     description: build packer image with snap. mainnet only.
@@ -714,51 +678,6 @@ jobs:
       - checkout
       - packer_build:
           template: tools/packer/lotus-snap.pkr.hcl
-  publish-dockerhub:
-    description: publish to dockerhub
-    machine:
-      image: ubuntu-2004:202010-01
-    parameters:
-      tag:
-        type: string
-        default: latest
-    steps:
-      - checkout
-      - run:
-          name: dockerhub login
-          command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin
-      - run:
-          name: docker build
-          command: |
-            docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus .
-            docker build --target lotus-gateway -t filecoin/lotus-gateway:<< parameters.tag >> -f Dockerfile.lotus .
-            docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus .
-            if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus .
-              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_SHA1 -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus .
-            fi
-            if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
-              docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus .
-              docker build --target lotus-gateway -t filecoin/lotus-gateway:$CIRCLE_TAG -f Dockerfile.lotus .
-              docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus .
-            fi
-      - run:
-          name: docker push
-          command: |
-            docker push filecoin/lotus:<< parameters.tag >>
-            docker push filecoin/lotus-gateway:<< parameters.tag >>
-            docker push filecoin/lotus-all-in-one:<< parameters.tag >>
-            if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then
-              docker push filecoin/lotus:$CIRCLE_SHA1
-              docker push filecoin/lotus-gateway:$CIRCLE_SHA1
-              docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1
-            fi
-            if [["[[ ! -z $CIRCLE_TAG ]]"]]; then
-              docker push filecoin/lotus:$CIRCLE_TAG
-              docker push filecoin/lotus-gateway:$CIRCLE_TAG
-              docker push filecoin/lotus-all-in-one:$CIRCLE_TAG
-            fi
 
 workflows:
   version: 2.1
@@ -881,24 +800,11 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+(-rc\d+)?$/
-      - build-and-push-image:
+      [[- range .SnapNames]]
-          name: "Publish ECR (lotus-all-in-one)"
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-dev
-          tag: '${CIRCLE_SHA1:0:8}'
-          target: lotus-all-in-one
-      - build-and-push-image:
-          name: "Publish ECR (lotus-test)"
-          dockerfile: Dockerfile.lotus
-          path: .
-          repo: lotus-test
-          tag: '${CIRCLE_SHA1:0:8}'
-          target: lotus-test
       - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / candidate)"
+          name: "Publish Snapcraft ([[.]] / stable)"
           channel: stable
-          snap-name: lotus-filecoin
+          snap-name: [[.]]
           filters:
             branches:
               ignore:
@@ -907,9 +813,9 @@ workflows:
               only:
                 - /^v\d+\.\d+\.\d+$/
       - publish-snapcraft:
-          name: "Publish Snapcraft (lotus-filecoin / candidate)"
+          name: "Publish Snapcraft ([[.]] / candidate)"
           channel: candidate
-          snap-name: lotus-filecoin
+          snap-name: [[.]]
           filters:
             branches:
               ignore:
@@ -917,10 +823,14 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+-rc\d+$/
-      - publish-snapcraft:
+      [[- end]]
-          name: "Publish Snapcraft (lotus / stable)"
+      [[- range .Networks]]
+      - build-docker:
+          name: "Docker push (lotus-all-in-one / stable / [[.]])"
+          image: lotus-all-in-one
           channel: stable
-          snap-name: lotus
+          network: [[.]]
+          push: true
           filters:
             branches:
               ignore:
@@ -928,10 +838,12 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+$/
-      - publish-snapcraft:
+      - build-docker:
-          name: "Publish Snapcraft (lotus / candidate)"
+          name: "Docker push (lotus-all-in-one / candidate / [[.]])"
+          image: lotus-all-in-one
           channel: candidate
-          snap-name: lotus
+          network: [[.]]
+          push: true
           filters:
             branches:
               ignore:
@@ -939,26 +851,16 @@ workflows:
             tags:
               only:
                 - /^v\d+\.\d+\.\d+-rc\d+$/
-      - publish-dockerhub:
+      - build-docker:
-          name: "Publish Dockerhub (stable)"
+          name: "Docker build (lotus-all-in-one / [[.]])"
-          tag: stable
+          image: lotus-all-in-one
+          network: [[.]]
+          push: false
           filters:
             branches:
-              ignore:
-                - /.*/
-            tags:
               only:
-                - /^v\d+\.\d+\.\d+$/
+                - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/
-      - publish-dockerhub:
+      [[- end]]
-          name: "Publish Dockerhub (candidate)"
-          tag: candidate
-          filters:
-            branches:
-              ignore:
-                - /.*/
-            tags:
-              only:
-                - /^v\d+\.\d+\.\d+-rc\d+$/
 
   nightly:
     triggers:
@@ -969,17 +871,20 @@ workflows:
               only:
                 - master
     jobs:
+      [[- range .SnapNames]]
       - publish-snapcraft:
-          name: "Publish Snapcraft Nightly (lotus-filecoin / edge)"
+          name: "Publish Snapcraft ([[.]] / edge)"
           channel: edge
-          snap-name: lotus-filecoin
+          snap-name: [[.]]
-      - publish-snapcraft:
+      [[- end]]
-          name: "Publish Snapcraft Nightly (lotus / edge)"
+      [[- range .Networks]]
-          channel: edge
+      - build-docker:
-          snap-name: lotus
+          name: "Docker (lotus-all-in-one / nightly / [[.]])"
-      - publish-dockerhub:
+          image: lotus-all-in-one
-          name: publish-dockerhub-nightly
+          channel: nightly
-          tag: nightly
+          network: [[.]]
+          push: true
+      [[- end]]
   biweekly:
     triggers:
       - schedule:

Dockerfile

@@ -0,0 +1,118 @@
+#####################################
+FROM golang:1.18.1-buster AS lotus-builder
+MAINTAINER Lotus Development Team
+
+RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev
+
+ARG RUST_VERSION=nightly
+ENV XDG_CACHE_HOME="/tmp"
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+
+RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+COPY ./ /opt/filecoin
+WORKDIR /opt/filecoin
+RUN make clean deps
+
+ARG RUSTFLAGS=""
+ARG GOFLAGS=""
+
+RUN make buildall
+
+#####################################
+FROM ubuntu:20.04 AS lotus-base
+MAINTAINER Lotus Development Team
+
+# Base resources
+COPY --from=lotus-builder /etc/ssl/certs                           /etc/ssl/certs
+COPY --from=lotus-builder /lib/x86_64-linux-gnu/libdl.so.2         /lib/
+COPY --from=lotus-builder /lib/x86_64-linux-gnu/librt.so.1         /lib/
+COPY --from=lotus-builder /lib/x86_64-linux-gnu/libgcc_s.so.1      /lib/
+COPY --from=lotus-builder /lib/x86_64-linux-gnu/libutil.so.1       /lib/
+COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libltdl.so.7   /lib/
+COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libnuma.so.1   /lib/
+COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5  /lib/
+COPY --from=lotus-builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/
+
+RUN useradd -r -u 532 -U fc \
+ && mkdir -p /etc/OpenCL/vendors \
+ && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+#####################################
+FROM lotus-base AS lotus
+MAINTAINER Lotus Development Team
+
+COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/
+COPY scripts/docker-lotus-entrypoint.sh /
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_PATH /var/lib/lotus
+ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
+ENV DOCKER_LOTUS_IMPORT_WALLET ""
+
+RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters
+
+VOLUME /var/lib/lotus
+VOLUME /var/tmp/filecoin-proof-parameters
+
+USER fc
+
+EXPOSE 1234
+
+ENTRYPOINT ["/docker-lotus-entrypoint.sh"]
+
+CMD ["-help"]
+
+#####################################
+FROM lotus-base AS lotus-all-in-one
+
+ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
+ENV LOTUS_MINER_PATH /var/lib/lotus-miner
+ENV LOTUS_PATH /var/lib/lotus
+ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
+ENV WALLET_PATH /var/lib/lotus-wallet
+
+COPY --from=lotus-builder /opt/filecoin/lotus          /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-seed     /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-shed     /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-wallet   /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-gateway  /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-miner    /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-worker   /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-stats    /usr/local/bin/
+COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/
+
+RUN mkdir /var/tmp/filecoin-proof-parameters
+RUN mkdir /var/lib/lotus
+RUN mkdir /var/lib/lotus-miner
+RUN mkdir /var/lib/lotus-worker
+RUN mkdir /var/lib/lotus-wallet
+RUN chown fc: /var/tmp/filecoin-proof-parameters
+RUN chown fc: /var/lib/lotus
+RUN chown fc: /var/lib/lotus-miner
+RUN chown fc: /var/lib/lotus-worker
+RUN chown fc: /var/lib/lotus-wallet
+
+
+VOLUME /var/tmp/filecoin-proof-parameters
+VOLUME /var/lib/lotus
+VOLUME /var/lib/lotus-miner
+VOLUME /var/lib/lotus-worker
+VOLUME /var/lib/lotus-wallet
+
+EXPOSE 1234
+EXPOSE 2345
+EXPOSE 3456
+EXPOSE 1777

Dockerfile.lotus

@@ -1,3 +1,5 @@
+##### DEPRECATED
+
 FROM golang:1.18.1-buster AS builder-deps
 MAINTAINER Lotus Development Team
 
@@ -43,7 +45,7 @@ ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
 RUN make clean deps
 
 
-FROM builder-local AS builder-test
+FROM builder-local AS builder-debug
 MAINTAINER Lotus Development Team
 
 WORKDIR /opt/filecoin
@@ -59,7 +61,7 @@ WORKDIR /opt/filecoin
 ARG RUSTFLAGS=""
 ARG GOFLAGS=""
 
-RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway lotus-stats
+RUN make build-devnets
 
 
 FROM ubuntu:20.04 AS base
@@ -192,14 +194,13 @@ CMD ["-help"]
 
 
 ###
-from base as lotus-all-in-one
+FROM base as lotus-all-in-one
 
 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_MINER_PATH /var/lib/lotus-miner
 ENV LOTUS_PATH /var/lib/lotus
 ENV LOTUS_WORKER_PATH /var/lib/lotus-worker
 ENV WALLET_PATH /var/lib/lotus-wallet
-ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car
 
 COPY --from=builder /opt/filecoin/lotus      /usr/local/bin/
 COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/
@@ -233,7 +234,7 @@ EXPOSE 3456
 EXPOSE 1777
 
 ###
-from base as lotus-test
+FROM base as lotus-test
 
 ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters
 ENV LOTUS_MINER_PATH /var/lib/lotus-miner