From 0ca29acf22795320c000827b3d1713a0b1b06323 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 16:27:19 +0100 Subject: [PATCH 1/7] add job to build docker image and push to AWS ECR private repo --- .circleci/config.yml | 8 +++++ Dockerfile.lotus | 74 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 Dockerfile.lotus diff --git a/.circleci/config.yml b/.circleci/config.yml index 04feeedf3..da8a4b458 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 orbs: go: gotest/tools@0.0.13 + aws-ecr: circleci/aws-ecr@6.15.2 executors: golang: @@ -537,3 +538,10 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+$/ + - aws-ecr/build-and-push-image: + dockerfile: Dockerfile.lotus + no-output-timeout: 30m + path: . + repo: lotus-dev + skip-when-tags-exist: false + tag: '${CIRCLE_SHA1:0:8}' diff --git a/Dockerfile.lotus b/Dockerfile.lotus new file mode 100644 index 000000000..43d8fbc23 --- /dev/null +++ b/Dockerfile.lotus @@ -0,0 +1,74 @@ +FROM golang:1.15.6 AS builder-deps +MAINTAINER Lotus Development Team + +RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev + +ARG RUST_VERSION=nightly +ENV XDG_CACHE_HOME="/tmp" + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \ + chmod +x rustup-init; \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \ + rm rustup-init; \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ + rustup --version; \ + cargo --version; \ + rustc --version; + + +FROM builder-deps AS builder-local +MAINTAINER Lotus Development Team + +COPY ./ /opt/filecoin +WORKDIR /opt/filecoin +RUN make clean deps + + +FROM builder-local AS builder +MAINTAINER Lotus Development Team + +WORKDIR /opt/filecoin + +ARG RUSTFLAGS="" +ARG GOFLAGS="" + +RUN make deps lotus lotus-miner lotus-worker lotus-shed lotus-chainwatch lotus-stats + + +FROM ubuntu:20.04 AS base +MAINTAINER Lotus Development Team + +# Base resources +COPY --from=builder /etc/ssl/certs /etc/ssl/certs +COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/ +COPY --from=builder /lib/x86_64-linux-gnu/librt.so.1 /lib/ +COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/ +COPY --from=builder /lib/x86_64-linux-gnu/libutil.so.1 /lib/ +COPY --from=builder /usr/lib/x86_64-linux-gnu/libltdl.so.7 /lib/ +COPY --from=builder /usr/lib/x86_64-linux-gnu/libnuma.so.1 /lib/ +COPY --from=builder /usr/lib/x86_64-linux-gnu/libhwloc.so.5 /lib/ +COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/ + +RUN useradd -r -u 532 -U fc + + +FROM base AS lotus +MAINTAINER Lotus Development Team + +COPY --from=builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/ + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_PATH /var/lib/lotus + +RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters && chown fc /var/lib/lotus /var/tmp/filecoin-proof-parameters + +USER fc + +ENTRYPOINT ["/usr/local/bin/lotus"] + +CMD ["-help"] From 1a582230cb86addbdc4e56c280304170a5a8adfa Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 18:14:40 +0100 Subject: [PATCH 2/7] wip --- .circleci/config.yml | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da8a4b458..617fe0c4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: go: gotest/tools@0.0.13 - aws-ecr: circleci/aws-ecr@6.15.2 + aws-cli: circleci/aws-cli@1.2 executors: golang: @@ -448,6 +448,81 @@ jobs: name: Publish release command: ./scripts/publish-release.sh + build_and_push: + description: build and push docker images + executor: ubuntu + parameters: + profile-name: + type: string + default: "default" + description: AWS profile name to be configured. + + aws-access-key-id: + type: env_var_name + default: AWS_ACCESS_KEY_ID + description: > + AWS access key id for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_ACCESS_KEY. + + aws-secret-access-key: + type: env_var_name + default: AWS_SECRET_ACCESS_KEY + description: > + AWS secret key for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_SECRET_ACCESS_KEY. + + region: + type: env_var_name + default: AWS_REGION + description: > + Name of env var storing your AWS region information, + defaults to AWS_REGION + + account-url: + type: env_var_name + default: AWS_ECR_ACCOUNT_URL + description: > + Env var storing Amazon ECR account URL that maps to an AWS account, + e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com + defaults to AWS_ECR_ACCOUNT_URL + steps: + - aws-cli/setup: + profile-name: <> + aws-access-key-id: <> + aws-secret-access-key: <> + aws-region: <> + + - run: + name: Log into Amazon ECR + command: | + aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> + + - run: + name: Build docker image + command: | + registry_id=$(echo $<> | sed "s;\..*;;g") + + docker_tag_args="" + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker_tag_args="$docker_tag_args -t $<>/<>:$tag" + done + + docker build \ + <<#parameters.extra-build-args>><><> \ + -f <>/<> \ + $docker_tag_args \ + <> + + - run: + name: Push image to Amazon ECR + command: | + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker push $<>/<>:${tag} + done workflows: version: 2.1 From 5d89d6d3e734c49c04c9c037ee7b238ac9d15fa3 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 19:10:49 +0100 Subject: [PATCH 3/7] example config.yml for public ECR registry --- .circleci/config-public-image.yml | 123 ++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 .circleci/config-public-image.yml diff --git a/.circleci/config-public-image.yml b/.circleci/config-public-image.yml new file mode 100644 index 000000000..35923d61f --- /dev/null +++ b/.circleci/config-public-image.yml @@ -0,0 +1,123 @@ +version: 2.1 +orbs: + aws-cli: circleci/aws-cli@1.3.2 + +jobs: + build-and-push-image: + description: build and push docker images + executor: aws-cli/default + parameters: + profile-name: + type: string + default: "default" + description: AWS profile name to be configured. + + aws-access-key-id: + type: env_var_name + default: AWS_ACCESS_KEY_ID + description: > + AWS access key id for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_ACCESS_KEY. + + aws-secret-access-key: + type: env_var_name + default: AWS_SECRET_ACCESS_KEY + description: > + AWS secret key for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_SECRET_ACCESS_KEY. + + region: + type: env_var_name + default: AWS_REGION + description: > + Name of env var storing your AWS region information, + defaults to AWS_REGION + + account-url: + type: env_var_name + default: AWS_ECR_ACCOUNT_URL + description: > + Env var storing Amazon ECR account URL that maps to an AWS account, + e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com + defaults to AWS_ECR_ACCOUNT_URL + + dockerfile: + type: string + default: Dockerfile + description: Name of dockerfile to use. Defaults to Dockerfile. + + path: + type: string + default: . + description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). + + extra-build-args: + type: string + default: "" + description: > + Extra flags to pass to docker build. For examples, see + https://docs.docker.com/engine/reference/commandline/build + + repo: + type: string + description: Name of an Amazon ECR repository + + tag: + type: string + default: "latest" + description: A comma-separated string containing docker image tags to build and push (default = latest) + + steps: + - aws-cli/setup: + profile-name: <> + aws-access-key-id: <> + aws-secret-access-key: <> + aws-region: <> + + - run: + name: Log into Amazon ECR + command: | + aws ecr get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> + + - checkout + + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: false + + - run: + name: Build docker image + command: | + registry_id=$(echo $<> | sed "s;\..*;;g") + + docker_tag_args="" + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker_tag_args="$docker_tag_args -t $<>/<>:$tag" + done + + docker build \ + <<#parameters.extra-build-args>><><> \ + -f <>/<> \ + $docker_tag_args \ + <> + + - run: + name: Push image to Amazon ECR + command: | + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker push $<>/<>:${tag} + done + +workflows: + version: 2.1 + ci: + jobs: + - build-and-push-image: + dockerfile: Dockerfile.lotus + path: . + repo: lotus-test-dev + tag: '${CIRCLE_SHA1:0:8}' From a9c0263db6cc88335ba18a0020c2e749dac76a4a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 19:25:51 +0100 Subject: [PATCH 4/7] fixup --- .circleci/config.yml | 78 +------------------------------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 617fe0c4e..2b3d34f60 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: go: gotest/tools@0.0.13 - aws-cli: circleci/aws-cli@1.2 + aws-ecr: circleci/aws-ecr@6.15.2 executors: golang: @@ -448,82 +448,6 @@ jobs: name: Publish release command: ./scripts/publish-release.sh - build_and_push: - description: build and push docker images - executor: ubuntu - parameters: - profile-name: - type: string - default: "default" - description: AWS profile name to be configured. - - aws-access-key-id: - type: env_var_name - default: AWS_ACCESS_KEY_ID - description: > - AWS access key id for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_ACCESS_KEY. - - aws-secret-access-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - description: > - AWS secret key for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_SECRET_ACCESS_KEY. - - region: - type: env_var_name - default: AWS_REGION - description: > - Name of env var storing your AWS region information, - defaults to AWS_REGION - - account-url: - type: env_var_name - default: AWS_ECR_ACCOUNT_URL - description: > - Env var storing Amazon ECR account URL that maps to an AWS account, - e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com - defaults to AWS_ECR_ACCOUNT_URL - steps: - - aws-cli/setup: - profile-name: <> - aws-access-key-id: <> - aws-secret-access-key: <> - aws-region: <> - - - run: - name: Log into Amazon ECR - command: | - aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - - - run: - name: Build docker image - command: | - registry_id=$(echo $<> | sed "s;\..*;;g") - - docker_tag_args="" - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker_tag_args="$docker_tag_args -t $<>/<>:$tag" - done - - docker build \ - <<#parameters.extra-build-args>><><> \ - -f <>/<> \ - $docker_tag_args \ - <> - - - run: - name: Push image to Amazon ECR - command: | - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker push $<>/<>:${tag} - done - workflows: version: 2.1 ci: From e3262da9155cf27eba3b1ba66aa3d3fa5176c02b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 19:59:47 +0100 Subject: [PATCH 5/7] remove circleci/aws-ecr orb and push built image to public registry --- .circleci/config-public-image.yml | 123 ------------------------------ .circleci/config.yml | 117 ++++++++++++++++++++++++++-- 2 files changed, 112 insertions(+), 128 deletions(-) delete mode 100644 .circleci/config-public-image.yml diff --git a/.circleci/config-public-image.yml b/.circleci/config-public-image.yml deleted file mode 100644 index 35923d61f..000000000 --- a/.circleci/config-public-image.yml +++ /dev/null @@ -1,123 +0,0 @@ -version: 2.1 -orbs: - aws-cli: circleci/aws-cli@1.3.2 - -jobs: - build-and-push-image: - description: build and push docker images - executor: aws-cli/default - parameters: - profile-name: - type: string - default: "default" - description: AWS profile name to be configured. - - aws-access-key-id: - type: env_var_name - default: AWS_ACCESS_KEY_ID - description: > - AWS access key id for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_ACCESS_KEY. - - aws-secret-access-key: - type: env_var_name - default: AWS_SECRET_ACCESS_KEY - description: > - AWS secret key for IAM role. Set this to the name of - the environment variable you will set to hold this - value, i.e. AWS_SECRET_ACCESS_KEY. - - region: - type: env_var_name - default: AWS_REGION - description: > - Name of env var storing your AWS region information, - defaults to AWS_REGION - - account-url: - type: env_var_name - default: AWS_ECR_ACCOUNT_URL - description: > - Env var storing Amazon ECR account URL that maps to an AWS account, - e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com - defaults to AWS_ECR_ACCOUNT_URL - - dockerfile: - type: string - default: Dockerfile - description: Name of dockerfile to use. Defaults to Dockerfile. - - path: - type: string - default: . - description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). - - extra-build-args: - type: string - default: "" - description: > - Extra flags to pass to docker build. For examples, see - https://docs.docker.com/engine/reference/commandline/build - - repo: - type: string - description: Name of an Amazon ECR repository - - tag: - type: string - default: "latest" - description: A comma-separated string containing docker image tags to build and push (default = latest) - - steps: - - aws-cli/setup: - profile-name: <> - aws-access-key-id: <> - aws-secret-access-key: <> - aws-region: <> - - - run: - name: Log into Amazon ECR - command: | - aws ecr get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - - - checkout - - - setup_remote_docker: - version: 19.03.13 - docker_layer_caching: false - - - run: - name: Build docker image - command: | - registry_id=$(echo $<> | sed "s;\..*;;g") - - docker_tag_args="" - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker_tag_args="$docker_tag_args -t $<>/<>:$tag" - done - - docker build \ - <<#parameters.extra-build-args>><><> \ - -f <>/<> \ - $docker_tag_args \ - <> - - - run: - name: Push image to Amazon ECR - command: | - IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" - for tag in "${DOCKER_TAGS[@]}"; do - docker push $<>/<>:${tag} - done - -workflows: - version: 2.1 - ci: - jobs: - - build-and-push-image: - dockerfile: Dockerfile.lotus - path: . - repo: lotus-test-dev - tag: '${CIRCLE_SHA1:0:8}' diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b3d34f60..349351b64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: go: gotest/tools@0.0.13 - aws-ecr: circleci/aws-ecr@6.15.2 + aws-cli: circleci/aws-cli@1.3.2 executors: golang: @@ -448,6 +448,115 @@ jobs: name: Publish release command: ./scripts/publish-release.sh + build-and-push-image: + description: build and push docker images to public AWS ECR registry + executor: aws-cli/default + parameters: + profile-name: + type: string + default: "default" + description: AWS profile name to be configured. + + aws-access-key-id: + type: env_var_name + default: AWS_ACCESS_KEY_ID + description: > + AWS access key id for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_ACCESS_KEY. + + aws-secret-access-key: + type: env_var_name + default: AWS_SECRET_ACCESS_KEY + description: > + AWS secret key for IAM role. Set this to the name of + the environment variable you will set to hold this + value, i.e. AWS_SECRET_ACCESS_KEY. + + region: + type: env_var_name + default: AWS_REGION + description: > + Name of env var storing your AWS region information, + defaults to AWS_REGION + + account-url: + type: env_var_name + default: AWS_ECR_ACCOUNT_URL + description: > + Env var storing Amazon ECR account URL that maps to an AWS account, + e.g. {awsAccountNum}.dkr.ecr.us-west-2.amazonaws.com + defaults to AWS_ECR_ACCOUNT_URL + + dockerfile: + type: string + default: Dockerfile + description: Name of dockerfile to use. Defaults to Dockerfile. + + path: + type: string + default: . + description: Path to the directory containing your Dockerfile and build context. Defaults to . (working directory). + + extra-build-args: + type: string + default: "" + description: > + Extra flags to pass to docker build. For examples, see + https://docs.docker.com/engine/reference/commandline/build + + repo: + type: string + description: Name of an Amazon ECR repository + + tag: + type: string + default: "latest" + description: A comma-separated string containing docker image tags to build and push (default = latest) + + steps: + - aws-cli/setup: + profile-name: <> + aws-access-key-id: <> + aws-secret-access-key: <> + aws-region: <> + + - run: + name: Log into Amazon ECR + command: | + aws ecr get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> + + - checkout + + - setup_remote_docker: + version: 19.03.13 + docker_layer_caching: false + + - run: + name: Build docker image + command: | + registry_id=$(echo $<> | sed "s;\..*;;g") + + docker_tag_args="" + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker_tag_args="$docker_tag_args -t $<>/<>:$tag" + done + + docker build \ + <<#parameters.extra-build-args>><><> \ + -f <>/<> \ + $docker_tag_args \ + <> + + - run: + name: Push image to Amazon ECR + command: | + IFS="," read -ra DOCKER_TAGS \<<< "<< parameters.tag >>" + for tag in "${DOCKER_TAGS[@]}"; do + docker push $<>/<>:${tag} + done + workflows: version: 2.1 ci: @@ -537,10 +646,8 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+$/ - - aws-ecr/build-and-push-image: + - build-and-push-image: dockerfile: Dockerfile.lotus - no-output-timeout: 30m path: . - repo: lotus-dev - skip-when-tags-exist: false + repo: lotus-test-dev tag: '${CIRCLE_SHA1:0:8}' From 24cc70f18efa00fee8c440e0aa8057e5fa3089c2 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 20:00:16 +0100 Subject: [PATCH 6/7] use ecr-public cmd --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 349351b64..bb77ade1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -524,7 +524,7 @@ jobs: - run: name: Log into Amazon ECR command: | - aws ecr get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> + aws ecr-public get-login-password --region $<> --profile <> | docker login --username AWS --password-stdin $<> - checkout From 790bcc783030c7e78e3f16fe9e0ccb64bac3a82b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 20 Jan 2021 20:04:15 +0100 Subject: [PATCH 7/7] change name of image to `lotus-dev` --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb77ade1f..41b18b048 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -649,5 +649,5 @@ workflows: - build-and-push-image: dockerfile: Dockerfile.lotus path: . - repo: lotus-test-dev + repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}'