diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a69a4a49..1c77c2128 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -735,6 +735,45 @@ jobs: - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" + publish-dockerhub: + description: publish to dockerhub + machine: + image: ubuntu-2004:202010-01 + parameters: + tag: + type: string + default: latest + steps: + - checkout + - run: + name: dockerhub login + command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin + - run: + name: docker build + command: | + docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus . + docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus . + if [[ ! -z $CIRCLE_SHA1 ]]; then + docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus . + docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus . + fi + if [[ ! -z $CIRCLE_TAG ]]; then + docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus . + docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus . + fi + - run: + name: docker push + command: | + docker push filecoin/lotus:<< parameters.tag >> + docker push filecoin/lotus-all-in-one:<< parameters.tag >> + if [[ ! -z $CIRCLE_SHA1 ]]; then + docker push filecoin/lotus:$CIRCLE_SHA1 + docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1 + fi + if [[ ! -z $CIRCLE_TAG ]]; then + docker push filecoin/lotus:$CIRCLE_TAG + docker push filecoin/lotus-all-in-one:$CIRCLE_TAG + fi workflows: version: 2.1 @@ -1017,6 +1056,16 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-dockerhub: + name: publish-dockerhub + tag: stable + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ nightly: triggers: @@ -1030,3 +1079,6 @@ workflows: - publish-snapcraft: name: publish-snapcraft-nightly channel: edge + - publish-dockerhub: + name: publish-dockerhub-nightly + tag: nightly diff --git a/.circleci/template.yml b/.circleci/template.yml index fb59f23ea..27036ab26 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -735,6 +735,45 @@ jobs: - packer/build: template: tools/packer/lotus.pkr.hcl args: "-var ci_workspace_bins=./linux-nerpanet -var lotus_network=nerpanet -var git_tag=$CIRCLE_TAG" + publish-dockerhub: + description: publish to dockerhub + machine: + image: ubuntu-2004:202010-01 + parameters: + tag: + type: string + default: latest + steps: + - checkout + - run: + name: dockerhub login + command: echo $DOCKERHUB_PASSWORD | docker login --username $DOCKERHUB_USERNAME --password-stdin + - run: + name: docker build + command: | + docker build --target lotus -t filecoin/lotus:<< parameters.tag >> -f Dockerfile.lotus . + docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:<< parameters.tag >> -f Dockerfile.lotus . + if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then + docker build --target lotus -t filecoin/lotus:$CIRCLE_SHA1 -f Dockerfile.lotus . + docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_SHA1 -f Dockerfile.lotus . + fi + if [["[[ ! -z $CIRCLE_TAG ]]"]]; then + docker build --target lotus -t filecoin/lotus:$CIRCLE_TAG -f Dockerfile.lotus . + docker build --target lotus-all-in-one -t filecoin/lotus-all-in-one:$CIRCLE_TAG -f Dockerfile.lotus . + fi + - run: + name: docker push + command: | + docker push filecoin/lotus:<< parameters.tag >> + docker push filecoin/lotus-all-in-one:<< parameters.tag >> + if [["[[ ! -z $CIRCLE_SHA1 ]]"]]; then + docker push filecoin/lotus:$CIRCLE_SHA1 + docker push filecoin/lotus-all-in-one:$CIRCLE_SHA1 + fi + if [["[[ ! -z $CIRCLE_TAG ]]"]]; then + docker push filecoin/lotus:$CIRCLE_TAG + docker push filecoin/lotus-all-in-one:$CIRCLE_TAG + fi workflows: version: 2.1 @@ -887,6 +926,16 @@ workflows: tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - publish-dockerhub: + name: publish-dockerhub + tag: stable + filters: + branches: + ignore: + - /.*/ + tags: + only: + - /^v\d+\.\d+\.\d+(-rc\d+)?$/ nightly: triggers: @@ -900,3 +949,6 @@ workflows: - publish-snapcraft: name: publish-snapcraft-nightly channel: edge + - publish-dockerhub: + name: publish-dockerhub-nightly + tag: nightly diff --git a/Dockerfile.lotus b/Dockerfile.lotus index 0b43ef806..72c609305 100644 --- a/Dockerfile.lotus +++ b/Dockerfile.lotus @@ -36,7 +36,7 @@ WORKDIR /opt/filecoin ARG RUSTFLAGS="" ARG GOFLAGS="" -RUN make deps lotus lotus-miner lotus-worker lotus-shed lotus-chainwatch lotus-stats +RUN make lotus lotus-miner lotus-worker lotus-shed lotus-wallet lotus-gateway FROM ubuntu:20.04 AS base @@ -56,19 +56,173 @@ COPY --from=builder /usr/lib/x86_64-linux-gnu/libOpenCL.so.1 /lib/ RUN useradd -r -u 532 -U fc +### FROM base AS lotus MAINTAINER Lotus Development Team -COPY --from=builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus /usr/local/bin/ COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/ +COPY scripts/docker-lotus-entrypoint.sh / ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters ENV LOTUS_PATH /var/lib/lotus +ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1 +ENV LOTUS_JAEGER_AGENT_PORT 6831 +ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car +ENV DOCKER_LOTUS_IMPORT_WALLET "" -RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters && chown fc /var/lib/lotus /var/tmp/filecoin-proof-parameters +RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters + +VOLUME /var/lib/lotus +VOLUME /var/tmp/filecoin-proof-parameters USER fc -ENTRYPOINT ["/usr/local/bin/lotus"] +EXPOSE 1234 + +ENTRYPOINT ["/docker-lotus-entrypoint.sh"] CMD ["-help"] + +### +FROM base AS lotus-wallet +MAINTAINER Lotus Development Team + +COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/ + +ENV WALLET_PATH /var/lib/lotus-wallet +ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1 +ENV LOTUS_JAEGER_AGENT_PORT 6831 + +RUN mkdir /var/lib/lotus-wallet +RUN chown fc: /var/lib/lotus-wallet + +VOLUME /var/lib/lotus-wallet + +USER fc + +EXPOSE 1777 + +ENTRYPOINT ["/usr/local/bin/lotus-wallet"] + +CMD ["-help"] + +### +FROM base AS lotus-gateway +MAINTAINER Lotus Development Team + +COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/ + +ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1 +ENV LOTUS_JAEGER_AGENT_PORT 6831 +ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http + +USER fc + +EXPOSE 1234 + +ENTRYPOINT ["/usr/local/bin/lotus-gateway"] + +CMD ["-help"] + + +### +FROM base AS lotus-miner +MAINTAINER Lotus Development Team + +COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/ +COPY scripts/docker-lotus-miner-entrypoint.sh / + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http +ENV LOTUS_MINER_PATH /var/lib/lotus-miner +ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1 +ENV LOTUS_JAEGER_AGENT_PORT 6831 +ENV DOCKER_LOTUS_MINER_INIT true + +RUN mkdir /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus-miner /var/tmp/filecoin-proof-parameters + +VOLUME /var/lib/lotus-miner +VOLUME /var/tmp/filecoin-proof-parameters + +USER fc + +EXPOSE 2345 + +ENTRYPOINT ["/docker-lotus-miner-entrypoint.sh"] + +CMD ["-help"] + + +### +FROM base AS lotus-worker +MAINTAINER Lotus Development Team + +COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/ + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http +ENV LOTUS_WORKER_PATH /var/lib/lotus-worker +ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1 +ENV LOTUS_JAEGER_AGENT_PORT 6831 + +RUN mkdir /var/lib/lotus-worker +RUN chown fc: /var/lib/lotus-worker + +VOLUME /var/lib/lotus-worker + +USER fc + +EXPOSE 3456 + +ENTRYPOINT ["/usr/local/bin/lotus-worker"] + +CMD ["-help"] + + +### +from base as lotus-all-in-one + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV FULLNODE_API_INFO /ip4/127.0.0.1/tcp/1234/http +ENV LOTUS_JAEGER_AGENT_HOST 127.0.0.1 +ENV LOTUS_JAEGER_AGENT_PORT 6831 +ENV LOTUS_MINER_PATH /var/lib/lotus-miner +ENV LOTUS_PATH /var/lib/lotus +ENV LOTUS_WORKER_PATH /var/lib/lotus-worker +ENV MINER_API_INFO /ip4/127.0.0.1/tcp/2345/http +ENV WALLET_PATH /var/lib/lotus-wallet +ENV DOCKER_LOTUS_IMPORT_SNAPSHOT https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car +ENV DOCKER_LOTUS_MINER_INIT true + +COPY --from=builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus-shed /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus-wallet /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus-gateway /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus-miner /usr/local/bin/ +COPY --from=builder /opt/filecoin/lotus-worker /usr/local/bin/ + +RUN mkdir /var/tmp/filecoin-proof-parameters +RUN mkdir /var/lib/lotus +RUN mkdir /var/lib/lotus-miner +RUN mkdir /var/lib/lotus-worker +RUN mkdir /var/lib/lotus-wallet +RUN chown fc: /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus +RUN chown fc: /var/lib/lotus-miner +RUN chown fc: /var/lib/lotus-worker +RUN chown fc: /var/lib/lotus-wallet + + +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/lotus +VOLUME /var/lib/lotus-miner +VOLUME /var/lib/lotus-worker +VOLUME /var/lib/lotus-wallet + +EXPOSE 1234 +EXPOSE 2345 +EXPOSE 3456 +EXPOSE 1777 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 000000000..b962d5cc2 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,145 @@ +# By default, this docker-compose file will start a lotus fullnode +# +# Some directives have been left commented out so they serve as an +# example for more advanced use. +# +# To provide a custom configuration file, or automatically import +# a wallet, uncomment the "configs" or "secrets" sections. +# +# start on a single node: +# +# docker-compose up +# +# start on docker swarm: +# +# docker swarm init (if you haven't already) +# docker stack deploy -c docker-compose.yaml mylotuscluster +# +# for more information, please visit docs.filecoin.io + +version: "3.8" + +volumes: + parameters: + lotus-repo: + lotus-miner-repo: + lotus-worker-repo: + +configs: + lotus-config-toml: + file: /path/to/lotus/config.toml + lotus-miner-config-toml: + file: /path/to/lotus-miner/config.toml + +secrets: + lotus-wallet: + file: /path/to/exported/lotus/wallet + +services: + lotus: + build: + context: . + target: lotus + dockerfile: Dockerfile.lotus + image: filecoin/lotus + volumes: + - parameters:/var/tmp/filecoin-proof-parameters + - lotus-repo:/var/lib/lotus + ports: + - 1234:1234 + environment: + - LOTUS_JAEGER_AGENT_HOST=jaeger + - LOTUS_JAEGER_AGENT_PORT=6831 + # - DOCKER_LOTUS_IMPORT_WALLET=/tmp/wallet + deploy: + restart_policy: + condition: on-failure + delay: 30s + # configs: + # - source: lotus-config-toml + # target: /var/lib/lotus/config.toml + # secrets: + # - source: lotus-wallet + # target: /tmp/wallet + command: + - daemon + lotus-gateway: + build: + context: . + target: lotus-gateway + dockerfile: Dockerfile.lotus + image: filecoin/lotus-gateway + depends_on: + - lotus + ports: + - 1235:1234 + environment: + - FULLNODE_API_INFO=/dns/lotus/tcp/1234/http + - LOTUS_JAEGER_AGENT_HOST=jaeger + - LOTUS_JAEGER_AGENT_PORT=6831 + deploy: + restart_policy: + condition: on-failure + delay: 30s + command: + - run + # + # Uncomment to run miner software + # + # lotus-miner: + # build: + # context: . + # target: lotus-miner + # dockerfile: Dockerfile.lotus + # image: filecoin/lotus-miner + # volumes: + # - parameters:/var/tmp/filecoin-proof-parameters + # - lotus-miner-repo:/var/lib/lotus-miner + # depends_on: + # - lotus + # ports: + # - 2345:2345 + # environment: + # - FULLNODE_API_INFO=/dns/lotus/tcp/1234/http + # - LOTUS_JAEGER_AGENT_HOST=jaeger + # - LOTUS_JAEGER_AGENT_PORT=6831 + # deploy: + # restart_policy: + # condition: on-failure + # delay: 30s + # configs: + # - source: lotus-miner-config-toml + # - target: /var/lib/lotus-miner/config.toml + # command: + # - run + # lotus-worker: + # build: + # context: . + # target: lotus-worker + # dockerfile: Dockerfile.lotus + # image: filecoin/lotus-worker + # volumes: + # - parameters:/var/tmp/filecoin-proof-parameters + # - lotus-worker-repo:/var/lib/lotus-worker + # depends_on: + # - lotus-worker + # environment: + # - MINER_API_INFO=/dns/lotus-miner/tcp/1234/http + # - LOTUS_JAEGER_AGENT_HOST=jaeger + # - LOTUS_JAEGER_AGENT_PORT=6831 + # deploy: + # restart_policy: + # condition: on-failure + # delay: 30s + # replicas: 2 + # command: + # - run + jaeger: + image: jaegertracing/all-in-one + ports: + - "6831:6831/udp" + - "16686:16686" + deploy: + restart_policy: + condition: on-failure + delay: 30s diff --git a/scripts/docker-lotus-entrypoint.sh b/scripts/docker-lotus-entrypoint.sh new file mode 100755 index 000000000..308a4b6eb --- /dev/null +++ b/scripts/docker-lotus-entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [ ! -z DOCKER_LOTUS_IMPORT_SNAPSHOT ]; then + GATE="$LOTUS_PATH"/date_initialized + # Don't init if already initialized. + if [ ! -f "$GATE" ]; then + echo importing minimal snapshot + /usr/local/bin/lotus daemon --import-snapshot "$DOCKER_LOTUS_IMPORT_SNAPSHOT" --halt-after-import + # Block future inits + date > "$GATE" + fi +fi + +# import wallet, if provided +if [ ! -z DOCKER_LOTUS_IMPORT_WALLET ]; then + /usr/local/bin/lotus-shed keyinfo import "$DOCKER_LOTUS_IMPORT_WALLET" +fi + +exec /usr/local/bin/lotus $@ diff --git a/scripts/docker-lotus-miner-entrypoint.sh b/scripts/docker-lotus-miner-entrypoint.sh new file mode 100755 index 000000000..1cb153176 --- /dev/null +++ b/scripts/docker-lotus-miner-entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [ ! -z DOCKER_LOTUS_MINER_INIT ]; then + GATE="$LOTUS_PATH"/date_initialized + + # Don't init if already initialized. + if [ -f "GATE" ]; then + echo lotus-miner already initialized. + exit 0 + fi + + echo starting init + /usr/local/bin/lotus-miner init + + # Block future inits + date > "$GATE" +fi + +exec /usr/local/bin/lotus-miner $@