diff --git a/README.md b/README.md index de55507..4e2f5c7 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# lotus-stack \ No newline at end of file +# lotus-stack + +Stack definitions for Lotus node deployment. + +[stack documentation](./stack-orchestrator/stacks/lotus-node/README.md) diff --git a/stack-orchestrator/compose/docker-compose-lotus-node.yml b/stack-orchestrator/compose/docker-compose-lotus-node.yml new file mode 100644 index 0000000..a99ef47 --- /dev/null +++ b/stack-orchestrator/compose/docker-compose-lotus-node.yml @@ -0,0 +1,41 @@ +volumes: + parameters: + lotus-repo: + lotus_shared: + +services: + lotus: + environment: + - DOCKER_LOTUS_IMPORT_SNAPSHOT=/var/lotus-snapshots/snapshot + - LOTUS_FEVM_ENABLEETHRPC=true + - LOTUS_API_LISTENADDRESS=/ip4/0.0.0.0/tcp/1234/http + - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/1235,/ip6/::/tcp/1235,/ip4/0.0.0.0/udp/1235/quic-v1,/ip6/::/udp/1235/quic-v1,/ip4/0.0.0.0/udp/1235/quic-v1/webtransport,/ip6/::/udp/1235/quic-v1/webtransport + - LOTUS_NETWORK=${CERC_LOTUS_NETWORK:-mainnet} + - LOTUS_EVENTS_ENABLEACTOREVENTSAPI=${CERC_LOTUS_EVENTS_ENABLEACTOREVENTSAPI:-true} + - LOTUS_INDEX_ENABLEMSGINDEX=${CERC_LOTUS_INDEX_ENABLEMSGINDEX:-true} + - LOTUS_CHAINSTORE_ENABLESPLITSTORE=${CERC_LOTUS_CHAINSTORE_ENABLESPLITSTORE:-true} + - GOLOG_LOG_FMT=${CERC_GOLOG_LOG_FMT:-json} + - LOTUS_FEVM_ETHTXHASHMAPPINGLIFETIMEDAYS=${CERC_LOTUS_FEVM_ETHTXHASHMAPPINGLIFETIMEDAYS:-0} + - LOTUS_SKIP_APPLY_TS_MESSAGE_CALL_WITH_GAS=${CERC_LOTUS_SKIP_APPLY_TS_MESSAGE_CALL_WITH_GAS:-0} + - LOTUS_FVM_CONCURRENCY=${CERC_LOTUS_FVM_CONCURRENCY:-4} + image: cerc/lotus-node:local + volumes: + - ${CERC_LOTUS_SNAPSHOT_PATH}:/var/lotus-snapshots/snapshot + - parameters:/var/tmp/filecoin-proof-parameters + - lotus-repo:/var/lib/lotus + - lotus_shared:/root/.lotus + - ../config/lotus-node/docker-lotus-entrypoint.sh:/docker-lotus-entrypoint.sh + ports: + - "1234" + - "1235" + deploy: + restart_policy: + condition: on-failure + delay: 30s + entrypoint: ["sh", "/docker-lotus-entrypoint.sh"] + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "1234"] + interval: 30s + timeout: 10s + retries: 40 + start_period: 3s diff --git a/stack-orchestrator/config/lotus-node/docker-lotus-entrypoint.sh b/stack-orchestrator/config/lotus-node/docker-lotus-entrypoint.sh new file mode 100755 index 0000000..fd5e939 --- /dev/null +++ b/stack-orchestrator/config/lotus-node/docker-lotus-entrypoint.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +TARGET_LOTUS="/usr/local/bin/lotus" +TARGET_LOTUS_SHED="/usr/local/bin/lotus-shed" + +if [ "$LOTUS_NETWORK" = "calibration" ]; then + echo "Using Calibration Lotus binary" + TARGET_LOTUS="/usr/local/bin/lotus-calibnet" + TARGET_LOTUS_SHED="/usr/local/bin/lotus-calibnet-shed" +fi + +if [ ! -z $DOCKER_LOTUS_IMPORT_SNAPSHOT ]; then + GATE="$LOTUS_PATH"/date_initialized + # Don't init if already initialized. + if [ ! -f "$GATE" ]; then + echo importing minimal snapshot + echo "yes" | $TARGET_LOTUS daemon --import-snapshot "$DOCKER_LOTUS_IMPORT_SNAPSHOT" --halt-after-import + # Block future inits + date > "$GATE" + fi +fi + +# import wallet, if provided +if [ ! -z $DOCKER_LOTUS_IMPORT_WALLET ]; then + $TARGET_LOTUS_SHED keyinfo import "$DOCKER_LOTUS_IMPORT_WALLET" +fi + +exec $TARGET_LOTUS daemon $@ diff --git a/stack-orchestrator/container-build/cerc-lotus-node/Dockerfile b/stack-orchestrator/container-build/cerc-lotus-node/Dockerfile new file mode 100644 index 0000000..80a5112 --- /dev/null +++ b/stack-orchestrator/container-build/cerc-lotus-node/Dockerfile @@ -0,0 +1,162 @@ +# Dockerfile is copied from https://git.vdb.to/cerc-io/lotus/src/tag/v1.27.0-rc3-interal-0.0.1/Dockerfile +# This file has changes for building Lotus binary for Calibration network and installing netcat for healthcheck + +##################################### +FROM golang:1.21.7-bullseye AS lotus-builder +MAINTAINER Lotus Development Team + +RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev + +ENV XDG_CACHE_HOME="/tmp" + +### taken from https://github.com/rust-lang/docker-rust/blob/master/1.63.0/buster/Dockerfile +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.63.0 + +RUN set -eux; \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ + amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \ + arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac; \ + url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \ + wget "$url"; \ + echo "${rustupSha256} *rustup-init" | sha256sum -c -; \ + chmod +x rustup-init; \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \ + rm rustup-init; \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ + rustup --version; \ + cargo --version; \ + rustc --version; + +COPY ./ /opt/filecoin + +WORKDIR /opt/filecoin + +### make configurable filecoin-ffi build +ARG FFI_BUILD_FROM_SOURCE=0 +ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE} + +RUN make clean deps + +ARG RUSTFLAGS="" +ARG GOFLAGS="" + +RUN make buildall + +### Create separate directory for calibnet +COPY ./ /opt/filecoin-calibnet +WORKDIR /opt/filecoin-calibnet + +### Build for calibnet +RUN make clean deps +RUN make calibnet + +##################################### +FROM ubuntu:20.04 AS lotus-base +MAINTAINER Lotus Development Team + +# Base resources +COPY --from=lotus-builder /etc/ssl/certs /etc/ssl/certs +COPY --from=lotus-builder /lib/*/libdl.so.2 /lib/ +COPY --from=lotus-builder /lib/*/librt.so.1 /lib/ +COPY --from=lotus-builder /lib/*/libgcc_s.so.1 /lib/ +COPY --from=lotus-builder /lib/*/libutil.so.1 /lib/ +COPY --from=lotus-builder /usr/lib/*/libltdl.so.7 /lib/ +COPY --from=lotus-builder /usr/lib/*/libnuma.so.1 /lib/ +COPY --from=lotus-builder /usr/lib/*/libhwloc.so.* /lib/ +COPY --from=lotus-builder /usr/lib/*/libOpenCL.so.1 /lib/ + +RUN useradd -r -u 532 -U fc \ + && mkdir -p /etc/OpenCL/vendors \ + && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd + +##################################### +FROM lotus-base AS lotus +MAINTAINER Lotus Development Team + +COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/ + +# Copy and create separate binaries for calibnet +COPY --from=lotus-builder /opt/filecoin-calibnet/lotus /usr/local/bin/lotus-calibnet +COPY --from=lotus-builder /opt/filecoin-calibnet/lotus-shed /usr/local/bin/lotus-calibnet-shed + +COPY scripts/docker-lotus-entrypoint.sh / + +ARG DOCKER_LOTUS_IMPORT_SNAPSHOT=https://forest-archive.chainsafe.dev/latest/mainnet/ +ENV DOCKER_LOTUS_IMPORT_SNAPSHOT ${DOCKER_LOTUS_IMPORT_SNAPSHOT} +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_PATH /var/lib/lotus +ENV DOCKER_LOTUS_IMPORT_WALLET "" + +RUN mkdir /var/lib/lotus /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus /var/tmp/filecoin-proof-parameters + +VOLUME /var/lib/lotus +VOLUME /var/tmp/filecoin-proof-parameters + +USER fc + +EXPOSE 1234 + +ENTRYPOINT ["/docker-lotus-entrypoint.sh"] + +CMD ["-help"] + +##################################### +FROM lotus-base AS lotus-all-in-one + +# Install netcat for healthcheck +RUN apt-get update && apt-get install -y netcat && apt-get install -y iproute2 + +ENV FILECOIN_PARAMETER_CACHE /var/tmp/filecoin-proof-parameters +ENV LOTUS_MINER_PATH /var/lib/lotus-miner +ENV LOTUS_PATH /var/lib/lotus +ENV LOTUS_WORKER_PATH /var/lib/lotus-worker +ENV WALLET_PATH /var/lib/lotus-wallet + +COPY --from=lotus-builder /opt/filecoin/lotus /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-seed /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-shed /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-wallet /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-gateway /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-miner /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-worker /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/curio /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-stats /usr/local/bin/ +COPY --from=lotus-builder /opt/filecoin/lotus-fountain /usr/local/bin/ + +# Copy and create separate binaries for calibnet +COPY --from=lotus-builder /opt/filecoin-calibnet/lotus /usr/local/bin/lotus-calibnet +COPY --from=lotus-builder /opt/filecoin-calibnet/lotus-seed /usr/local/bin/lotus-calibnet-seed + +RUN mkdir /var/tmp/filecoin-proof-parameters +RUN mkdir /var/lib/lotus +RUN mkdir /var/lib/lotus-miner +RUN mkdir /var/lib/lotus-worker +RUN mkdir /var/lib/lotus-wallet +RUN mkdir /var/lib/curio +RUN chown fc: /var/tmp/filecoin-proof-parameters +RUN chown fc: /var/lib/lotus +RUN chown fc: /var/lib/lotus-miner +RUN chown fc: /var/lib/lotus-worker +RUN chown fc: /var/lib/lotus-wallet +RUN chown fc: /var/lib/curio + + +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/lotus +VOLUME /var/lib/lotus-miner +VOLUME /var/lib/lotus-worker +VOLUME /var/lib/lotus-wallet +VOLUME /var/lib/curio + +EXPOSE 1234 +EXPOSE 2345 +EXPOSE 3456 +EXPOSE 1777 \ No newline at end of file diff --git a/stack-orchestrator/container-build/cerc-lotus-node/build.sh b/stack-orchestrator/container-build/cerc-lotus-node/build.sh new file mode 100755 index 0000000..7a4d45f --- /dev/null +++ b/stack-orchestrator/container-build/cerc-lotus-node/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/lotus-node +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Replace repo's Dockerfile with modified one +cp ${SCRIPT_DIR}/Dockerfile ${CERC_REPO_BASE_DIR}/lotus/Dockerfile + +docker build -t cerc/lotus-node:local ${build_command_args} ${CERC_REPO_BASE_DIR}/lotus diff --git a/stack-orchestrator/stacks/lotus-node/README.md b/stack-orchestrator/stacks/lotus-node/README.md new file mode 100644 index 0000000..2f31c5d --- /dev/null +++ b/stack-orchestrator/stacks/lotus-node/README.md @@ -0,0 +1,126 @@ +# lotus-node + +Instructions for deploying a Lotus snap-synced node (for both mainnet and calibnet) + +## Prerequisite + +Downloaded chain snapshot + +- For Mainnet + ```bash + aria2c -o mainnet-snapshot.car.zst -x5 https://forest-archive.chainsafe.dev/latest/mainnet/ + + # or using wget + wget -O mainnet-snapshot.car.zst https://forest-archive.chainsafe.dev/latest/mainnet/ + ``` + +- For Calibration Testnet + ```bash + aria2c -o calibnet-snapshot.car.zst -x5 https://forest-archive.chainsafe.dev/latest/calibnet/ + + # or using wget + wget -O calibnet-snapshot.car.zst https://forest-archive.chainsafe.dev/latest/calibnet/ + ``` + +## Clone the stack repo + +```bash +$ laconic-so fetch-stack git.vdb.to/cerc-io/lotus-stack +``` + +## Clone required repositories + +```bash +$ laconic-so --stack ~/cerc/lotus-stack/stack-orchestrator/stacks/lotus-node setup-repositories --branches git.vdb.to/cerc-io/lotus@v1.27.0-rc3-interal-0.0.1 +``` + +## Build the Lotus containers + +```bash +$ laconic-so --stack ~/cerc/lotus-stack/stack-orchestrator/stacks/lotus-node build-containers +``` + +## Create a deployment + +- Create deployment spec file + + - For Mainnet + ```bash + $ laconic-so --stack ~/cerc/lotus-stack/stack-orchestrator/stacks/lotus-node deploy init --map-ports-to-host any-same --output lotus-node.yml --config CERC_LOTUS_NETWORK=mainnet,CERC_LOTUS_SNAPSHOT_PATH=/path-to-file/mainnet-snapshot.car.zst + ``` + + - For Calibration + ```bash + $ laconic-so --stack ~/cerc/lotus-stack/stack-orchestrator/stacks/lotus-node deploy init --map-ports-to-host any-same --output lotus-node.yml --config CERC_LOTUS_NETWORK=calibration,CERC_LOTUS_SNAPSHOT_PATH=/path-to-file/calibnet-snapshot.car.zst + ``` + + NOTE: In commands above, set `CERC_LOTUS_SNAPSHOT_PATH` to the absolute file path of the downloaded snapshot ([Prerequisite](#prerequisite)) + +- Create deployment directory + ```bash + $ laconic-so --stack ~/cerc/lotus-stack/stack-orchestrator/stacks/lotus-node deploy create --spec-file lotus-node.yml --deployment-dir lotus-node-deployment + ``` + +## Setup config + +Environment variables for Lotus node can be configured by setting them in `config.env` inside deployment directory + +- Check [compose file](../../compose/docker-compose-lotus-node.yml) for configured environment variables + +- Refer to [default Lotus config](https://git.vdb.to/cerc-io/lotus/src/tag/v1.27.0-rc3-interal-0.0.1/documentation/en/default-lotus-config.toml) + +## Start the container + +```bash +$ laconic-so deployment --dir lotus-node-deployment up +``` + +## Check status + +- Check if container is healthy + ``` + docker ps | grep lotus + ``` + NOTE: Starting node for the first time takes time (~20 mins) as it imports the snapshot + +- To check whether Lotus node has synced till head + ```bash + laconic-so deployment --dir lotus-node-deployment exec lotus "lotus sync wait" + + # lotus sync wait + # Worker: 78534; Base: 2403099; Target: 2403099 (diff: 0) + # State: message sync; Current Epoch: 2403099; Todo: 0 + + # Done! + ``` + When the lotus sync wait command shows done you are fully synced with the chain + - Get latest block using CURL + ``` + curl -X POST \ + -H "Content-Type: application/json" \ + --data '{ + "jsonrpc": "2.0", + "method": "eth_getBlockByNumber", + "params": ["latest", false], + "id": 1 + }' \ + http://localhost:1234/rpc/v1 + ``` + +## Clean up + +Stop all services running in the background: + +```bash +$ laconic-so deployment --dir lotus-node-deployment down +``` + +To stop all services and also delete data: + +```bash +# Stop the docker containers +$ laconic-so deployment --dir lotus-node-deployment down --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +$ rm -r lotus-node-deployment +``` diff --git a/stack-orchestrator/stacks/lotus-node/stack.yml b/stack-orchestrator/stacks/lotus-node/stack.yml new file mode 100644 index 0000000..b989d90 --- /dev/null +++ b/stack-orchestrator/stacks/lotus-node/stack.yml @@ -0,0 +1,9 @@ +version: "1.0" +name: lotus-node +description: "Lotus node stack" +repos: + - git.vdb.to/cerc-io/lotus +containers: + - cerc/lotus-node +pods: + - lotus-node