From 20d633f81c75e3cbd17ae357e1ea9c0274d02c15 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 25 Oct 2023 14:42:52 -0500 Subject: [PATCH] Plugeth-based full mainnet stack. (#592) * Plugeth-based full mainnet stack. --------- Co-authored-by: David Boreham --- ...docker-compose-mainnet-eth-ipld-eth-db.yml | 29 ++++ ...er-compose-mainnet-eth-ipld-eth-server.yml | 24 +++ .../docker-compose-mainnet-eth-keycloak.yml | 4 +- .../docker-compose-mainnet-eth-plugeth.yml | 72 +++++++++ .../config/mainnet-eth-ipld-eth-db/db.env | 15 ++ .../mainnet-eth-ipld-eth-server/config.toml | 33 ++++ .../mainnet-eth-ipld-eth-server/srv.env | 27 ++++ .../config/mainnet-eth-keycloak/keycloak.env | 3 + .../config/mainnet-eth-keycloak/nginx.example | 42 ++++-- app/data/config/mainnet-eth-plugeth/geth.env | 75 ++++++++++ .../config/mainnet-eth-plugeth/lighthouse.env | 33 ++++ .../mainnet-eth-plugeth/scripts/run-geth.sh | 121 +++++++++++++++ .../scripts/run-lighthouse.sh | 30 ++++ app/data/config/mainnet-eth/geth.env | 2 +- .../cerc-plugeth-with-plugins/Dockerfile | 22 +++ .../cerc-plugeth-with-plugins/build.sh | 9 ++ app/data/stacks/mainnet-eth-plugeth/README.md | 141 ++++++++++++++++++ .../mainnet-eth-plugeth/deploy/commands.py | 32 ++++ app/data/stacks/mainnet-eth-plugeth/stack.yml | 29 ++++ 19 files changed, 724 insertions(+), 19 deletions(-) create mode 100644 app/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml create mode 100644 app/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml create mode 100644 app/data/compose/docker-compose-mainnet-eth-plugeth.yml create mode 100644 app/data/config/mainnet-eth-ipld-eth-db/db.env create mode 100644 app/data/config/mainnet-eth-ipld-eth-server/config.toml create mode 100644 app/data/config/mainnet-eth-ipld-eth-server/srv.env create mode 100644 app/data/config/mainnet-eth-plugeth/geth.env create mode 100644 app/data/config/mainnet-eth-plugeth/lighthouse.env create mode 100755 app/data/config/mainnet-eth-plugeth/scripts/run-geth.sh create mode 100755 app/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh create mode 100644 app/data/container-build/cerc-plugeth-with-plugins/Dockerfile create mode 100755 app/data/container-build/cerc-plugeth-with-plugins/build.sh create mode 100644 app/data/stacks/mainnet-eth-plugeth/README.md create mode 100644 app/data/stacks/mainnet-eth-plugeth/deploy/commands.py create mode 100644 app/data/stacks/mainnet-eth-plugeth/stack.yml diff --git a/app/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml b/app/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml new file mode 100644 index 00000000..49cc2de3 --- /dev/null +++ b/app/data/compose/docker-compose-mainnet-eth-ipld-eth-db.yml @@ -0,0 +1,29 @@ +version: "3.2" + +services: + migrations: + restart: on-failure + depends_on: + ipld-eth-db: + condition: service_healthy + image: cerc/ipld-eth-db:local + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + + ipld-eth-db: + image: timescale/timescaledb:2.8.1-pg14 + restart: always + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + volumes: + - mainnet_eth_ipld_eth_db:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + ports: + - "5432" +volumes: + mainnet_eth_ipld_eth_db: diff --git a/app/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml b/app/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml new file mode 100644 index 00000000..4341c6a1 --- /dev/null +++ b/app/data/compose/docker-compose-mainnet-eth-ipld-eth-server.yml @@ -0,0 +1,24 @@ +version: "3.7" +services: + ipld-eth-server: + restart: always + depends_on: + ipld-eth-db: + condition: service_healthy + image: cerc/ipld-eth-server:local + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + - ../config/mainnet-eth-ipld-eth-server/srv.env + volumes: + - ../config/mainnet-eth-ipld-eth-server/config.toml:/app/config.toml:ro + ports: + - "8081" + - "8082" + - "8090" + - "40001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8081"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s diff --git a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml index dfa9a804..1674c62e 100644 --- a/app/data/compose/docker-compose-mainnet-eth-keycloak.yml +++ b/app/data/compose/docker-compose-mainnet-eth-keycloak.yml @@ -6,7 +6,7 @@ services: env_file: - ../config/mainnet-eth-keycloak/keycloak.env healthcheck: - test: ["CMD", "nc", "-v", "localhost", "5432"] + test: ["CMD", "nc", "-v", "localhost", "35432"] interval: 30s timeout: 10s retries: 10 @@ -14,7 +14,7 @@ services: volumes: - mainnet_eth_keycloak_db:/var/lib/postgresql/data ports: - - 5432 + - 35432 keycloak: image: cerc/keycloak:local diff --git a/app/data/compose/docker-compose-mainnet-eth-plugeth.yml b/app/data/compose/docker-compose-mainnet-eth-plugeth.yml new file mode 100644 index 00000000..a8b301d2 --- /dev/null +++ b/app/data/compose/docker-compose-mainnet-eth-plugeth.yml @@ -0,0 +1,72 @@ + +services: + + mainnet-eth-geth-1: + restart: always + hostname: mainnet-eth-geth-1 + cap_add: + - SYS_PTRACE + image: cerc/plugeth-with-plugins:local + entrypoint: /bin/sh + command: -c "/opt/run-geth.sh" + env_file: + - ../config/mainnet-eth-ipld-eth-db/db.env + - ../config/mainnet-eth-plugeth/geth.env + volumes: + - mainnet_eth_plugeth_geth_1_data:/data + - mainnet_eth_plugeth_config_data:/etc/mainnet-eth + - ../config/mainnet-eth-plugeth/scripts/run-geth.sh:/opt/run-geth.sh + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8545"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + ports: + # http api + - "8545" + # ws api + - "8546" + # ws el + - "8551" + # p2p + - "30303" + - "30303/udp" + # debugging + - "40000" + # metrics + - "6060" + + mainnet-eth-lighthouse-1: + restart: always + hostname: mainnet-eth-lighthouse-1 + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 30s + environment: + LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551" + env_file: + - ../config/mainnet-eth-plugeth/lighthouse.env + image: cerc/lighthouse:local + entrypoint: /bin/sh + command: -c "/opt/run-lighthouse.sh" + volumes: + - mainnet_eth_plugeth_lighthouse_1_data:/data + - mainnet_eth_plugeth_config_data:/etc/mainnet-eth + - ../config/mainnet-eth-plugeth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh + ports: + # api + - "5052" + # metrics + - "5054" + # p2p + - "9000" + - "9000/udp" + +volumes: + mainnet_eth_plugeth_config_data: + mainnet_eth_plugeth_geth_1_data: + mainnet_eth_plugeth_lighthouse_1_data: diff --git a/app/data/config/mainnet-eth-ipld-eth-db/db.env b/app/data/config/mainnet-eth-ipld-eth-db/db.env new file mode 100644 index 00000000..4ec11109 --- /dev/null +++ b/app/data/config/mainnet-eth-ipld-eth-db/db.env @@ -0,0 +1,15 @@ +DATABASE_HOSTNAME="ipld-eth-db" +DATABASE_NAME="cerc" +DATABASE_PASSWORD="CHANGEME" +DATABASE_PORT=5432 +DATABASE_USER="vdbm" + +POSTGRES_DB="${DATABASE_NAME}" +POSTGRES_PASSWORD="${DATABASE_PASSWORD}" +POSTGRES_USER="${DATABASE_USER}" + +CERC_STATEDIFF_DB_HOST="${DATABASE_HOSTNAME}" +CERC_STATEDIFF_DB_NAME="${DATABASE_NAME}" +CERC_STATEDIFF_DB_PASSWORD="${DATABASE_PASSWORD}" +CERC_STATEDIFF_DB_PORT=${DATABASE_PORT} +CERC_STATEDIFF_DB_USER="${DATABASE_USER}" diff --git a/app/data/config/mainnet-eth-ipld-eth-server/config.toml b/app/data/config/mainnet-eth-ipld-eth-server/config.toml new file mode 100644 index 00000000..c433df28 --- /dev/null +++ b/app/data/config/mainnet-eth-ipld-eth-server/config.toml @@ -0,0 +1,33 @@ +[database] + name = "" # $DATABASE_NAME + hostname = "" # $DATABASE_HOSTNAME + port = 5432 # $DATABASE_PORT + user = "" # $DATABASE_USER + password = "" # $DATABASE_PASSWORD + +[log] + level = "info" # $LOG_LEVEL + +[server] + ipc = false + ipcPath = "" # $SERVER_IPC_PATH + ws = false + wsPath = "0.0.0.0:8080" # $SERVER_WS_PATH + http = true + httpPath = "0.0.0.0:8081" # $SERVER_HTTP_PATH + graphql = false # $SERVER_GRAPHQL + graphqlPath = "0.0.0.0:8082" # $SERVER_GRAPHQL_PATH + +[ethereum] + chainConfig = "" # ETH_CHAIN_CONFIG + chainID = "1" # $ETH_CHAIN_ID + rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP + httpPath = "mainnet-eth-geth-1:8545" # $ETH_HTTP_PATH + supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF + stateDiffTimeout = "4m" # $ETH_STATEDIFF_TIMEOUT + forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS + proxyOnError = true # $ETH_PROXY_ON_ERROR + nodeID = "" # $ETH_NODE_ID + clientName = "" # $ETH_CLIENT_NAME + genesisBlock = "" # $ETH_GENESIS_BLOCK + networkID = "1" # $ETH_NETWORK_ID diff --git a/app/data/config/mainnet-eth-ipld-eth-server/srv.env b/app/data/config/mainnet-eth-ipld-eth-server/srv.env new file mode 100644 index 00000000..34c79ce4 --- /dev/null +++ b/app/data/config/mainnet-eth-ipld-eth-server/srv.env @@ -0,0 +1,27 @@ +CERC_REMOTE_DEBUG="false" + +LOG_LEVEL="debug" + +ETH_CHAIN_ID=1 +ETH_CLIENT_NAME="Geth" +ETH_FORWARD_ETH_CALLS="false" +ETH_FORWARD_GET_STORAGE_AT="false" +ETH_GENESIS_BLOCK="0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" +ETH_HTTP_PATH="mainnet-eth-geth-1:8545" +ETH_NETWORK_ID=1 +ETH_NODE_ID=1112 +ETH_PROXY_ON_ERROR="true" +ETH_RPC_GAS_CAP=1000000000000 +ETH_SUPPORTS_STATEDIFF="true" +ETH_STATEDIFF_TIMEOUT=4m + +SERVER_HTTP_PATH=0.0.0.0:8081 +SERVER_GRAPHQL="false" +SERVER_GRAPHQLPATH=0.0.0.0:8082 + +METRICS="true" +PROM_HTTP="true" +PROM_HTTP_ADDR="0.0.0.0" +PROM_HTTP_PORT="8090" + +VDB_COMMAND="serve" diff --git a/app/data/config/mainnet-eth-keycloak/keycloak.env b/app/data/config/mainnet-eth-keycloak/keycloak.env index f37fdd30..31a19079 100644 --- a/app/data/config/mainnet-eth-keycloak/keycloak.env +++ b/app/data/config/mainnet-eth-keycloak/keycloak.env @@ -1,8 +1,11 @@ POSTGRES_DB=keycloak POSTGRES_USER=keycloak POSTGRES_PASSWORD=keycloak +# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml +PGPORT=35432 KC_DB=postgres KC_DB_URL_HOST=keycloak-db +KC_DB_URL_PORT=${PGPORT} KC_DB_URL_DATABASE=${POSTGRES_DB} KC_DB_USERNAME=${POSTGRES_USER} KC_DB_PASSWORD=${POSTGRES_PASSWORD} diff --git a/app/data/config/mainnet-eth-keycloak/nginx.example b/app/data/config/mainnet-eth-keycloak/nginx.example index 67095551..758f0ce1 100644 --- a/app/data/config/mainnet-eth-keycloak/nginx.example +++ b/app/data/config/mainnet-eth-keycloak/nginx.example @@ -15,42 +15,49 @@ server { } upstream geth-pool { - keepalive 100; - hash $user_id consistent; - server server-a:8545; - server server-b:8545; - server server-c:8545; + server server-a:8545 max_fails=10 fail_timeout=2s; + server server-c:8545 max_fails=10 fail_timeout=2s backup; + server server-b:8545 max_fails=10 fail_timeout=2s backup; + keepalive 200; } -# self-reg happens on one server for clarity upstream reg-ui-pool { - keepalive 100; + keepalive 2; server server-a:8085; } upstream reg-api-pool { - keepalive 100; + keepalive 2; server server-a:8086; } -# auth uses server-a if available +# auth uses the reg server when available upstream auth-pool { - keepalive 100; + keepalive 10; server server-a:8080; server server-b:8080 backup; server server-c:8080 backup; } -log_format upstreamlog '[$time_local] $remote_addr $user_id - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time'; -proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m; + +log_format upstreamlog '[$time_local] $msec $remote_addr $user_id - $server_name($host) to $upstream_addr: $request $status upstream_response_time $upstream_response_time request_time $request_time'; +proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m; + server { listen 443 ssl http2; server_name my.example.com; + keepalive_requests 500000; + keepalive_timeout 90s; + http2_max_requests 5000000; + http2_max_concurrent_streams 1024; + http2_idle_timeout 3m; + http2_recv_timeout 30s; access_log /var/log/nginx/my.example.com-access.log upstreamlog; error_log /var/log/nginx/my.example.com-error.log; ssl_certificate /etc/nginx/ssl/my.example.com/cert.pem; ssl_certificate_key /etc/nginx/ssl/my.example.com/key.pem; + ssl_session_cache shared:SSL:10m; error_page 500 502 503 504 /50x.html; location = /50x.html { @@ -60,7 +67,6 @@ server { #rewrite ^/?$ /newuser/; rewrite ^/?$ https://www.example.com/; - # geth-pool ETH API location ~ ^/v1/eth/?([^/]*)$ { set $apiKey $1; @@ -71,8 +77,8 @@ server { auth_request_set $user_id $sent_http_x_user_id; rewrite /.*$ / break; - client_max_body_size 3m; - client_body_buffer_size 3m; + client_max_body_size 3m; + client_body_buffer_size 3m; proxy_buffer_size 32k; proxy_buffers 16 32k; proxy_busy_buffers_size 96k; @@ -80,8 +86,10 @@ server { proxy_pass http://geth-pool; proxy_set_header X-Original-Remote-Addr $remote_addr; proxy_set_header X-User-Id $user_id; + proxy_http_version 1.1; + proxy_set_header Connection ""; } - + # keycloak location = /auth { internal; @@ -95,6 +103,8 @@ server { proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Original-Remote-Addr $remote_addr; proxy_set_header X-Original-Host $host; + proxy_http_version 1.1; + proxy_set_header Connection ""; } location /newuser/ { diff --git a/app/data/config/mainnet-eth-plugeth/geth.env b/app/data/config/mainnet-eth-plugeth/geth.env new file mode 100644 index 00000000..5c936d36 --- /dev/null +++ b/app/data/config/mainnet-eth-plugeth/geth.env @@ -0,0 +1,75 @@ +# Enable remote debugging using dlv +CERC_REMOTE_DEBUG=false + +# Enable startup script debug output. +CERC_SCRIPT_DEBUG=false + +# Simple toggle to choose either a 'full' node or an 'archive' node +# (controls the values of --syncmode --gcmode --snapshot) +CERC_GETH_MODE_QUICK_SET=archive + +# Path to plugeth plugins. +CERC_PLUGINS_DIR="/usr/local/lib/plugeth" + +# Will turn on statediffing automatically if CERC_STATEDIFF_DB_HOST exists (see ../mainnet-eth-ipld-eth-db/db.env). +CERC_RUN_STATEDIFF="detect" + +# The minimum necessary verion of the DB to enable statediffing. +CERC_STATEDIFF_DB_GOOSE_MIN_VER=18 + +# Whether all statediff-related DB statements should be logged (useful for debugging). +CERC_STATEDIFF_DB_LOG_STATEMENTS=false + +# The number of concurrent workers to process state diff objects +CERC_STATEDIFF_WORKERS=16 + +# Each statediffing node should have a unique node ID. +CERC_STATEDIFF_DB_NODE_ID=1111 + +# Optional custom node name. +# GETH_NODE_NAME="" + +# Specify any other geth CLI options. +GETH_OPTS="" + +# --cache +GETH_CACHE=1024 + +# --cache.database +GETH_CACHE_DB=50 + +# --cache.gc +GETH_CACHE_GC=25 + +# --cache.trie +GETH_CACHE_TRIE=15 + +# --datadir +GETH_DATADIR="/data" + +# --http.api +GETH_HTTP_API="eth,web3,net" + +# --authrpc.jwtsecret +GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret" + +# --maxpeers +GETH_MAX_PEERS=100 + +# --rpc.evmtimeout +GETH_RPC_EVMTIMEOUT=0 + +# --rpc.gascap +GETH_RPC_GASCAP=0 + +# --txlookuplimit +GETH_TXLOOKUPLIMIT=0 + +# --verbosity +GETH_VERBOSITY=3 + +# --log.vmodule +GETH_VMODULE="rpc/*=4" + +# --ws.api +GETH_WS_API="eth,web3,net" diff --git a/app/data/config/mainnet-eth-plugeth/lighthouse.env b/app/data/config/mainnet-eth-plugeth/lighthouse.env new file mode 100644 index 00000000..11fc6b69 --- /dev/null +++ b/app/data/config/mainnet-eth-plugeth/lighthouse.env @@ -0,0 +1,33 @@ +# Enable startup script debug output. +CERC_SCRIPT_DEBUG=false + +# Specify any other lighthouse CLI options. +LIGHTHOUSE_OPTS="" + +# Override the advertised public IP (optional) +# --enr-address +#LIGHTHOUSE_ENR_ADDRESS="" + +# --checkpoint-sync-url +LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc" + +# --checkpoint-sync-url-timeout +LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300 + +# --datadir +LIGHTHOUSE_DATADIR=/data + +# --debug-level +LIGHTHOUSE_DEBUG_LEVEL=info + +# --http-port +LIGHTHOUSE_HTTP_PORT=5052 + +# --execution-jwt +LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret + +# --metrics-port +LIGHTHOUSE_METRICS_PORT=5054 + +# --port --enr-udp-port --enr-tcp-port +LIGHTHOUSE_NETWORK_PORT=9000 diff --git a/app/data/config/mainnet-eth-plugeth/scripts/run-geth.sh b/app/data/config/mainnet-eth-plugeth/scripts/run-geth.sh new file mode 100755 index 00000000..1971c2d0 --- /dev/null +++ b/app/data/config/mainnet-eth-plugeth/scripts/run-geth.sh @@ -0,0 +1,121 @@ +#!/bin/sh +if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then + set -x +fi + +START_CMD="geth" +if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then + START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" +fi + +# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script +cleanup() { + echo "Signal received, cleaning up..." + + # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process) + pkill -P ${geth_pid} + sleep 2 + kill $(jobs -p) + + wait + echo "Done" +} +trap 'cleanup' SIGINT SIGTERM + +MODE_FLAGS="" +if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}" +else + MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-full} --snapshot=${GETH_SNAPSHOT:-true}" +fi + +if [[ "${CERC_RUN_STATEDIFF}" == "detect" ]] && [[ -n "$CERC_STATEDIFF_DB_HOST" ]]; then + dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short) + dig_status_code=$? + if [[ $dig_status_code = 0 && -n $dig_result ]]; then + echo "Statediff DB at $CERC_STATEDIFF_DB_HOST" + CERC_RUN_STATEDIFF="true" + else + echo "No statediff DB available." + CERC_RUN_STATEDIFF="false" + fi +fi + +STATEDIFF_OPTS="" +if [[ "${CERC_RUN_STATEDIFF}" == "true" ]]; then + ready=0 + echo "Waiting for statediff DB..." + while [ $ready -eq 0 ]; do + sleep 1 + export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD" + result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \ + -p "$CERC_STATEDIFF_DB_PORT" \ + -U "$CERC_STATEDIFF_DB_USER" \ + -d "$CERC_STATEDIFF_DB_NAME" \ + -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }') + if [ -n "$result" ]; then + echo "DB ready..." + if [[ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]]; then + ready=1 + else + echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)" + fi + fi + done + + STATEDIFF_OPTS="--statediff \ + --statediff.db.host=$CERC_STATEDIFF_DB_HOST \ + --statediff.db.name=$CERC_STATEDIFF_DB_NAME \ + --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \ + --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \ + --statediff.db.port=$CERC_STATEDIFF_DB_PORT \ + --statediff.db.user=$CERC_STATEDIFF_DB_USER \ + --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \ + --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \ + --statediff.waitforsync=${CERC_STATEDIFF_WAIT_FO_SYNC:-true} \ + --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ + --statediff.writing=${CERC_STATEDIFF_WRITING:-true}" + + if [[ -d "${CERC_PLUGINS_DIR}" ]]; then + # With plugeth, we separate the statediff options by prefixing with ' -- ' + STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}" + fi +fi + +$START_CMD \ + $MODE_FLAGS \ + --datadir="${GETH_DATADIR}"\ + --identity="${GETH_NODE_NAME}" \ + --maxpeers=${GETH_MAX_PEERS} \ + --cache=${GETH_CACHE} \ + --cache.gc=${GETH_CACHE_GC} \ + --cache.database=${GETH_CACHE_DB} \ + --cache.trie=${GETH_CACHE_TRIE} \ + --authrpc.addr='0.0.0.0' \ + --authrpc.vhosts='*' \ + --authrpc.jwtsecret="${GETH_JWTSECRET}" \ + --http \ + --http.addr='0.0.0.0' \ + --http.api="${GETH_HTTP_API}" \ + --http.vhosts='*' \ + --metrics \ + --metrics.addr='0.0.0.0' \ + --ws \ + --ws.addr='0.0.0.0' \ + --ws.api="${GETH_WS_API}" \ + --rpc.gascap=${GETH_RPC_GASCAP} \ + --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \ + --txlookuplimit=${GETH_TXLOOKUPLIMIT} \ + --verbosity=${GETH_VERBOSITY} \ + --log.vmodule="${GETH_VMODULE}" \ + ${STATEDIFF_OPTS} \ + ${GETH_OPTS} & + +geth_pid=$! +wait $geth_pid + +if [[ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]]; then + while [[ 1 -eq 1 ]]; do + sleep 60 + done +fi diff --git a/app/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh new file mode 100755 index 00000000..efda735b --- /dev/null +++ b/app/data/config/mainnet-eth-plugeth/scripts/run-lighthouse.sh @@ -0,0 +1,30 @@ +#!/bin/bash +if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then + set -x +fi + +ENR_OPTS="" +if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then + ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS" +fi + +exec lighthouse bn \ + --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \ + --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \ + --datadir "$LIGHTHOUSE_DATADIR" \ + --debug-level $LIGHTHOUSE_DEBUG_LEVEL \ + --disable-deposit-contract-sync \ + --disable-upnp \ + --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \ + --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \ + --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \ + --execution-jwt /etc/mainnet-eth/jwtsecret \ + --http \ + --http-address 0.0.0.0 \ + --http-port $LIGHTHOUSE_HTTP_PORT \ + --metrics \ + --metrics-address=0.0.0.0 \ + --metrics-port $LIGHTHOUSE_METRICS_PORT \ + --network mainnet \ + --port $LIGHTHOUSE_NETWORK_PORT \ + $ENR_OPTS $LIGHTHOUSE_OPTS diff --git a/app/data/config/mainnet-eth/geth.env b/app/data/config/mainnet-eth/geth.env index a01444df..365bb5fb 100644 --- a/app/data/config/mainnet-eth/geth.env +++ b/app/data/config/mainnet-eth/geth.env @@ -25,7 +25,7 @@ GETH_CACHE_GC=25 # --cache.trie GETH_CACHE_TRIE=15 -j + # --datadir GETH_DATADIR="/data" diff --git a/app/data/container-build/cerc-plugeth-with-plugins/Dockerfile b/app/data/container-build/cerc-plugeth-with-plugins/Dockerfile new file mode 100644 index 00000000..87d050ea --- /dev/null +++ b/app/data/container-build/cerc-plugeth-with-plugins/Dockerfile @@ -0,0 +1,22 @@ +# Using the same golang image as used to build plugeth: https://git.vdb.to/cerc-io/plugeth/src/branch/statediff/Dockerfile +FROM golang:1.20-alpine3.18 as delve + +# Add delve so that we can do remote debugging. +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM cerc/plugeth-statediff:local as statediff +FROM cerc/plugeth:local as plugeth + +FROM alpine:3.18 + +# Install tools often used in scripting, like bash, wget, and jq. +RUN apk add --no-cache ca-certificates bash wget curl python3 bind-tools postgresql-client jq + +COPY --from=delve /go/bin/dlv /usr/local/bin/ +COPY --from=plugeth /usr/local/bin/geth /usr/local/bin/ + +# Place all plugeth plugins in /usr/local/lib/plugeth +COPY --from=statediff /usr/local/lib/statediff.so /usr/local/lib/plugeth/ + +EXPOSE 8545 8546 8551 6060 30303 30303/udp 40000 +ENTRYPOINT ["geth"] diff --git a/app/data/container-build/cerc-plugeth-with-plugins/build.sh b/app/data/container-build/cerc-plugeth-with-plugins/build.sh new file mode 100755 index 00000000..9ab44946 --- /dev/null +++ b/app/data/container-build/cerc-plugeth-with-plugins/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/cerc-plugeth-with-plugins +set -x + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/plugeth-with-plugins:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR diff --git a/app/data/stacks/mainnet-eth-plugeth/README.md b/app/data/stacks/mainnet-eth-plugeth/README.md new file mode 100644 index 00000000..8ed6bebb --- /dev/null +++ b/app/data/stacks/mainnet-eth-plugeth/README.md @@ -0,0 +1,141 @@ +# mainnet-eth-plugeth + +Deploys a "head-tracking" mainnet Ethereum stack comprising a [plugeth](https://git.vdb.to/cerc-io/plugeth) execution layer node and a [lighthouse](https://github.com/sigp/lighthouse) consensus layer node, with [plugeth-statediff](https://git.vdb.to/cerc-io/plugeth-statediff) for statediffing, [ipld-eth-db](https://git.vdb.to/cerc-io/ipld-eth-db) for storage, and [ipld-eth-server](https://git.vdb.to/cerc-io/ipld-eth-server) for indexed ETH IPLD objects. + +## Clone required repositories + +``` +$ laconic-so --stack mainnet-eth-plugeth setup-repositories +``` + +## Build containers + +``` +$ laconic-so --stack mainnet-eth-plugeth build-containers +``` + +## Create a deployment + +``` +$ laconic-so --stack mainnet-eth-plugeth deploy init --map-ports-to-host any-same --output mainnet-eth-plugeth-spec.yml +$ laconic-so --stack mainnet-eth-plugeth deploy create --spec-file mainnet-eth-plugeth-spec.yml --deployment-dir mainnet-eth-plugeth-deployment +``` +## Start the stack +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment start +``` +Display stack status: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment ps +Running containers: +id: f39608eca04d72d6b0f1f3acefc5ebb52908da06e221d20c7138f7e3dff5e423, name: laconic-ef641b4d13eb61ed561b19be67063241-foundry-1, ports: +id: 4052b1eddd886ae0d6b41f9ff22e68a70f267b2bfde10f4b7b79b5bd1eeddcac, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-geth-1-1, ports: 30303/tcp, 30303/udp, 0.0.0.0:49184->40000/tcp, 0.0.0.0:49185->6060/tcp, 0.0.0.0:49186->8545/tcp, 8546/tcp +id: ac331232e597944b621b3b8942ace5dafb14524302cab338ff946c7f6e5a1d52, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1, ports: 0.0.0.0:49187->8001/tcp +``` +See stack logs: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment logs +time="2023-07-25T09:46:29-06:00" level=warning msg="The \"CERC_SCRIPT_DEBUG\" variable is not set. Defaulting to a blank string." +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.362 INFO Logging to file path: "/var/lighthouse-data-dir/beacon/logs/beacon.log" +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Lighthouse started version: Lighthouse/v4.1.0-693886b +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Configured for network name: mainnet +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Data directory initialised datadir: /var/lighthouse-data-dir +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Deposit contract address: 0x00000000219ab540356cbb839cbe05303d7705fa, deploy_block: 11184524 +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.424 INFO Starting checkpoint sync remote_url: https://beaconstate.ethstaker.cc/, service: beacon +``` +## Monitoring stack sync progress +Both go-ethereum and lighthouse will engage in an initial chain sync phase that will last up to several hours depending on hardware performance and network capacity. +Syncing can be monitored by looking for these log messages: +``` +Jul 24 12:34:17.001 INFO Downloading historical blocks est_time: 5 days 11 hrs, speed: 14.67 slots/sec, distance: 6932481 slots (137 weeks 3 days), service: slot_notifier +INFO [07-24|12:14:52.493] Syncing beacon headers downloaded=145,920 left=17,617,968 eta=1h23m32.815s +INFO [07-24|12:33:15.238] Syncing: chain download in progress synced=1.86% chain=148.94MiB headers=368,640@95.03MiB bodies=330,081@40.56MiB receipts=330,081@13.35MiB eta=37m54.505s +INFO [07-24|12:35:13.028] Syncing: state download in progress synced=1.32% state=4.64GiB accounts=2,850,314@677.57MiB slots=18,663,070@3.87GiB codes=26662@111.14MiB eta=3h18m0.699s +``` +Once synced up these log messages will be observed: +``` +INFO Synced slot: 6952515, block: 0x5bcb…f6d9, epoch: 217266, finalized_epoch: 217264, finalized_root: 0x6342…2c5c, exec_hash: 0x8d8c…2443 (verified), peers: 31, service: slot_notifier +INFO [07-25|03:04:48.941] Imported new potential chain segment number=17,767,316 hash=84f6e7..bc2cb0 blocks=1 txs=137 mgas=16.123 elapsed=57.087ms mgasps=282.434 dirty=461.46MiB +INFO [07-25|03:04:49.042] Chain head was updated number=17,767,316 hash=84f6e7..bc2cb0 root=ca58b2..8258c1 elapsed=2.480111ms +``` +## Clean up + +Stop the stack: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop +``` +This leaves data volumes in place, allowing the stack to be subsequently re-started. +To permanently *delete* the stack's data volumes run: +``` +$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop --delete-data-volumes +``` +After deleting the volumes, any subsequent re-start will begin chain sync from cold. + +## Ports +It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. +Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be +customized by editing the "spec" file generated by `laconic-so deploy init`. + +In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while +port `40000` has been left with random assignment: +``` +$ cat mainnet-eth-plugeth-spec.yml +stack: mainnet-eth-plugeth +ports: + mainnet-eth-plugeth-geth-1: + - '10.10.10.10:8545:8545' + - '40000' + mainnet-eth-plugeth-lighthouse-1: + - '10.10.10.10:5052:5052' +volumes: + mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data + mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data + mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data +``` +In addition, a stack-wide port mapping "recipe" can be applied at the time the +`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: +| Recipe | Host Port Mapping | +|--------|-------------------| +| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) | +| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers | +| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers | +| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| +| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | +## Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory: +``` +$ cat mainnet-eth-plugeth-spec.yml +stack: mainnet-eth-plugeth +ports: + mainnet-eth-plugeth-geth-1: + - '10.10.10.10:8545:8545' + - '40000' + mainnet-eth-plugeth-lighthouse-1: + - '10.10.10.10:5052:5052' +volumes: + mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data + mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data + mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data +``` +A synced-up stack will consume around 900GB of data volume space: +``` +$ sudo du -h mainnet-eth-plugeth-deployment/data/ +150M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/freezer_db +25G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/chain_db +16K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/network +368M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/logs +26G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon +26G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data +8.0K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_config_data +4.0K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/keystore +527G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient/chain +527G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient +859G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata +4.8M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/nodes +242M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/ethash +669M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/triecache +860G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth +860G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data +885G mainnet-eth-plugeth-deployment/data/ +``` diff --git a/app/data/stacks/mainnet-eth-plugeth/deploy/commands.py b/app/data/stacks/mainnet-eth-plugeth/deploy/commands.py new file mode 100644 index 00000000..5aba9547 --- /dev/null +++ b/app/data/stacks/mainnet-eth-plugeth/deploy/commands.py @@ -0,0 +1,32 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from secrets import token_hex + + +def init(ctx): + return None + + +def setup(ctx): + return None + + +def create(ctx, extra_args): + # Generate the JWT secret and save to its config file + secret = token_hex(32) + jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_plugeth_config_data", "jwtsecret") + with open(jwt_file_path, 'w+') as jwt_file: + jwt_file.write(secret) diff --git a/app/data/stacks/mainnet-eth-plugeth/stack.yml b/app/data/stacks/mainnet-eth-plugeth/stack.yml new file mode 100644 index 00000000..7ade244c --- /dev/null +++ b/app/data/stacks/mainnet-eth-plugeth/stack.yml @@ -0,0 +1,29 @@ +version: "1.2" +name: mainnet-eth +description: "Ethereum Mainnet" +repos: + - git.vdb.to/cerc-io/plugeth@statediff + - git.vdb.to/cerc-io/plugeth-statediff + - git.vdb.to/cerc-io/lighthouse + - git.vdb.to/cerc-io/ipld-eth-db@v5 + - git.vdb.to/cerc-io/ipld-eth-server@v5 + - git.vdb.to/cerc-io/keycloak-reg-api + - git.vdb.to/cerc-io/keycloak-reg-ui +containers: + - cerc/plugeth-statediff + - cerc/plugeth + - cerc/plugeth-with-plugins + - cerc/lighthouse + - cerc/lighthouse-cli + - cerc/ipld-eth-db + - cerc/ipld-eth-server + - cerc/keycloak + - cerc/webapp-base + - cerc/keycloak-reg-api + - cerc/keycloak-reg-ui +pods: + - mainnet-eth-plugeth + - mainnet-eth-ipld-eth-db + - mainnet-eth-ipld-eth-server + - mainnet-eth-keycloak + - mainnet-eth-metrics