From c3db0e9734b3ba3c2039168423f027684cd8fc48 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 20 Jul 2023 05:55:32 -0600 Subject: [PATCH 01/15] Initial commit --- app/data/stacks/mainnet-eth/README.md | 47 +++++++++++++++++++++++++++ app/data/stacks/mainnet-eth/stack.yml | 17 ++++++++++ 2 files changed, 64 insertions(+) create mode 100644 app/data/stacks/mainnet-eth/README.md create mode 100644 app/data/stacks/mainnet-eth/stack.yml diff --git a/app/data/stacks/mainnet-eth/README.md b/app/data/stacks/mainnet-eth/README.md new file mode 100644 index 00000000..457860b9 --- /dev/null +++ b/app/data/stacks/mainnet-eth/README.md @@ -0,0 +1,47 @@ +# mainnet-eth + +Mainnet Ethereum stack (experimental) + +## Clone required repositories + +``` +$ laconic-so --stack mainnet-eth setup-repositories +``` + +## Build the fixturenet-eth containers + +``` +$ laconic-so --stack mainnet-eth build-containers +``` + +This should create several container images in the local image registry: + +* cerc/go-ethereum +* cerc/lighthouse +* cerc/fixturenet-eth-geth +* cerc/fixturenet-eth-lighthouse + +## Create a deployment + +``` +$ laconic-so --stack mainnet-eth deploy create +``` + + +## Clean up + +Stop all services running in the background: + +```bash +$ laconic-so deployment --dir down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +$ docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data" + +# Remove all the listed volumes +$ docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data") +``` diff --git a/app/data/stacks/mainnet-eth/stack.yml b/app/data/stacks/mainnet-eth/stack.yml new file mode 100644 index 00000000..ae06e2d2 --- /dev/null +++ b/app/data/stacks/mainnet-eth/stack.yml @@ -0,0 +1,17 @@ +version: "1.1" +name: fixturenet-eth +decription: "Ethereum Fixturenet" +repos: + - github.com/cerc-io/go-ethereum + - github.com/cerc-io/lighthouse + - github.com/dboreham/foundry +containers: + - cerc/go-ethereum + - cerc/lighthouse + - cerc/lighthouse-cli + - cerc/fixturenet-eth-geth + - cerc/fixturenet-eth-lighthouse + - cerc/foundry +pods: + - fixturenet-eth + - foundry -- 2.45.2 From a8f169149889b7e10d65f27a5c1131cf82e656c0 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 21 Jul 2023 10:56:47 -0600 Subject: [PATCH 02/15] Fix stop command --- app/deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/deployment.py b/app/deployment.py index eef12222..921f69d7 100644 --- a/app/deployment.py +++ b/app/deployment.py @@ -91,7 +91,7 @@ def down(ctx, delete_volumes, extra_args): def stop(ctx, delete_volumes, extra_args): # TODO: add cluster name and env file here ctx.obj = make_deploy_context(ctx) - down_operation(ctx, delete_volumes, extra_args, None) + down_operation(ctx, delete_volumes, extra_args) @command.command() -- 2.45.2 From 41f82300c5fe3cc21550abc57d4173bbea93e8e9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 21 Jul 2023 11:12:41 -0600 Subject: [PATCH 03/15] Fix data dir create path --- app/deployment_create.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/deployment_create.py b/app/deployment_create.py index fb83f39b..722d8e93 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -67,7 +67,8 @@ def _get_named_volumes(stack): def _create_bind_dir_if_relative(volume, path_string, compose_dir): path = Path(path_string) if not path.is_absolute(): - absolute_path = Path(compose_dir).parent.joinpath(path) + absolute_path = Path(compose_dir).joinpath(path) + print(f"Creating this directory: {absolute_path}") absolute_path.mkdir(parents=True, exist_ok=True) else: if not path.exists(): @@ -84,7 +85,7 @@ def _fixup_pod_file(pod, spec, compose_dir): for volume in pod_volumes.keys(): if volume in spec_volumes: volume_spec = spec_volumes[volume] - volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" + volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f"{volume_spec}" _create_bind_dir_if_relative(volume, volume_spec, compose_dir) new_volume_spec = {"driver": "local", "driver_opts": { -- 2.45.2 From 60d915ed6d3150bfdb675a246e7392de12e461d9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 21 Jul 2023 11:19:46 -0600 Subject: [PATCH 04/15] Remove debugging code --- app/deployment_create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/deployment_create.py b/app/deployment_create.py index 722d8e93..98aad990 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -68,7 +68,6 @@ def _create_bind_dir_if_relative(volume, path_string, compose_dir): path = Path(path_string) if not path.is_absolute(): absolute_path = Path(compose_dir).joinpath(path) - print(f"Creating this directory: {absolute_path}") absolute_path.mkdir(parents=True, exist_ok=True) else: if not path.exists(): -- 2.45.2 From 56c9e5b7178863b9bb2d419373271035b4228f04 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 21 Jul 2023 11:34:51 -0600 Subject: [PATCH 05/15] Update mainnet-eth stack --- .../compose/docker-compose-mainnet-eth.yml | 56 +++++++++++++++++++ app/data/stacks/mainnet-eth/stack.yml | 8 +-- 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 app/data/compose/docker-compose-mainnet-eth.yml diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/app/data/compose/docker-compose-mainnet-eth.yml new file mode 100644 index 00000000..9273edfb --- /dev/null +++ b/app/data/compose/docker-compose-mainnet-eth.yml @@ -0,0 +1,56 @@ + +services: + + mainnet-eth-geth-1: + restart: always + hostname: mainnet-eth-geth-1 + cap_add: + - SYS_PTRACE + environment: + CERC_REMOTE_DEBUG: "true" + CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect} + CERC_STATEDIFF_DB_NODE_ID: 1 + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + image: cerc/go-ethereum:local + volumes: + - mainnet_eth_geth_1_data:/root/ethdata + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "8545"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 3s + ports: + - "8545" + - "40000" + - "6060" + + mainnet-eth-lighthouse-1: + restart: always + hostname: mainnet-eth-lighthouse-1 + healthcheck: + test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8001/eth/v2/beacon/blocks/head"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 30s + env_file: + - ../config/fixturenet-eth/fixturenet-eth.env + environment: + NODE_NUMBER: "1" + ETH1_ENDPOINT: "http://mainnet-eth-geth-1:8545" + EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551" + image: cerc/lighthouse:local + volumes: + - mainnet_eth_lighthouse_1_data:/opt/testnet/build/cl + depends_on: + mainnet-eth-geth-1: + condition: service_healthy + ports: + - "8001" + +volumes: + mainnet_eth_geth_1_data: + mainnet_eth_lighthouse_1_data: diff --git a/app/data/stacks/mainnet-eth/stack.yml b/app/data/stacks/mainnet-eth/stack.yml index ae06e2d2..5051eb9a 100644 --- a/app/data/stacks/mainnet-eth/stack.yml +++ b/app/data/stacks/mainnet-eth/stack.yml @@ -1,6 +1,6 @@ version: "1.1" -name: fixturenet-eth -decription: "Ethereum Fixturenet" +name: mainnet-eth +decription: "Ethereum Mainnet" repos: - github.com/cerc-io/go-ethereum - github.com/cerc-io/lighthouse @@ -9,9 +9,7 @@ containers: - cerc/go-ethereum - cerc/lighthouse - cerc/lighthouse-cli - - cerc/fixturenet-eth-geth - - cerc/fixturenet-eth-lighthouse - cerc/foundry pods: - - fixturenet-eth + - mainnet-eth - foundry -- 2.45.2 From f1e68233dd97a42eb35d549f3927b5ade098b782 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 21 Jul 2023 15:52:58 -0600 Subject: [PATCH 06/15] Start up scripting --- .../compose/docker-compose-mainnet-eth.yml | 17 +++-- .../config/mainnet-eth/scripts/run-geth.sh | 67 +++++++++++++++++++ .../mainnet-eth/scripts/run-lighthouse.sh | 34 ++++++++++ 3 files changed, 111 insertions(+), 7 deletions(-) create mode 100755 app/data/config/mainnet-eth/scripts/run-geth.sh create mode 100755 app/data/config/mainnet-eth/scripts/run-lighthouse.sh diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/app/data/compose/docker-compose-mainnet-eth.yml index 9273edfb..6ea29a54 100644 --- a/app/data/compose/docker-compose-mainnet-eth.yml +++ b/app/data/compose/docker-compose-mainnet-eth.yml @@ -11,11 +11,13 @@ services: CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect} CERC_STATEDIFF_DB_NODE_ID: 1 CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - env_file: - - ../config/fixturenet-eth/fixturenet-eth.env image: cerc/go-ethereum:local + entrypoint: /bin/sh + command: -c "/opt/run-geth.sh" volumes: - mainnet_eth_geth_1_data:/root/ethdata + - mainnet_eth_config_data:/etc/mainnet-eth + - ../config/mainnet-eth/scripts/run-geth.sh:/opt/run-geth.sh healthcheck: test: ["CMD", "nc", "-v", "localhost", "8545"] interval: 30s @@ -36,15 +38,15 @@ services: timeout: 10s retries: 10 start_period: 30s - env_file: - - ../config/fixturenet-eth/fixturenet-eth.env environment: - NODE_NUMBER: "1" - ETH1_ENDPOINT: "http://mainnet-eth-geth-1:8545" EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551" image: cerc/lighthouse:local + entrypoint: /bin/sh + command: -c "/opt/run-lighthouse.sh" volumes: - - mainnet_eth_lighthouse_1_data:/opt/testnet/build/cl + - mainnet_eth_lighthouse_1_data:/var/lighthouse-data-dir + - mainnet_eth_config_data:/etc/mainnet-eth + - ../config/mainnet-eth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh depends_on: mainnet-eth-geth-1: condition: service_healthy @@ -52,5 +54,6 @@ services: - "8001" volumes: + mainnet_eth_config_data: mainnet_eth_geth_1_data: mainnet_eth_lighthouse_1_data: diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh new file mode 100755 index 00000000..d50698c7 --- /dev/null +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -0,0 +1,67 @@ +#!/bin/bash +if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then + set -x +fi + +CERC_ETH_DATADIR=/root/ethdata + +START_CMD="geth" +if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then + START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" +fi + +# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script +cleanup() { + echo "Signal received, cleaning up..." + + # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process) + pkill -P ${geth_pid} + sleep 2 + kill $(jobs -p) + + wait + echo "Done" +} + +$START_CMD \ + --datadir="${CERC_ETH_DATADIR}" \ + --allow-insecure-unlock \ + --http \ + --http.addr="0.0.0.0" \ + --http.vhosts="*" \ + --http.api="${CERC_GETH_HTTP_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ + --http.corsdomain="*" \ + --authrpc.addr="0.0.0.0" \ + --authrpc.vhosts="*" \ + --authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \ + --ws \ + --ws.addr="0.0.0.0" \ + --ws.origins="*" \ + --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ + --http.corsdomain="*" \ + --networkid="${NETWORK_ID}" \ + --netrestrict="${NETRESTRICT}" \ + --gcmode archive \ + --txlookuplimit=0 \ + --cache.preimages \ + --syncmode=full \ + --mine \ + --miner.threads=1 \ + --metrics \ + --metrics.addr="0.0.0.0" \ + --verbosity=${CERC_GETH_VERBOSITY:-3} \ + --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \ + --miner.etherbase="${ETHERBASE}" \ + ${STATEDIFF_OPTS} \ + & + +geth_pid=$! + + +wait $geth_pid + +if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then + while [ 1 -eq 1 ]; do + sleep 60 + done +fi diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh new file mode 100755 index 00000000..2005a6d4 --- /dev/null +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -0,0 +1,34 @@ +#!/bin/bash +if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then + set -x +fi + +DEBUG_LEVEL=${CERC_LIGHTHOUSE_DEBUG_LEVEL:-info} + +# Get positional arguments +data_dir=$DATADIR/node_${NODE_NUMBER} +network_port=9001 +http_port=8001 +authrpc_port=8551 + +exec lighthouse \ + bn \ + --debug-level $DEBUG_LEVEL \ + --boot-nodes "$ENR" \ + --datadir $data_dir \ + --testnet-dir $TESTNET_DIR \ + --enable-private-discovery \ + --staking \ + --enr-address $ENR_IP \ + --enr-udp-port $network_port \ + --enr-tcp-port $network_port \ + --port $network_port \ + --http-address 0.0.0.0 \ + --http-port $http_port \ + --disable-packet-filter \ + --execution-endpoint $EXECUTION_ENDPOINT \ + --execution-jwt $JWTSECRET \ + --terminal-total-difficulty-override $ETH1_TTD \ + --suggested-fee-recipient $SUGGESTED_FEE_RECIPIENT \ + --target-peers $((BN_COUNT - 1)) \ + --http-allow-sync-stalled \ -- 2.45.2 From 285a3826182e9f1017f5ab8be2843f7991e8f8a7 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 21 Jul 2023 16:25:20 -0600 Subject: [PATCH 07/15] Update startup scripts --- .../config/mainnet-eth/scripts/run-geth.sh | 23 ++++-------------- .../mainnet-eth/scripts/run-lighthouse.sh | 24 +++++-------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index d50698c7..c72b371f 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -25,34 +25,19 @@ cleanup() { $START_CMD \ --datadir="${CERC_ETH_DATADIR}" \ - --allow-insecure-unlock \ - --http \ - --http.addr="0.0.0.0" \ - --http.vhosts="*" \ - --http.api="${CERC_GETH_HTTP_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ - --http.corsdomain="*" \ --authrpc.addr="0.0.0.0" \ + --authrpc.port 8551 \ --authrpc.vhosts="*" \ - --authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \ + --authrpc.jwtsecret="/etc/mainnet-eth/jwtsecret" \ --ws \ --ws.addr="0.0.0.0" \ --ws.origins="*" \ --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ --http.corsdomain="*" \ - --networkid="${NETWORK_ID}" \ - --netrestrict="${NETRESTRICT}" \ - --gcmode archive \ + --gcmode full \ --txlookuplimit=0 \ --cache.preimages \ - --syncmode=full \ - --mine \ - --miner.threads=1 \ - --metrics \ - --metrics.addr="0.0.0.0" \ - --verbosity=${CERC_GETH_VERBOSITY:-3} \ - --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \ - --miner.etherbase="${ETHERBASE}" \ - ${STATEDIFF_OPTS} \ + --syncmode=snap \ & geth_pid=$! diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index 2005a6d4..ac4ffc1b 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -5,8 +5,8 @@ fi DEBUG_LEVEL=${CERC_LIGHTHOUSE_DEBUG_LEVEL:-info} -# Get positional arguments -data_dir=$DATADIR/node_${NODE_NUMBER} +data_dir=/var/lighthouse-data-dir + network_port=9001 http_port=8001 authrpc_port=8551 @@ -14,21 +14,9 @@ authrpc_port=8551 exec lighthouse \ bn \ --debug-level $DEBUG_LEVEL \ - --boot-nodes "$ENR" \ --datadir $data_dir \ - --testnet-dir $TESTNET_DIR \ - --enable-private-discovery \ - --staking \ - --enr-address $ENR_IP \ - --enr-udp-port $network_port \ - --enr-tcp-port $network_port \ - --port $network_port \ - --http-address 0.0.0.0 \ - --http-port $http_port \ - --disable-packet-filter \ + --network mainnet \ --execution-endpoint $EXECUTION_ENDPOINT \ - --execution-jwt $JWTSECRET \ - --terminal-total-difficulty-override $ETH1_TTD \ - --suggested-fee-recipient $SUGGESTED_FEE_RECIPIENT \ - --target-peers $((BN_COUNT - 1)) \ - --http-allow-sync-stalled \ + --execution-jwt /etc/mainnet-eth/jwtsecret \ + --disable-deposit-contract-sync \ + --checkpoint-sync-url https://mainnet.checkpoint.sigp.io -- 2.45.2 From ed0cc4bb37905d93ace8258a4a19c3bf40296e4c Mon Sep 17 00:00:00 2001 From: David Boreham Date: Sun, 23 Jul 2023 21:33:38 -0600 Subject: [PATCH 08/15] Basic deploy commands for stack --- .../stacks/mainnet-eth/deploy/commands.py | 25 +++++++++++++++++++ app/deployment_create.py | 19 +++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 app/data/stacks/mainnet-eth/deploy/commands.py diff --git a/app/data/stacks/mainnet-eth/deploy/commands.py b/app/data/stacks/mainnet-eth/deploy/commands.py new file mode 100644 index 00000000..28964073 --- /dev/null +++ b/app/data/stacks/mainnet-eth/deploy/commands.py @@ -0,0 +1,25 @@ +# Copyright © 2023 Cerc + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +def init(ctx): + return None + + +def setup(ctx): + return None + + +def create(ctx): + print("Yay it worked") diff --git a/app/deployment_create.py b/app/deployment_create.py index ff06664a..6d41e8d6 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -59,7 +59,7 @@ def _get_named_volumes(stack): def _create_bind_dir_if_relative(volume, path_string, compose_dir): path = Path(path_string) if not path.is_absolute(): - absolute_path = Path(compose_dir).joinpath(path) + absolute_path = Path(compose_dir).parent.joinpath(path) absolute_path.mkdir(parents=True, exist_ok=True) else: if not path.exists(): @@ -76,7 +76,7 @@ def _fixup_pod_file(pod, spec, compose_dir): for volume in pod_volumes.keys(): if volume in spec_volumes: volume_spec = spec_volumes[volume] - volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f"{volume_spec}" + volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}" _create_bind_dir_if_relative(volume, volume_spec, compose_dir) new_volume_spec = {"driver": "local", "driver_opts": { @@ -111,6 +111,18 @@ def call_stack_deploy_setup(stack): return imported_stack.setup(None) +# TODO: fold this with function above +def call_stack_deploy_create(stack): + # Link with the python file in the stack + # Call a function in it + # If no function found, return None + python_file_path = get_stack_file_path(stack).parent.joinpath("deploy", "commands.py") + spec = util.spec_from_file_location("commands", python_file_path) + imported_stack = util.module_from_spec(spec) + spec.loader.exec_module(imported_stack) + return imported_stack.create(None) + + # Inspect the pod yaml to find config files referenced in subdirectories # other than the one associated with the pod def _find_extra_config_dirs(parsed_pod_file, pod): @@ -138,7 +150,8 @@ def init(ctx, output): verbose = global_options(ctx).verbose default_spec_file_content = call_stack_deploy_init(stack) spec_file_content = {"stack": stack} - spec_file_content.update(default_spec_file_content) + if default_spec_file_content: + spec_file_content.update(default_spec_file_content) if verbose: print(f"Creating spec file for stack: {stack}") named_volumes = _get_named_volumes(stack) -- 2.45.2 From b14488d04073e280760d45f21781d47136073448 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Sun, 23 Jul 2023 22:24:12 -0600 Subject: [PATCH 09/15] Generate jwt --- app/data/stacks/mainnet-eth/deploy/commands.py | 8 +++++++- app/deployment_create.py | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/data/stacks/mainnet-eth/deploy/commands.py b/app/data/stacks/mainnet-eth/deploy/commands.py index 28964073..194c970d 100644 --- a/app/data/stacks/mainnet-eth/deploy/commands.py +++ b/app/data/stacks/mainnet-eth/deploy/commands.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from secrets import token_hex + def init(ctx): return None @@ -22,4 +24,8 @@ def setup(ctx): def create(ctx): - print("Yay it worked") + # Generate the JWT secret and save to its config file + secret = token_hex(16) + jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_config_data", "jwtsecret") + with open(jwt_file_path, 'w+') as jwt_file: + jwt_file.write(secret) diff --git a/app/deployment_create.py b/app/deployment_create.py index 6d41e8d6..652bbdba 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import click +from dataclasses import dataclass from importlib import util import os from pathlib import Path @@ -21,6 +22,11 @@ from shutil import copyfile, copytree import sys from app.util import get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml +@dataclass +class DeploymentContext: + stack: str + deployment_dir: Path + def _make_default_deployment_dir(): return "deployment-001" @@ -112,15 +118,15 @@ def call_stack_deploy_setup(stack): # TODO: fold this with function above -def call_stack_deploy_create(stack): +def call_stack_deploy_create(deployment_context): # Link with the python file in the stack # Call a function in it # If no function found, return None - python_file_path = get_stack_file_path(stack).parent.joinpath("deploy", "commands.py") + python_file_path = get_stack_file_path(deployment_context.stack).parent.joinpath("deploy", "commands.py") spec = util.spec_from_file_location("commands", python_file_path) imported_stack = util.module_from_spec(spec) spec.loader.exec_module(imported_stack) - return imported_stack.create(None) + return imported_stack.create(deployment_context) # Inspect the pod yaml to find config files referenced in subdirectories @@ -210,6 +216,9 @@ def create(ctx, spec_file, deployment_dir): # If the same config dir appears in multiple pods, it may already have been copied if not os.path.exists(destination_config_dir): copytree(source_config_dir, destination_config_dir) + # Delegate to the stack's Python code + deployment_context = DeploymentContext(stack_name, Path(deployment_dir)) + call_stack_deploy_create(deployment_context) @click.command() @@ -222,4 +231,3 @@ def create(ctx, spec_file, deployment_dir): def setup(ctx, node_moniker, key_name, initialize_network, join_network, create_network): stack = global_options(ctx).stack call_stack_deploy_setup(stack) - -- 2.45.2 From af6d75503605a4c3e800d0cce2458740b409a527 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Sun, 23 Jul 2023 22:54:55 -0600 Subject: [PATCH 10/15] Generate jwt secret --- app/data/config/mainnet-eth/scripts/run-geth.sh | 2 +- app/data/config/mainnet-eth/scripts/run-lighthouse.sh | 2 +- app/data/stacks/mainnet-eth/deploy/commands.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/data/config/mainnet-eth/scripts/run-geth.sh b/app/data/config/mainnet-eth/scripts/run-geth.sh index c72b371f..c249618b 100755 --- a/app/data/config/mainnet-eth/scripts/run-geth.sh +++ b/app/data/config/mainnet-eth/scripts/run-geth.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then set -x fi diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index ac4ffc1b..48dbbaca 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then set -x fi diff --git a/app/data/stacks/mainnet-eth/deploy/commands.py b/app/data/stacks/mainnet-eth/deploy/commands.py index 194c970d..09586aac 100644 --- a/app/data/stacks/mainnet-eth/deploy/commands.py +++ b/app/data/stacks/mainnet-eth/deploy/commands.py @@ -25,7 +25,7 @@ def setup(ctx): def create(ctx): # Generate the JWT secret and save to its config file - secret = token_hex(16) + secret = token_hex(32) jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_config_data", "jwtsecret") with open(jwt_file_path, 'w+') as jwt_file: jwt_file.write(secret) -- 2.45.2 From ef72e885d74ce375eab83df4fceceb97bfcd0333 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 24 Jul 2023 05:41:47 -0600 Subject: [PATCH 11/15] Implement --stay-attached --- app/deploy.py | 4 ++-- app/deployment.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/deploy.py b/app/deploy.py index 41af70f6..4ddd81a7 100644 --- a/app/deploy.py +++ b/app/deploy.py @@ -61,7 +61,7 @@ def create_deploy_context(global_context, stack, include, exclude, cluster, env_ return DeployCommandContext(cluster_context, docker) -def up_operation(ctx, services_list): +def up_operation(ctx, services_list, stay_attached=False): global_context = ctx.parent.parent.obj deploy_context = ctx.obj if not global_context.dry_run: @@ -73,7 +73,7 @@ def up_operation(ctx, services_list): print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}") for pre_start_command in cluster_context.pre_start_commands: _run_command(global_context, cluster_context.cluster, pre_start_command) - deploy_context.docker.compose.up(detach=True, services=services_list) + deploy_context.docker.compose.up(detach=not stay_attached, services=services_list) for post_start_command in cluster_context.post_start_commands: _run_command(global_context, cluster_context.cluster, post_start_command) _orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.docker, container_exec_env) diff --git a/app/deployment.py b/app/deployment.py index e876b77e..132568ca 100644 --- a/app/deployment.py +++ b/app/deployment.py @@ -54,22 +54,24 @@ def make_deploy_context(ctx): @command.command() +@click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout") @click.argument('extra_args', nargs=-1) # help: command: up @click.pass_context -def up(ctx, extra_args): +def up(ctx, stay_attached, extra_args): ctx.obj = make_deploy_context(ctx) services_list = list(extra_args) or None - up_operation(ctx, services_list) + up_operation(ctx, services_list, stay_attached) # start is the preferred alias for up @command.command() +@click.option("--stay-attached/--detatch-terminal", default=False, help="detatch or not to see container stdout") @click.argument('extra_args', nargs=-1) # help: command: up @click.pass_context -def start(ctx, extra_args): +def start(ctx, stay_attached, extra_args): ctx.obj = make_deploy_context(ctx) services_list = list(extra_args) or None - up_operation(ctx, services_list) + up_operation(ctx, services_list, stay_attached) @command.command() -- 2.45.2 From 6c8a7616bccb4cb29e96726ed67305379ac01d7b Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 24 Jul 2023 06:11:05 -0600 Subject: [PATCH 12/15] Lighthouse doesn't depend on geth --- app/data/compose/docker-compose-mainnet-eth.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/data/compose/docker-compose-mainnet-eth.yml b/app/data/compose/docker-compose-mainnet-eth.yml index 6ea29a54..7b44a88e 100644 --- a/app/data/compose/docker-compose-mainnet-eth.yml +++ b/app/data/compose/docker-compose-mainnet-eth.yml @@ -47,9 +47,6 @@ services: - mainnet_eth_lighthouse_1_data:/var/lighthouse-data-dir - mainnet_eth_config_data:/etc/mainnet-eth - ../config/mainnet-eth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh - depends_on: - mainnet-eth-geth-1: - condition: service_healthy ports: - "8001" -- 2.45.2 From c79e2f25f74aa5dcbfea2452ed2d7e3d97b76412 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 24 Jul 2023 06:11:20 -0600 Subject: [PATCH 13/15] Bash is in the lighthouse image --- app/data/config/mainnet-eth/scripts/run-lighthouse.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh index 48dbbaca..02c97922 100755 --- a/app/data/config/mainnet-eth/scripts/run-lighthouse.sh +++ b/app/data/config/mainnet-eth/scripts/run-lighthouse.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then set -x fi @@ -19,4 +19,4 @@ exec lighthouse \ --execution-endpoint $EXECUTION_ENDPOINT \ --execution-jwt /etc/mainnet-eth/jwtsecret \ --disable-deposit-contract-sync \ - --checkpoint-sync-url https://mainnet.checkpoint.sigp.io + --checkpoint-sync-url https://beaconstate.ethstaker.cc -- 2.45.2 From 6ac8f084cfad9293a175e041c6fba4cb64eb2c21 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 25 Jul 2023 09:37:12 -0600 Subject: [PATCH 14/15] Add test script --- tests/mainnet-eth/run-test.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 tests/mainnet-eth/run-test.sh diff --git a/tests/mainnet-eth/run-test.sh b/tests/mainnet-eth/run-test.sh new file mode 100755 index 00000000..dd56c183 --- /dev/null +++ b/tests/mainnet-eth/run-test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +echo "Running stack-orchestrator Ethereum mainnet test" +# Bit of a hack, test the most recent package +TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) +# Set a new unique repo dir +export CERC_REPO_BASE_DIR=$(mktemp -d stack-orchestrator-mainnet-eth-test.XXXXXXXXXX) +DEPLOYMENT_DIR=mainnet-eth-deployment-test +echo "Testing this package: $TEST_TARGET_SO" +echo "Test version command" +reported_version_string=$( $TEST_TARGET_SO version ) +echo "Version reported is: ${reported_version_string}" +echo "Cloning repositories into: $CERC_REPO_BASE_DIR" +$TEST_TARGET_SO --stack mainnet-eth setup-repositories +$TEST_TARGET_SO --stack mainnet-eth build-containers +$TEST_TARGET_SO --stack mainnet-eth deploy init --output mainnet-eth-spec.yml +$TEST_TARGET_SO deploy create --spec-file mainnet-eth-spec.yml --deployment-dir $DEPLOYMENT_DIR +# Start the stack +$TEST_TARGET_SO deployment --dir $DEPLOYMENT_DIR start +# Verify that the stack is up and running +$TEST_TARGET_SO deployment --dir $DEPLOYMENT_DIR ps +#TODO: add a check that the container logs show good startup +$TEST_TARGET_SO deployment --dir $DEPLOYMENT_DIR stop --delete-volumes +echo "Removing deployment directory" +rm -rf $DEPLOYMENT_DIR +echo "Removing cloned repositories" +rm -rf $CERC_REPO_BASE_DIR +exit $test_result -- 2.45.2 From 43b1da696690fd94b63a7600e7f88cd1e0ab8469 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 25 Jul 2023 10:11:45 -0600 Subject: [PATCH 15/15] Add readme for mainnet-eth stack --- app/data/stacks/mainnet-eth/README.md | 107 ++++++++++++++++++++------ 1 file changed, 82 insertions(+), 25 deletions(-) diff --git a/app/data/stacks/mainnet-eth/README.md b/app/data/stacks/mainnet-eth/README.md index 457860b9..8299ce54 100644 --- a/app/data/stacks/mainnet-eth/README.md +++ b/app/data/stacks/mainnet-eth/README.md @@ -1,6 +1,6 @@ # mainnet-eth -Mainnet Ethereum stack (experimental) +Deploys a "head-tracking" mainnet Ethereum stack comprising a [go-ethereum](https://github.com/cerc-io/go-ethereum) execution layer node and a [lighthouse](https://github.com/sigp/lighthouse) consensus layer node. ## Clone required repositories @@ -8,40 +8,97 @@ Mainnet Ethereum stack (experimental) $ laconic-so --stack mainnet-eth setup-repositories ``` -## Build the fixturenet-eth containers +## Build containers ``` $ laconic-so --stack mainnet-eth build-containers ``` -This should create several container images in the local image registry: - -* cerc/go-ethereum -* cerc/lighthouse -* cerc/fixturenet-eth-geth -* cerc/fixturenet-eth-lighthouse - ## Create a deployment ``` -$ laconic-so --stack mainnet-eth deploy create +$ laconic-so --stack mainnet-eth deploy init --output mainnet-eth-spec.yml +$ laconic-so deploy create --spec-file mainnet-eth-spec.yml --deployment-dir mainnet-eth-deployment +``` +## Start the stack +``` +$ laconic-so deployment --dir mainnet-eth-deployment start +``` +Display stack status: +``` +$ laconic-so deployment --dir mainnet-eth-deployment ps +Running containers: +id: f39608eca04d72d6b0f1f3acefc5ebb52908da06e221d20c7138f7e3dff5e423, name: laconic-ef641b4d13eb61ed561b19be67063241-foundry-1, ports: +id: 4052b1eddd886ae0d6b41f9ff22e68a70f267b2bfde10f4b7b79b5bd1eeddcac, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-geth-1-1, ports: 30303/tcp, 30303/udp, 0.0.0.0:49184->40000/tcp, 0.0.0.0:49185->6060/tcp, 0.0.0.0:49186->8545/tcp, 8546/tcp +id: ac331232e597944b621b3b8942ace5dafb14524302cab338ff946c7f6e5a1d52, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1, ports: 0.0.0.0:49187->8001/tcp +``` +See stack logs: +``` +$ laconic-so deployment --dir mainnet-eth-deployment logs +time="2023-07-25T09:46:29-06:00" level=warning msg="The \"CERC_SCRIPT_DEBUG\" variable is not set. Defaulting to a blank string." +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1 | Jul 25 15:45:13.362 INFO Logging to file path: "/var/lighthouse-data-dir/beacon/logs/beacon.log" +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Lighthouse started version: Lighthouse/v4.1.0-693886b +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Configured for network name: mainnet +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Data directory initialised datadir: /var/lighthouse-data-dir +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Deposit contract address: 0x00000000219ab540356cbb839cbe05303d7705fa, deploy_block: 11184524 +laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-lighthouse-1-1 | Jul 25 15:45:13.424 INFO Starting checkpoint sync remote_url: https://beaconstate.ethstaker.cc/, service: beacon +``` +## Monitoring stack sync progress +Both go-ethereum and lighthouse will engage in an initial chain sync phase that will last up to several hours depending on hardware performance and network capacity. +Syncing can be monitored by looking for these log messages: +``` +Jul 24 12:34:17.001 INFO Downloading historical blocks est_time: 5 days 11 hrs, speed: 14.67 slots/sec, distance: 6932481 slots (137 weeks 3 days), service: slot_notifier +INFO [07-24|12:14:52.493] Syncing beacon headers downloaded=145,920 left=17,617,968 eta=1h23m32.815s +INFO [07-24|12:33:15.238] Syncing: chain download in progress synced=1.86% chain=148.94MiB headers=368,640@95.03MiB bodies=330,081@40.56MiB receipts=330,081@13.35MiB eta=37m54.505s +INFO [07-24|12:35:13.028] Syncing: state download in progress synced=1.32% state=4.64GiB accounts=2,850,314@677.57MiB slots=18,663,070@3.87GiB codes=26662@111.14MiB eta=3h18m0.699s +``` +Once synced up these log messages will be observed: +``` +INFO Synced slot: 6952515, block: 0x5bcb…f6d9, epoch: 217266, finalized_epoch: 217264, finalized_root: 0x6342…2c5c, exec_hash: 0x8d8c…2443 (verified), peers: 31, service: slot_notifier +INFO [07-25|03:04:48.941] Imported new potential chain segment number=17,767,316 hash=84f6e7..bc2cb0 blocks=1 txs=137 mgas=16.123 elapsed=57.087ms mgasps=282.434 dirty=461.46MiB +INFO [07-25|03:04:49.042] Chain head was updated number=17,767,316 hash=84f6e7..bc2cb0 root=ca58b2..8258c1 elapsed=2.480111ms ``` - - ## Clean up -Stop all services running in the background: - -```bash -$ laconic-so deployment --dir down +Stop the stack: ``` - -Clear volumes created by this stack: - -```bash -# List all relevant volumes -$ docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data" - -# Remove all the listed volumes -$ docker volume rm $(docker volume ls -q --filter "name=.*fixturenet_eth_bootnode_geth_data|.*fixturenet_eth_geth_1_data|.*fixturenet_eth_geth_2_data|.*fixturenet_eth_lighthouse_1_data|.*fixturenet_eth_lighthouse_2_data") +$ laconic-so deployment --dir mainnet-eth-deployment stop ``` +This leaves data volumes in place, allowing the stack to be subsequently re-started. +To permanently *delete* the stack's data volumes run: +``` +$ laconic-so deployment --dir mainnet-eth-deployment stop --delete-data-volumes +``` +After deleting the volumes, any subsequent re-start will begin chain sync from cold. +## Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory: +``` +$ cat mainnet-eth-spec.yml +stack: mainnet-eth +volumes: + mainnet_eth_config_data: ./data/mainnet_eth_config_data + mainnet_eth_geth_1_data: ./data/mainnet_eth_geth_1_data + mainnet_eth_lighthouse_1_data: ./data/mainnet_eth_lighthouse_1_data +``` +A synced-up stack will consume around 900GB of data volume space: +``` +$ sudo du -h mainnet-eth-deployment/data/ +150M mainnet-eth-deployment/data/mainnet_eth_lighthouse_1_data/beacon/freezer_db +25G mainnet-eth-deployment/data/mainnet_eth_lighthouse_1_data/beacon/chain_db +16K mainnet-eth-deployment/data/mainnet_eth_lighthouse_1_data/beacon/network +368M mainnet-eth-deployment/data/mainnet_eth_lighthouse_1_data/beacon/logs +26G mainnet-eth-deployment/data/mainnet_eth_lighthouse_1_data/beacon +26G mainnet-eth-deployment/data/mainnet_eth_lighthouse_1_data +8.0K mainnet-eth-deployment/data/mainnet_eth_config_data +4.0K mainnet-eth-deployment/data/mainnet_eth_geth_1_data/keystore +527G mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth/chaindata/ancient/chain +527G mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth/chaindata/ancient +859G mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth/chaindata +4.8M mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth/nodes +242M mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth/ethash +669M mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth/triecache +860G mainnet-eth-deployment/data/mainnet_eth_geth_1_data/geth +860G mainnet-eth-deployment/data/mainnet_eth_geth_1_data +885G mainnet-eth-deployment/data/ +``` \ No newline at end of file -- 2.45.2