Merge pull request #435 from cerc-io/reth

Add reth stack

app/data/compose/docker-compose-reth.yml

@@ -0,0 +1,35 @@
+version: "3.8"
+
+services:
+  reth:
+    restart: unless-stopped
+    hostname: reth
+    image: cerc/reth:local
+    entrypoint: ["sh", "/docker-entrypoint-scripts.d/start-reth.sh"]
+    volumes:
+      - ../config/reth/start-reth.sh:/docker-entrypoint-scripts.d/start-reth.sh
+      - reth_data:/root/.local/share/reth
+      - shared_data:/root/.shared_data
+    ports:
+      - "8545:8545" # http rpc
+      - "8546:8546" # ws rpc
+      - "30303:30303" # network listening port
+      - "30303:30303/udp"
+      - "8551" # consensus auth
+
+  lighthouse:
+    restart: unless-stopped
+    hostname: lighthouse
+    image: cerc/lighthouse:local
+    entrypoint: ["sh", "/docker-entrypoint-scripts.d/start-lighthouse.sh"]
+    volumes:
+      - ../config/reth/start-lighthouse.sh:/docker-entrypoint-scripts.d/start-lighthouse.sh
+      - lighthouse_data:/root/.lighthouse/mainnet
+      - shared_data:/root/.shared_data
+    ports:
+      - "8001"
+
+volumes:
+  reth_data:
+  lighthouse_data:
+  shared_data:

app/data/compose/docker-compose-test.yml

@@ -5,7 +5,7 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     volumes:
-      - test-data:/var
+      - test-data:/data
     ports:
       - "80"
 

app/data/config/reth/start-lighthouse.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Wait for reth container to create jwt auth token
+while [ ! -f /root/.shared_data/jwt.hex ]; do
+  echo "Jwt auth token not found, sleeping for 5s..."
+  sleep 5
+done
+
+echo "Jwt token found. Starting Lighthouse..."
+export RUST_LOG=info
+lighthouse bn \
+  --network mainnet \
+  --execution-endpoint http://reth:8551 \
+  --execution-jwt /root/.shared_data/jwt.hex \
+  --checkpoint-sync-url https://mainnet.checkpoint.sigp.io \
+  --disable-deposit-contract-sync

app/data/config/reth/start-reth.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# generate jwt token for reth/lighthouse authentication
+echo "Installing OpenSSL..."
+apt update
+apt install openssl
+echo "Generating jwt token for lighthouse auth..."
+openssl rand -hex 32 | tr -d "\n" | tee /root/.shared_data/jwt.hex
+
+# start reth
+echo "Starting Reth..."
+export RUST_LOG=info
+reth node \
+  --authrpc.jwtsecret /root/.shared_data/jwt.hex \
+  --authrpc.addr 0.0.0.0 \
+  --authrpc.port 8551 \
+  --http \
+  --http.addr 0.0.0.0 \
+  --http.corsdomain * \
+  --http.api eth,web3,net,rpc \
+  --ws \
+  --ws.addr 0.0.0.0 \
+  --ws.origins * \
+  --ws.api eth,web3,net,rpc

app/data/container-build/cerc-reth/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build cerc/go-opera
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+docker build -t cerc/reth:local ${build_command_args} ${CERC_REPO_BASE_DIR}/reth

app/data/container-build/cerc-test-container/run.sh

@@ -4,7 +4,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 # Test if the container's filesystem is old (run previously) or new
-EXISTSFILENAME=/var/exists
+EXISTSFILENAME=/data/exists
 echo "Test container starting"
 if [[ -f "$EXISTSFILENAME" ]];
 then

app/data/container-image-list.txt

@@ -43,3 +43,4 @@ cerc/watcher-gelato
 cerc/lotus
 cerc/go-opera
 cerc/lasso
+cerc/reth

app/data/pod-list.txt

@@ -29,3 +29,4 @@ watcher-gelato
 fixturenet-lotus
 mainnet-go-opera
 lasso
+reth

app/data/repository-list.txt

@@ -36,3 +36,4 @@ github.com/filecoin-project/lotus
 git.vdb.to/cerc-io/test-project
 github.com/Fantom-foundation/go-opera
 github.com/cerc-io/lasso
+github.com/paradigmxyz/reth

app/data/stacks/reth/README.md

@@ -0,0 +1,65 @@
+# Reth
+Deploy a Reth API node alongside Lighthouse.
+
+## Clone required repositories
+
+```
+$ laconic-so --stack reth setup-repositories
+```
+
+## Build the Reth stack containers
+
+```
+$ laconic-so --stack reth build-containers
+```
+
+## Deploy the stack
+
+```
+$ laconic-so --stack reth deploy up
+```
+
+## Check logs
+
+```
+$ laconic-so --stack reth deploy logs
+```
+
+Verify that your node is syncing. You should see entries similar to this from the Lighthouse container:
+
+```
+laconic-200e8f8ff7891515d777cd0f719078e3-lighthouse-1  | Jun 23 20:59:01.226 INFO New block received                      root: 0x9cd4a2dd9333cf802c2963c2f029deb0f94e511d2481fa0724ae8752e4c49b15, slot: 6727493
+```
+and entries similar to this from the Reth container:
+```
+laconic-200e8f8ff7891515d777cd0f719078e3-reth-1        | 2023-06-23T20:59:11.557389Z  INFO reth::node::events: Stage committed progress pipeline_stages=1/13 stage=Headers block=0 checkpoint=4.9% eta=1h 3m 57s
+```
+
+## Test the API
+
+Reth's http api is accessible on port `8545` and the websocket api is accessible on port `8546`.
+```
+$ curl --request POST \
+    --url http://localhost:8545/ \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "jsonrpc": "2.0",
+    "method": "eth_blockNumber",
+    "params": [],
+    "id": 0
+  }'
+
+# Response
+{"jsonrpc":"2.0","result":"0x0","id":0}
+```
+
+## Clean up
+
+Stop all services running in the background:
+
+```bash
+$ laconic-so --stack reth deploy down
+```
+To also delete the docker data volumes:
+```bash
+$ laconic-so --stack reth deploy down --delete-volumes

app/data/stacks/reth/stack.yml

@@ -0,0 +1,10 @@
+version: "1.1"
+name: reth
+decription: "Reth node"
+repos:
+  - github.com/paradigmxyz/reth
+containers:
+  - cerc/reth
+  - cerc/lighthouse
+pods:
+  - reth

app/deployment_create.py

@@ -60,8 +60,22 @@ def _get_named_volumes(stack):
     return named_volumes
 
 
+# If we're mounting a volume from a relatie path, then we
+# assume the directory doesn't exist yet and create it
+# so the deployment will start
+# Also warn if the path is absolute and doesn't exist
+def _create_bind_dir_if_relative(volume, path_string, compose_dir):
+    path = Path(path_string)
+    if not path.is_absolute():
+        absolute_path = Path(compose_dir).parent.joinpath(path)
+        absolute_path.mkdir(parents=True, exist_ok=True)
+    else:
+        if not path.exists():
+            print(f"WARNING: mount path for volume {volume} does not exist: {path_string}")
+
+
 # See: https://stackoverflow.com/questions/45699189/editing-docker-compose-yml-with-pyyaml
-def _fixup_pod_file(pod, spec):
+def _fixup_pod_file(pod, spec, compose_dir):
     # Fix up volumes
     if "volumes" in spec:
         spec_volumes = spec["volumes"]
@@ -70,10 +84,12 @@ def _fixup_pod_file(pod, spec):
             for volume in pod_volumes.keys():
                 if volume in spec_volumes:
                     volume_spec = spec_volumes[volume]
-                    new_volume_spec = {"name": volume,
+                    volume_spec_fixedup = volume_spec if Path(volume_spec).is_absolute() else f".{volume_spec}"
+                    _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
+                    new_volume_spec = {"driver": "local",
                                        "driver_opts": {
                                            "type": "none",
-                                           "device": volume_spec,
+                                           "device": volume_spec_fixedup,
                                            "o": "bind"
                                         }
                                        }
@@ -94,7 +110,7 @@ def init(ctx, output):
     if named_volumes:
         volume_descriptors = {}
         for named_volume in named_volumes:
-            volume_descriptors[named_volume] = f"./data/{named_volume}"
+            volume_descriptors[named_volume] = f"../data/{named_volume}"
         spec_file_content["volumes"] = volume_descriptors
     with open(output, "w") as output_file:
         yaml.dump(spec_file_content, output_file)
@@ -130,7 +146,7 @@ def create(ctx, spec_file, deployment_dir):
     for pod in pods:
         pod_file_path = os.path.join(_get_compose_file_dir(), f"docker-compose-{pod}.yml")
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
-        _fixup_pod_file(parsed_pod_file, parsed_spec)
+        _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
         with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file:
             yaml.dump(parsed_pod_file, output_file)
         # Copy the config files for the pod, if any