Update mainnet stack.

app/data/compose/docker-compose-mainnet-eth-keycloak.yml

@@ -0,0 +1,35 @@
+version: '3.8'
+
+services:
+  keycloak-db:
+    image: postgres:14-alpine
+    env_file:
+      - ../config/mainnet-eth-keycloak/keycloak.env
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    volumes:
+      - mainnet_eth_keycloak_db:/var/lib/postgresql/data
+    ports:
+      - 5432
+
+  keycloak:
+    image: cerc/keycloak:local
+    env_file:
+      - ../config/mainnet-eth-keycloak/keycloak.env
+    environment:
+      JAVA_OPTS_APPEND: "-Dkeycloak.migration.action=import -Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=/import -Dkeycloak.migration.strategy=IGNORE_EXISTING"
+    volumes:
+      - ../config/mainnet-eth-keycloak/import:/import
+    ports:
+      - 8080
+    command: ["start"]
+    depends_on:
+      keycloak-db:
+        condition: service_healthy
+
+volumes:
+  mainnet_eth_keycloak_db:

app/data/compose/docker-compose-mainnet-eth-metrics.yml

@@ -0,0 +1,23 @@
+version: "3.2"
+services:
+  prometheus:
+    restart: always
+    image: prom/prometheus
+    depends_on:
+      mainnet-eth-geth-1:
+        condition: service_healthy
+    volumes:
+      - ../config/mainnet-eth-metrics/prometheus/etc:/etc/prometheus
+    ports:
+      - "9090"
+  grafana:
+    restart: always
+    image: grafana/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=changeme6325
+    volumes:
+      - ../config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ../config/mainnet-eth-metrics/grafana/etc/provisioning/datasources:/etc/grafana/provisioning/datasources
+      - ../config/mainnet-eth-metrics/grafana/etc/dashboards:/etc/grafana/dashboards
+    ports:
+      - "3000"

app/data/compose/docker-compose-mainnet-eth.yml

@@ -6,16 +6,13 @@ services:
     hostname: mainnet-eth-geth-1
     cap_add:
       - SYS_PTRACE
-    environment:
-      CERC_REMOTE_DEBUG: "true"
-      CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
-      CERC_STATEDIFF_DB_NODE_ID: 1
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     image: cerc/go-ethereum:local
     entrypoint: /bin/sh
     command: -c "/opt/run-geth.sh"
+    env_file:
+      - ../config/mainnet-eth/geth.env
     volumes:
-      - mainnet_eth_geth_1_data:/root/ethdata
+      - mainnet_eth_geth_1_data:/data
       - mainnet_eth_config_data:/etc/mainnet-eth
       - ../config/mainnet-eth/scripts/run-geth.sh:/opt/run-geth.sh
     healthcheck:
@@ -26,6 +23,7 @@ services:
       start_period: 3s
     ports:
       - "8545"
+      - "8551"
       - "40000"
       - "6060"
 
@@ -39,16 +37,19 @@ services:
       retries: 10
       start_period: 30s
     environment:
-      EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
+      LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
+    env_file:
+      - ../config/mainnet-eth/lighthouse.env
     image: cerc/lighthouse:local
     entrypoint: /bin/sh
     command: -c "/opt/run-lighthouse.sh"
     volumes:
-      - mainnet_eth_lighthouse_1_data:/var/lighthouse-data-dir
+      - mainnet_eth_lighthouse_1_data:/data
       - mainnet_eth_config_data:/etc/mainnet-eth
       - ../config/mainnet-eth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh
     ports:
-      - "8001"
+      - "5052"
+      - "9000"
 
 volumes:
   mainnet_eth_config_data:

app/data/config/mainnet-eth-keycloak/keycloak.env

@@ -0,0 +1,17 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%api_key%"

app/data/config/mainnet-eth-keycloak/nginx/keycloak_proxy.conf

@@ -0,0 +1,72 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  localhost;
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+### geth
+    location ~ ^/eth/?([^/]*)$ {
+        set $apiKey $1;
+        if ($apiKey = '') {
+          set $apiKey $http_X_API_KEY;
+        }
+        auth_request /auth;
+        proxy_buffering off;
+        rewrite /.*$ / break;
+        proxy_pass  http://fixturenet-eth-geth-1:8545;
+    }
+
+## ipld-eth-server
+#    location ~ ^/ipld/eth/([^/]*)$ {
+#        set $apiKey $1;
+#        if ($apiKey = '') {
+#          set $apiKey $http_X_API_KEY;
+#        }
+#        auth_request /auth;
+#        auth_request_set $user_id $sent_http_x_user_id;
+#        proxy_buffering off;
+#        rewrite /.*$ / break;
+#        proxy_pass  http://ipld-eth-server:8081;
+#        proxy_set_header X-Original-Remote-Addr $remote_addr;
+#        proxy_set_header X-User-Id $user_id;
+#    }
+#
+#    location ~ ^/ipld/gql/([^/]*)$ {
+#        set $apiKey $1;
+#        if ($apiKey = '') {
+#          set $apiKey $http_X_API_KEY;
+#        }
+#        auth_request /auth;
+#        proxy_buffering off;
+#        rewrite /.*$ / break;
+#        proxy_pass  http://ipld-eth-server:8082;
+#    }
+
+## lighthouse
+    location /beacon/ {
+        set $apiKey $http_X_API_KEY;
+        auth_request /auth;
+        proxy_buffering off;
+        proxy_pass  http://fixturenet-eth-lighthouse-1:8001/;
+    }
+
+    location = /auth {
+        internal;
+        proxy_buffering off;
+        resolver 127.0.0.11 ipv6=off;
+        proxy_pass http://keycloak:8080/auth/realms/cerc/check?apiKey=$apiKey;
+        proxy_pass_request_body off;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Original-Remote-Addr $remote_addr;
+        proxy_set_header X-Original-Host $host;
+    }
+
+    location = /stub_status {
+        stub_status;
+    }
+}

app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+providers:
+  - name: dashboards
+    type: file
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true

app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml

@@ -0,0 +1,19 @@
+apiVersion: 1
+
+datasources:
+  - id: 1
+    uid: jZUuGao4k
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg
+    access: proxy
+    url: http://prometheus:9090
+    user: ""
+    database: ""
+    basicAuth: false
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+    readOnly: false

app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 15s
+
+scrape_configs:
+  # geth
+  - job_name: 'geth'
+    metrics_path: /debug/metrics/prometheus
+    scheme: http
+    static_configs:
+    - targets: ['mainnet-eth-geth-1:6060']
+
+  # keycloak
+  - job_name: 'keycloak'
+    scrape_interval: 5s
+    metrics_path: /auth/realms/cerc/metrics
+    scheme: http
+    static_configs:
+      - targets: ['keycloak:8080']

app/data/config/mainnet-eth/geth.env

@@ -0,0 +1,57 @@
+# Enable remote debugging using dlv
+CERC_REMOTE_DEBUG=false
+
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Simple toggle to choose either a 'full' node or an 'archive' node
+# (controls the values of --syncmode --gcmode --snapshot)
+CERC_GETH_MODE_QUICK_SET=archive
+
+# Optional custom node name.
+# GETH_NODE_NAME=""
+
+# Specify any other geth CLI options.
+GETH_OPTS=""
+
+# --cache
+GETH_CACHE=1024
+
+# --cache.database
+GETH_CACHE_DB=50
+
+# --cache.gc
+GETH_CACHE_GC=25
+
+# --cache.trie
+GETH_CACHE_TRIE=15
+j
+# --datadir
+GETH_DATADIR="/data"
+
+# --http.api
+GETH_HTTP_API="eth,web3,net"
+
+# --authrpc.jwtsecret
+GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret"
+
+# --maxpeers
+GETH_MAX_PEERS=100
+
+# --rpc.evmtimeout
+GETH_RPC_EVMTIMEOUT=0
+
+# --rpc.gascap
+GETH_RPC_GASCAP=0
+
+# --txlookuplimit
+GETH_TXLOOKUPLIMIT=0
+
+# --verbosity
+GETH_VERBOSITY=3
+
+# --log.vmodule
+GETH_VMODULE="rpc/*=4"
+
+# --ws.api
+GETH_WS_API="eth,web3,net"

app/data/config/mainnet-eth/lighthouse.env

@@ -0,0 +1,30 @@
+# Enable remote debugging using dlv
+CERC_REMOTE_DEBUG=false
+
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://beaconstate.ethstaker.cc
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000

app/data/config/mainnet-eth/scripts/run-geth.sh

@@ -3,10 +3,8 @@ if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
     set -x
 fi
 
-CERC_ETH_DATADIR=/root/ethdata
-
 START_CMD="geth"
-if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
+if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then
     START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
 fi
 
@@ -22,29 +20,44 @@ cleanup() {
     wait
     echo "Done"
 }
-
 trap 'cleanup' SIGINT SIGTERM
 
+MODE_FLAGS=""
+if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then
+  MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}"
+else
+  MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-snap} --snapshot=${GETH_SNAPSHOT:-true}"
+fi
+
 $START_CMD \
-    --datadir="${CERC_ETH_DATADIR}" \
-    --authrpc.addr="0.0.0.0" \
-    --authrpc.port 8551 \
-    --authrpc.vhosts="*" \
-    --authrpc.jwtsecret="/etc/mainnet-eth/jwtsecret" \
-    --ws \
-    --ws.addr="0.0.0.0" \
-    --ws.origins="*" \
-    --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \
-    --http.corsdomain="*" \
-    --gcmode full \
-    --txlookuplimit=0 \
-    --cache.preimages \
-    --syncmode=snap \
-    &
+  $MODE_FLAGS \
+  --datadir="${GETH_DATA}"\
+  --identity="${GETH_NODE_NAME}" \
+  --maxpeers=${GETH_MAX_PEERS} \
+  --cache=${GETH_CACHE} \
+  --cache.gc=${GETH_CACHE_GC} \
+  --cache.database=${GETH_CACHE_DB} \
+  --cache.trie=${GETH_CACHE_TRIE}
+  --authrpc.addr='0.0.0.0' \
+  --authrpc.vhosts='*'  \
+  --authrpc.jwtsecret="${GETH_JWTSECRET}" \
+  --http \
+  --http.addr='0.0.0.0' \
+  --http.api="${GETH_HTTP_API}" \
+  --http.vhosts='*' \
+  --metrics \
+  --metrics.addr='0.0.0.0' \
+  --ws \
+  --ws.addr='0.0.0.0' \
+  --ws.api="${GETH_WS_API}" \
+  --rpc.gascap=${GETH_RPC_GASCAP} \
+  --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \
+  --txlookuplimit=${GETH_TXLOOKUPLIMIT}
+  --verbosity=${GETH_VERBOSITY} \
+  --log.vmodule="${GETH_VMODULE}" \
+  ${GETH_OPTS} &
 
 geth_pid=$!
-
-
 wait $geth_pid
 
 if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then

app/data/config/mainnet-eth/scripts/run-lighthouse.sh

@@ -3,20 +3,22 @@ if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
     set -x
 fi
 
-DEBUG_LEVEL=${CERC_LIGHTHOUSE_DEBUG_LEVEL:-info}
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
 
-data_dir=/var/lighthouse-data-dir
-
-network_port=9001
-http_port=8001
-authrpc_port=8551
-
-exec lighthouse \
-  bn \
-  --debug-level $DEBUG_LEVEL \
-  --datadir $data_dir \
-  --network mainnet \
-  --execution-endpoint $EXECUTION_ENDPOINT \
-  --execution-jwt /etc/mainnet-eth/jwtsecret \
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
   --disable-deposit-contract-sync \
-  --checkpoint-sync-url https://beaconstate.ethstaker.cc
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS

app/data/stacks/mainnet-eth/stack.yml

@@ -10,6 +10,9 @@ containers:
   - cerc/lighthouse
   - cerc/lighthouse-cli
   - cerc/foundry
+  - cerc/keycloak
 pods:
   - mainnet-eth
+  - mainnet-eth-keycloak
+  - mainnet-eth-metrics
   - foundry