From ce0c7ee9ba68fb44d3bcbc9fa3301e2784e354fc Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 26 Jan 2023 13:53:02 -0600 Subject: [PATCH 1/2] Include all eth API operation metrics. (#162) --- .../etc/dashboards/fixturenet_dashboard.json | 407 ++++++++++++++++-- .../config/fixturenet-eth/fixturenet-eth.env | 2 + 2 files changed, 368 insertions(+), 41 deletions(-) diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json b/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json index 222c22b1..21932a24 100644 --- a/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json +++ b/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json @@ -326,8 +326,9 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "rate(rpc_duration_eth_call_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "legendFormat": "eth_call", + "expr": "rate(rpc_duration_eth_blockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_blockNumber", "range": true, "refId": "A" }, @@ -337,48 +338,12 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "expr": "rate(rpc_duration_eth_call_success_count{instance=\"${node}:6060\"}[$__rate_interval])", "hide": false, - "legendFormat": "eth_getBlocksByHash", + "legendFormat": "eth_call", "range": true, "refId": "B" }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBlocksByNumber", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getStorageAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "getStorageAt", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_statediff_writeStateDiffAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "statediff_writeStateDiffAt", - "range": true, - "refId": "E" - }, { "datasource": { "type": "prometheus", @@ -389,8 +354,356 @@ "hide": false, "legendFormat": "eth_chainId", "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_estimateGas_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_estimateGas", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_gasPrice_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_gasPrice", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBalance_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBalance", + "range": true, "refId": "F" }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockByHash", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockByNumber", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockReceipts_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockReceipts", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockTransactionCountByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockTransactionCountByHash", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockTransactionCountByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlockTransactionCountByNumber", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getCode_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getCode", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getFilterChanges_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getFilterChanges", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getFilterLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getFilterLogs", + "range": true, + "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getLogs", + "range": true, + "refId": "O" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getStorageAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getStorageAt", + "range": true, + "refId": "P" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionByBlockHashAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionByBlockHashAndIndex", + "range": true, + "refId": "Q" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionByBlockNumberAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionByBlockNumberAndIndex", + "range": true, + "refId": "R" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionByHash", + "range": true, + "refId": "S" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionCount_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionCount", + "range": true, + "refId": "T" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getTransactionReceipt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getTransactionReceipt", + "range": true, + "refId": "U" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getUncleCountByBlockHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getUncleCountByBlockHash", + "range": true, + "refId": "V" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getUncleCountByBlockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getUncleCountByBlockNumber", + "range": true, + "refId": "W" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_mining_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_mining_success", + "range": true, + "refId": "X" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_newBlockFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_newBlockFilter", + "range": true, + "refId": "Y" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_newFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_newFilter", + "range": true, + "refId": "Z" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_newPendingTransactionFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_newPendingTransactionFilter", + "range": true, + "refId": "AA" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_sendRawTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_sendRawTransaction", + "range": true, + "refId": "AB" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_signTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_signTransaction", + "range": true, + "refId": "AC" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_subscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_subscribe", + "range": true, + "refId": "AD" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_syncing_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_syncing", + "range": true, + "refId": "AE" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_unsubscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_unsubscribe", + "range": true, + "refId": "AF" + }, { "datasource": { "type": "prometheus", @@ -401,7 +714,19 @@ "hide": false, "legendFormat": "statediff_writeStateDiffFor", "range": true, - "refId": "G" + "refId": "AG" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_statediff_writeStateDiffAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "statediff_writeStateDiffAt", + "range": true, + "refId": "AH" } ], "title": "geth API Requests/sec", diff --git a/app/data/config/fixturenet-eth/fixturenet-eth.env b/app/data/config/fixturenet-eth/fixturenet-eth.env index 20a9fc05..1c214b56 100644 --- a/app/data/config/fixturenet-eth/fixturenet-eth.env +++ b/app/data/config/fixturenet-eth/fixturenet-eth.env @@ -19,3 +19,5 @@ CERC_STATEDIFF_DB_USER="vdbm" CERC_STATEDIFF_DB_PASSWORD="password" CERC_STATEDIFF_DB_GOOSE_MIN_VER=23 CERC_STATEDIFF_DB_LOG_STATEMENTS="false" + +CERC_GETH_VMODULE="statediff/*=5,rpc/*=5" -- 2.45.2 From e4229e4cebb4532ed6f6ae1102ba16ab91ab8056 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Thu, 26 Jan 2023 16:47:06 -0700 Subject: [PATCH 2/2] Fixes to fixturenet-eth --- app/data/compose/docker-compose-fixturenet-eth.yml | 1 + .../container-build/cerc-fixturenet-eth-geth/run-el.sh | 8 +++++++- .../cerc-fixturenet-eth-lighthouse/Dockerfile | 4 ++++ .../scripts/status-internal.sh | 10 ++++++++++ .../cerc-fixturenet-eth-lighthouse/scripts/status.sh | 8 ++++++-- app/data/stacks/fixturenet-eth/README.md | 6 +++--- app/data/stacks/fixturenet-eth/stack.yml | 3 ++- app/deploy_system.py | 8 +++++++- 8 files changed, 40 insertions(+), 8 deletions(-) create mode 100755 app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml index 517b4832..978b983f 100644 --- a/app/data/compose/docker-compose-fixturenet-eth.yml +++ b/app/data/compose/docker-compose-fixturenet-eth.yml @@ -20,6 +20,7 @@ services: CERC_REMOTE_DEBUG: "true" CERC_RUN_STATEDIFF: "detect" CERC_STATEDIFF_DB_NODE_ID: 1 + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} env_file: - ../config/fixturenet-eth/fixturenet-eth.env image: cerc/fixturenet-eth-geth:local diff --git a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh index 93e34988..3aa0eaf3 100755 --- a/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh +++ b/app/data/container-build/cerc-fixturenet-eth-geth/run-el.sh @@ -1,5 +1,9 @@ #!/bin/bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2` NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'` NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'` @@ -28,7 +32,9 @@ else echo -n "$JWT" > /opt/testnet/build/el/jwtsecret if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then - if [ -n "$(dig $CERC_STATEDIFF_DB_HOST +short)" ]; then + dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short) + dig_status_code=$? + if [[ $dig_status_code = 0 && -n $dig_result ]]; then echo "Statediff DB at $CERC_STATEDIFF_DB_HOST" CERC_RUN_STATEDIFF="true" else diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile b/app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile index 6e8add4a..2295262d 100644 --- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile +++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile @@ -27,4 +27,8 @@ RUN cd /opt/testnet && make genesis-cl # Work around some bugs in lcli where the default path is always used. RUN mkdir -p /root/.lighthouse && cd /root/.lighthouse && ln -s /opt/testnet/build/cl/testnet +RUN mkdir -p /scripts +COPY scripts/status-internal.sh /scripts +COPY scripts/status.sh /scripts + ENTRYPOINT ["/opt/testnet/run.sh"] diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh new file mode 100755 index 00000000..52c61a39 --- /dev/null +++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status-internal.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Wrapper to facilitate using status.sh inside the container +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +export LIGHTHOUSE_BASE_URL="http://fixturenet-eth-lighthouse-1:8001" +export GETH_BASE_URL="http://fixturenet-eth-geth-1-1:8545" +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +$SCRIPT_DIR/status.sh diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh index a6999e3f..faca9a49 100755 --- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh +++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh @@ -1,5 +1,7 @@ -#!/bin/bash - +#!/usr/bin/env bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi STATUSES=("geth to generate DAG" "beacon phase0" "beacon altair" "beacon bellatrix pre-merge" "beacon bellatrix merge") STATUS=0 @@ -7,6 +9,8 @@ STATUS=0 LIGHTHOUSE_BASE_URL=${LIGHTHOUSE_BASE_URL} GETH_BASE_URL=${GETH_BASE_URL} +# TODO: Docker commands below should be replaced by some interface into stack orchestrator +# or some execution environment-neutral mechanism. if [ -z "$LIGHTHOUSE_BASE_URL" ]; then LIGHTHOUSE_CONTAINER=`docker ps -q -f "name=fixturenet-eth-lighthouse-1-1"` LIGHTHOUSE_PORT=`docker port $LIGHTHOUSE_CONTAINER 8001 | cut -d':' -f2` diff --git a/app/data/stacks/fixturenet-eth/README.md b/app/data/stacks/fixturenet-eth/README.md index bf77c25c..a164f078 100644 --- a/app/data/stacks/fixturenet-eth/README.md +++ b/app/data/stacks/fixturenet-eth/README.md @@ -4,12 +4,12 @@ Instructions for deploying a local a geth + lighthouse blockchain "fixturenet" f ## Clone required repositories ``` -$ laconic-so setup-repositories --include cerc-io/go-ethereum +$ laconic-so --stack fixturenet-eth setup-repositories ``` ## Build the fixturenet-eth containers ``` -$ laconic-so build-containers --include cerc/go-ethereum,cerc/lighthouse,cerc/fixturenet-eth-geth,cerc/fixturenet-eth-lighthouse +$ laconic-so --stack fixturenet-eth build-containers ``` This should create several container images in the local image registry: @@ -20,7 +20,7 @@ This should create several container images in the local image registry: ## Deploy the stack ``` -$ laconic-so deploy-system --include fixturenet-eth up +$ laconic-so --stack fixturenet-eth deploy-system up ``` ## Check status diff --git a/app/data/stacks/fixturenet-eth/stack.yml b/app/data/stacks/fixturenet-eth/stack.yml index 3c23e841..500ae186 100644 --- a/app/data/stacks/fixturenet-eth/stack.yml +++ b/app/data/stacks/fixturenet-eth/stack.yml @@ -3,10 +3,11 @@ name: fixturenet-eth decription: "Ethereum Fixturenet" repos: - cerc-io/go-ethereum + - cerc-io/tx-spammer containers: - cerc/go-ethereum - cerc/lighthouse - cerc/fixturenet-eth-geth - cerc/fixturenet-eth-lighthouse pods: - - fixturenet-eth \ No newline at end of file + - fixturenet-eth diff --git a/app/deploy_system.py b/app/deploy_system.py index 904c7f63..7c2e5b7e 100644 --- a/app/deploy_system.py +++ b/app/deploy_system.py @@ -37,6 +37,7 @@ def command(ctx, include, exclude, cluster, command, extra_args): # TODO: implement option exclusion and command value constraint lost with the move from argparse to click + debug = ctx.obj.debug quiet = ctx.obj.quiet verbose = ctx.obj.verbose dry_run = ctx.obj.dry_run @@ -91,6 +92,8 @@ def command(ctx, include, exclude, cluster, command, extra_args): if not dry_run: if command == "up": + if debug: + os.environ["CERC_SCRIPT_DEBUG"] = "true" if verbose: print(f"Running compose up for extra_args: {extra_args_list}") docker.compose.up(detach=True, services=extra_args_list) @@ -104,9 +107,12 @@ def command(ctx, include, exclude, cluster, command, extra_args): sys.exit(1) service_name = extra_args_list[0] command_to_exec = extra_args_list[1:] + container_exec_env = { + "CERC_SCRIPT_DEBUG": "true" + } if debug else None if verbose: print(f"Running compose exec {service_name} {command_to_exec}") - docker.compose.execute(service_name, command_to_exec) + docker.compose.execute(service_name, command_to_exec, envs=container_exec_env) elif command == "port": if extra_args_list is None or len(extra_args_list) < 2: print("Usage: port ") -- 2.45.2