From 16afea988b66d4d5018b547925f22279262c1443 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 24 Jan 2023 16:49:26 -0600 Subject: [PATCH 1/6] Integrate grafana/prometheus with fixturenet-eth. --- .../docker-compose-fixturenet-eth-metrics.yml | 21 ++++++++ .../compose/docker-compose-fixturenet-eth.yml | 6 +-- .../docker-compose-ipld-eth-server.yml | 18 +++++-- app/data/compose/docker-compose-keycloak.yml | 8 ++++ .../docker-compose-prometheus-grafana.yml | 21 -------- .../config/keycloak/nginx/keycloak_proxy.conf | 48 +++++++++---------- .../scripts/status.sh | 6 ++- app/data/pod-list.txt | 2 +- 8 files changed, 74 insertions(+), 56 deletions(-) create mode 100644 app/data/compose/docker-compose-fixturenet-eth-metrics.yml delete mode 100644 app/data/compose/docker-compose-prometheus-grafana.yml diff --git a/app/data/compose/docker-compose-fixturenet-eth-metrics.yml b/app/data/compose/docker-compose-fixturenet-eth-metrics.yml new file mode 100644 index 00000000..4d3ccdf5 --- /dev/null +++ b/app/data/compose/docker-compose-fixturenet-eth-metrics.yml @@ -0,0 +1,21 @@ +version: "3.2" +services: + prometheus: + restart: always + image: prom/prometheus + depends_on: + fixturenet-eth-geth-1: + condition: service_healthy + volumes: + - ../config/fixturenet-eth-metrics/prometheus/etc:/etc/prometheus + ports: + - "9090" + grafana: + restart: always + image: grafana/grafana + volumes: + - ../config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources:/etc/grafana/provisioning/datasources + - ../config/fixturenet-eth-metrics/grafana/etc/dashboards:/etc/grafana/dashboards + ports: + - "3000" diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml index 517b4832..a80e68cc 100644 --- a/app/data/compose/docker-compose-fixturenet-eth.yml +++ b/app/data/compose/docker-compose-fixturenet-eth.yml @@ -32,9 +32,9 @@ services: depends_on: - fixturenet-eth-bootnode-geth ports: - - "8545" - - "40000" - - "6060" + - "8545:8545" + - "40000:40000" + - "6060:6060" fixturenet-eth-geth-2: hostname: fixturenet-eth-geth-2 diff --git a/app/data/compose/docker-compose-ipld-eth-server.yml b/app/data/compose/docker-compose-ipld-eth-server.yml index ee9c8885..9578fe97 100644 --- a/app/data/compose/docker-compose-ipld-eth-server.yml +++ b/app/data/compose/docker-compose-ipld-eth-server.yml @@ -20,16 +20,24 @@ services: DATABASE_USER: "vdbm" DATABASE_PASSWORD: "password" ETH_CHAIN_ID: 99 - ETH_FORWARD_ETH_CALLS: $eth_forward_eth_calls - ETH_PROXY_ON_ERROR: $eth_proxy_on_error - ETH_HTTP_PATH: $eth_http_path + ETH_FORWARD_ETH_CALLS: "false" + ETH_FORWARD_GET_STORAGE_AT: "false" + ETH_PROXY_ON_ERROR: "true" + ETH_HTTP_PATH: "fixturenet-eth-geth-1:8545" + METRICS: "true" + PROM_HTTP: "true" + PROM_HTTP_ADDR: "0.0.0.0" + PROM_HTTP_PORT: "8090" + LOGRUS_LEVEL: "debug" + CERC_REMOTE_DEBUG: "true" volumes: - type: bind source: ../config/ipld-eth-server/chain.json target: /tmp/chain.json ports: - - "127.0.0.1:8081:8081" - - "127.0.0.1:8082:8082" + - "8081" + - "8082" + - "8090" healthcheck: test: ["CMD", "nc", "-v", "localhost", "8081"] interval: 20s diff --git a/app/data/compose/docker-compose-keycloak.yml b/app/data/compose/docker-compose-keycloak.yml index 294387cd..1586cdb7 100644 --- a/app/data/compose/docker-compose-keycloak.yml +++ b/app/data/compose/docker-compose-keycloak.yml @@ -29,9 +29,17 @@ services: condition: service_healthy keycloak-nginx: image: nginx:1.23-alpine + restart: always volumes: - ../config/keycloak/nginx:/etc/nginx/conf.d ports: - 80 depends_on: - keycloak + keycloak-nginx-prometheus-exporter: + image: nginx/nginx-prometheus-exporter + restart: always + environment: + - SCRAPE_URI=http://keycloak-nginx:80/stub_status + depends_on: + - keycloak-nginx diff --git a/app/data/compose/docker-compose-prometheus-grafana.yml b/app/data/compose/docker-compose-prometheus-grafana.yml deleted file mode 100644 index 494748bc..00000000 --- a/app/data/compose/docker-compose-prometheus-grafana.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: "3.2" -services: - # If you want prometheus to work, you must update the following file in the ops repo locally. - # localhost:6060 --> go-ethereum:6060 - prometheus: - restart: always - user: "987" - image: prom/prometheus - volumes: - - ${cerc_ops}/metrics/etc:/etc/prometheus - - ./prometheus-data:/prometheus - ports: - - "127.0.0.1:9090:9090" - grafana: - restart: always - user: "472" - image: grafana/grafana - volumes: - - ./grafana-data:/var/lib/grafana - ports: - - "127.0.0.1:3000:3000" diff --git a/app/data/config/keycloak/nginx/keycloak_proxy.conf b/app/data/config/keycloak/nginx/keycloak_proxy.conf index f06c6700..dd69680b 100644 --- a/app/data/config/keycloak/nginx/keycloak_proxy.conf +++ b/app/data/config/keycloak/nginx/keycloak_proxy.conf @@ -21,19 +21,19 @@ server { } ## ipld-eth-server - location ~ ^/ipld/eth/([^/]*)$ { - set $apiKey $1; - if ($apiKey = '') { - set $apiKey $http_X_API_KEY; - } - auth_request /auth; - auth_request_set $user_id $sent_http_x_user_id; - proxy_buffering off; - rewrite /.*$ / break; - proxy_pass http://ipld-eth-server:8081; - proxy_set_header X-Original-Remote-Addr $remote_addr; - proxy_set_header X-User-Id $user_id; - } +# location ~ ^/ipld/eth/([^/]*)$ { +# set $apiKey $1; +# if ($apiKey = '') { +# set $apiKey $http_X_API_KEY; +# } +# auth_request /auth; +# auth_request_set $user_id $sent_http_x_user_id; +# proxy_buffering off; +# rewrite /.*$ / break; +# proxy_pass http://ipld-eth-server:8081; +# proxy_set_header X-Original-Remote-Addr $remote_addr; +# proxy_set_header X-User-Id $user_id; +# } # # location ~ ^/ipld/gql/([^/]*)$ { # set $apiKey $1; @@ -45,14 +45,14 @@ server { # rewrite /.*$ / break; # proxy_pass http://ipld-eth-server:8082; # } -# -### lighthouse -# location /beacon/ { -# set $apiKey $http_X_API_KEY; -# auth_request /auth; -# proxy_buffering off; -# proxy_pass http://fixturenet-eth-lighthouse-1:8001/; -# } + +## lighthouse + location /beacon/ { + set $apiKey $http_X_API_KEY; + auth_request /auth; + proxy_buffering off; + proxy_pass http://fixturenet-eth-lighthouse-1:8001/; + } location = /auth { internal; @@ -66,7 +66,7 @@ server { proxy_set_header X-Original-Host $host; } -# location = /basic_status { -# stub_status; -# } + location = /stub_status { + stub_status; + } } diff --git a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh index e02fc14b..a6999e3f 100755 --- a/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh +++ b/app/data/container-build/cerc-fixturenet-eth-lighthouse/scripts/status.sh @@ -8,12 +8,14 @@ LIGHTHOUSE_BASE_URL=${LIGHTHOUSE_BASE_URL} GETH_BASE_URL=${GETH_BASE_URL} if [ -z "$LIGHTHOUSE_BASE_URL" ]; then - LIGHTHOUSE_PORT=`docker ps -f "name=fixturenet-eth-lighthouse-1-1" --format "{{.Ports}}" | head -1 | cut -d':' -f2 | cut -d'-' -f1` + LIGHTHOUSE_CONTAINER=`docker ps -q -f "name=fixturenet-eth-lighthouse-1-1"` + LIGHTHOUSE_PORT=`docker port $LIGHTHOUSE_CONTAINER 8001 | cut -d':' -f2` LIGHTHOUSE_BASE_URL="http://localhost:${LIGHTHOUSE_PORT}" fi if [ -z "$GETH_BASE_URL" ]; then - GETH_PORT=`docker ps -f "name=fixturenet-eth-geth-1-1" --format "{{.Ports}}" | head -1 | cut -d':' -f2 | cut -d'-' -f1` + GETH_CONTAINER=`docker ps -q -f "name=fixturenet-eth-geth-1-1"` + GETH_PORT=`docker port $GETH_CONTAINER 8545 | cut -d':' -f2` GETH_BASE_URL="http://localhost:${GETH_PORT}" fi diff --git a/app/data/pod-list.txt b/app/data/pod-list.txt index 661a04e4..64dd0412 100644 --- a/app/data/pod-list.txt +++ b/app/data/pod-list.txt @@ -6,10 +6,10 @@ ipld-eth-beacon-db ipld-eth-beacon-indexer ipld-eth-server lighthouse -prometheus-grafana laconicd fixturenet-laconicd fixturenet-eth +fixturenet-eth-metrics watcher-mobymask watcher-erc20 watcher-erc721 -- 2.45.2 From 80baaba0e7cc0723a4e3d4738ca8b76fb2d3424d Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 24 Jan 2023 16:49:49 -0600 Subject: [PATCH 2/6] Integrate grafana/prometheus with fixturenet-eth. --- .../etc/dashboards/fixturenet_dashboard.json | 1089 +++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/datasources/prometheus.yml | 19 + .../prometheus/etc/prometheus.yml | 34 + 4 files changed, 1151 insertions(+) create mode 100644 app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json create mode 100644 app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml create mode 100644 app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml create mode 100644 app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json b/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json new file mode 100644 index 00000000..222c22b1 --- /dev/null +++ b/app/data/config/fixturenet-eth-metrics/grafana/etc/dashboards/fixturenet_dashboard.json @@ -0,0 +1,1089 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Row title", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(nginx_http_requests_total{instance=\"keycloak-nginx-prometheus-exporter:9113\"}[$__rate_interval])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "nginx Requests/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "builder", + "expr": "rate(ipld_eth_server_http_count{instance=\"ipld-eth-server:8090\"}[$__rate_interval])", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "ipld-eth-server Requests/sec", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "{instance=\"ipld-eth-server:8090\", job=\"ipld-eth-server\"}": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_call_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "legendFormat": "eth_call", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlocksByHash", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getBlockByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_getBlocksByNumber", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_getStorageAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "getStorageAt", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_statediff_writeStateDiffAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "statediff_writeStateDiffAt", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_eth_chainId_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "eth_chainId", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(rpc_duration_statediff_writeStateDiffFor_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "hide": false, + "legendFormat": "statediff_writeStateDiffFor", + "range": true, + "refId": "G" + } + ], + "title": "geth API Requests/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(keycloak_logins{instance=\"keycloak:8080\"}[90s]) * 60", + "format": "time_series", + "instant": false, + "interval": "2m", + "legendFormat": "{{client_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Keycloak Logins/min", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": true, + "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": true, + "13761bf2-2c0e-42fd-acc8-09a677ab179e": true, + "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": true, + "9c41e289-92f0-456f-bd65-8f73024a9e45": true, + "c9718dd1-06d6-4f98-a821-9adbeb4b663b": true + }, + "indexByName": {}, + "renameByName": { + "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": "", + "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": "rickm", + "13761bf2-2c0e-42fd-acc8-09a677ab179e": "telackey", + "2753f8be-0036-49ba-a53a-4963573fc15c": "cerc-io", + "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": "", + "9c41e289-92f0-456f-bd65-8f73024a9e45": "", + "c9718dd1-06d6-4f98-a821-9adbeb4b663b": "dboreham", + "f5083aa7-5b5d-4164-b189-d7a559c4aad0": "infura0" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "chain_head_block{instance=\"${node}:6060\"}", + "hide": false, + "legendFormat": "geth", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "statediff_service_last_statediff_height{instance=\"${node}:6060\"}", + "legendFormat": "statediff", + "range": true, + "refId": "A" + } + ], + "title": "Block Positions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Distance" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 17 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "clamp_min(chain_head_block{instance=\"${node}:6060\"} - on() statediff_service_last_statediff_height{instance=\"${node}:6060\"}, 0)", + "interval": "1m", + "legendFormat": "Distance", + "range": true, + "refId": "A" + } + ], + "title": "Block Distance (geth to statediff)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(statediff_service_succeeded{instance=\"${node}:6060\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "1m", + "legendFormat": "Succeeded", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "increase(statediff_service_failed{instance=\"${node}:6060\"}[1m])", + "hide": false, + "interval": "1m", + "legendFormat": "Failed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "statediff_service_underway{instance=\"${node}:6060\"}", + "hide": false, + "interval": "1s", + "legendFormat": "Underway", + "range": true, + "refId": "C" + } + ], + "title": "Statediff Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(statediff_service_total_processing_time{instance=\"${node}:6060\"}[$__rate_interval])", + "instant": false, + "interval": "1m", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Statediff Duty Cycle", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(statediff_service_api_requests{instance=\"${node}:6060\"}[$__rate_interval])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Statediff API Requests/sec", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "fixturenet-eth-geth-1", + "value": "fixturenet-eth-geth-1" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "node", + "options": [ + { + "selected": true, + "text": "fixturenet-eth-geth-1", + "value": "fixturenet-eth-geth-1" + }, + { + "selected": false, + "text": "fixturenet-eth-geth-2", + "value": "fixturenet-eth-geth-2" + } + ], + "query": "fixturenet-eth-geth-1,fixturenet-eth-geth-2", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "cerc-io overview", + "uid": "nT9VeZoVk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml b/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..255071b7 --- /dev/null +++ b/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + updateIntervalSeconds: 10 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: true \ No newline at end of file diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml b/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..f5181ebd --- /dev/null +++ b/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml @@ -0,0 +1,19 @@ +apiVersion: 1 + +datasources: + - id: 1 + uid: jZUuGao4k + orgId: 1 + name: Prometheus + type: prometheus + typeName: Prometheus + typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg + access: proxy + url: http://prometheus:9090 + user: "" + database: "" + basicAuth: false + isDefault: true + jsonData: + httpMethod: POST + readOnly: false diff --git a/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml b/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml new file mode 100644 index 00000000..7deadc6e --- /dev/null +++ b/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml @@ -0,0 +1,34 @@ +global: + scrape_interval: 5s + evaluation_interval: 15s + +scrape_configs: + # ipld-eth-server + - job_name: 'ipld-eth-server' + metrics_path: /metrics + scrape_interval: 5s + static_configs: + - targets: ['ipld-eth-server:8090'] + + # geth + - job_name: 'geth' + metrics_path: /debug/metrics/prometheus + scheme: http + static_configs: + - targets: ['fixturenet-eth-geth-1:6060'] + + # nginx + - job_name: 'nginx' + scrape_interval: 5s + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['keycloak-nginx-prometheus-exporter:9113'] + + # keycloak + - job_name: 'keycloak' + scrape_interval: 5s + metrics_path: /auth/realms/cerc/metrics + scheme: http + static_configs: + - targets: ['keycloak:8080'] \ No newline at end of file -- 2.45.2 From 1b9de0c518a9bed25c738d52097d1ad27619ace8 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 24 Jan 2023 16:56:06 -0600 Subject: [PATCH 3/6] Revert port change --- app/data/compose/docker-compose-fixturenet-eth.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/data/compose/docker-compose-fixturenet-eth.yml b/app/data/compose/docker-compose-fixturenet-eth.yml index a80e68cc..517b4832 100644 --- a/app/data/compose/docker-compose-fixturenet-eth.yml +++ b/app/data/compose/docker-compose-fixturenet-eth.yml @@ -32,9 +32,9 @@ services: depends_on: - fixturenet-eth-bootnode-geth ports: - - "8545:8545" - - "40000:40000" - - "6060:6060" + - "8545" + - "40000" + - "6060" fixturenet-eth-geth-2: hostname: fixturenet-eth-geth-2 -- 2.45.2 From 61cc1f9e5865ab986a5ca55476f48368398e7dae Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 24 Jan 2023 16:57:05 -0600 Subject: [PATCH 4/6] EOL --- .../grafana/etc/provisioning/dashboards/dashboards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml b/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml index 255071b7..4705688e 100644 --- a/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml +++ b/app/data/config/fixturenet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml @@ -6,4 +6,4 @@ providers: updateIntervalSeconds: 10 options: path: /etc/grafana/dashboards - foldersFromFilesStructure: true \ No newline at end of file + foldersFromFilesStructure: true -- 2.45.2 From 2d684572baa46123b3ef7b116710f85c55004572 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 24 Jan 2023 16:57:46 -0600 Subject: [PATCH 5/6] EOL --- .../config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml b/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml index 7deadc6e..5718d4ef 100644 --- a/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml +++ b/app/data/config/fixturenet-eth-metrics/prometheus/etc/prometheus.yml @@ -31,4 +31,4 @@ scrape_configs: metrics_path: /auth/realms/cerc/metrics scheme: http static_configs: - - targets: ['keycloak:8080'] \ No newline at end of file + - targets: ['keycloak:8080'] -- 2.45.2 From fe6a17de2c4af6a436fa93592b964ef6b0fca5a6 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 25 Jan 2023 11:43:24 -0600 Subject: [PATCH 6/6] Simplify ipld-eth-server config. --- app/data/compose/docker-compose-ipld-eth-server.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/data/compose/docker-compose-ipld-eth-server.yml b/app/data/compose/docker-compose-ipld-eth-server.yml index 9578fe97..e02f73ca 100644 --- a/app/data/compose/docker-compose-ipld-eth-server.yml +++ b/app/data/compose/docker-compose-ipld-eth-server.yml @@ -22,8 +22,7 @@ services: ETH_CHAIN_ID: 99 ETH_FORWARD_ETH_CALLS: "false" ETH_FORWARD_GET_STORAGE_AT: "false" - ETH_PROXY_ON_ERROR: "true" - ETH_HTTP_PATH: "fixturenet-eth-geth-1:8545" + ETH_PROXY_ON_ERROR: "false" METRICS: "true" PROM_HTTP: "true" PROM_HTTP_ADDR: "0.0.0.0" @@ -38,6 +37,7 @@ services: - "8081" - "8082" - "8090" + - "40000" healthcheck: test: ["CMD", "nc", "-v", "localhost", "8081"] interval: 20s -- 2.45.2