From 0b2718a4affbfb7de9708c3be6fc013c56982312 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 11 Oct 2019 17:13:16 -0700 Subject: [PATCH 1/4] Add chain stats tool Simple chain stats tool for graphing the chain using influxdb and grafana. License: MIT Signed-off-by: Jakub Sztandera --- Makefile | 5 + go.mod | 1 + go.sum | 2 + tools/stats/.gitignore | 1 + tools/stats/README.md | 39 + tools/stats/chain.dashboard.json | 2118 ++++++++++++++++++++++++++++++ tools/stats/docker-compose.yml | 26 + tools/stats/env.stats | 3 + tools/stats/main.go | 118 ++ tools/stats/metrics.go | 241 ++++ tools/stats/rpc.go | 155 +++ tools/stats/setup.bash | 29 + 12 files changed, 2738 insertions(+) create mode 100644 tools/stats/.gitignore create mode 100644 tools/stats/README.md create mode 100644 tools/stats/chain.dashboard.json create mode 100644 tools/stats/docker-compose.yml create mode 100644 tools/stats/env.stats create mode 100644 tools/stats/main.go create mode 100644 tools/stats/metrics.go create mode 100644 tools/stats/rpc.go create mode 100755 tools/stats/setup.bash diff --git a/Makefile b/Makefile index 399fe6796..cb550bf57 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,11 @@ fountain: go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain .PHONY: fountain +stats: + rm -f stats + go build -o stats ./tools/stats +.PHONY: stats + clean: rm -rf $(CLEAN) -$(MAKE) -C $(BLS_PATH) clean diff --git a/go.mod b/go.mod index 8df04c5d1..48098edda 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-ole/go-ole v1.2.4 // indirect github.com/gorilla/websocket v1.4.0 github.com/hashicorp/golang-lru v0.5.3 + github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c diff --git a/go.sum b/go.sum index 7902d2270..be3b0270d 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e h1:txQltCyjXAqVVSZDArPEhUTg35hKwVIuXwtQo7eAMNQ= +github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= diff --git a/tools/stats/.gitignore b/tools/stats/.gitignore new file mode 100644 index 000000000..07a22a1a0 --- /dev/null +++ b/tools/stats/.gitignore @@ -0,0 +1 @@ +env.stats diff --git a/tools/stats/README.md b/tools/stats/README.md new file mode 100644 index 000000000..a75671220 --- /dev/null +++ b/tools/stats/README.md @@ -0,0 +1,39 @@ +# Stats + +Stats is a small tool to push chain information into influxdb + +## Setup + +Influx configuration can be configured through env variables. + +``` +INFLUX_ADDR="http://localhost:8086" +INFLUX_USER="" +INFLUX_PASS="" +``` + +## Usage + +Stats will be default look in `~/.lotus` to connect to a running daemon and resume collecting stats from last record block height. + +For other usage see `./stats --help` + +``` +go build -o stats *.go +. env.stats && ./stats +``` + + +## Development + +Start grafana and influxdb containers and import the dashboard to grafana. +The url of the imported dashboard will be returned. + +If the script doesn't work, you can manually setup the datasource and import the dashboard. + +``` +docker-compose up -d +./setup.bash +``` + +The default username and password for grafana are both `admin`. diff --git a/tools/stats/chain.dashboard.json b/tools/stats/chain.dashboard.json new file mode 100644 index 000000000..27ffe374d --- /dev/null +++ b/tools/stats/chain.dashboard.json @@ -0,0 +1,2118 @@ +{ + "__inputs": [ + { + "name": "DS_INFLUXDB", + "label": "InfluxDB", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.5.0-pre" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorPostfix": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.height", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Block Height", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.blocktime", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "tags": [] + } + ], + "thresholds": "30,90", + "timeFrom": null, + "timeShift": null, + "title": "Last Blocktime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.power", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Total Power", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.message_gasprice", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Avg Gas Price", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.message_size", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Avg Message Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.blockheader_size", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "1024,2048", + "timeFrom": null, + "timeShift": null, + "title": "Avg Blockheader Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "decimals": 0, + "format": "dateTimeFromNow", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 16, + "interval": "", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.blocktime", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "*1000" + ], + "type": "math" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Head Updated", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.block_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Blocks In Tipset", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "FIL", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.pledge_collateral", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Pledge Collateral", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [], + "measurement": "chain.blocktime", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT difference(mean(\"value\")) FROM \"chain.blocktime\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Avg Blocktime", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_INFLUXDB}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "pluginVersion": "6.4.2", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": 0 + }, + "tableColumn": "", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "chain.message_count", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM (SELECT \"value\" FROM \"chain.message_count\" GROUP BY \"height\") WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Avg Messages in Tipset", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "chain.height.difference", + "yaxis": 2 + }, + { + "alias": "Null Blocks", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Block Time", + "groupBy": [], + "measurement": "chain.blocktime", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT difference(\"value\") FROM \"chain.blocktime\" WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "tags": [] + }, + { + "alias": "Null Blocks", + "groupBy": [], + "measurement": "chain.height", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "difference" + }, + { + "params": [ + "-1" + ], + "type": "math" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Tipsets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time between tipsets", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": "Number of Null blocks", + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "${DS_INFLUXDB}", + "fontSize": "100%", + "gridPos": { + "h": 16, + "w": 4, + "x": 0, + "y": 10 + }, + "id": 28, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "power", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [], + "measurement": "chain.miner_power", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT last(\"value\") as \"power\" FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY \"miner\" SLIMIT 20", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top Power Table", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 4, + "y": 10 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": true, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_miner", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "miner" + ], + "type": "tag" + }, + { + "params": [ + "previous" + ], + "type": "fill" + } + ], + "limit": "", + "measurement": "chain.miner_power", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time($__interval), \"miner\" fill(previous)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "slimit": "20", + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top Miner Power", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "bytes", + "label": "Power", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "${DS_INFLUXDB}", + "fontSize": "100%", + "gridPos": { + "h": 16, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 18, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Height", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "mappingType": 1, + "pattern": "chain.height", + "preserveFormat": false, + "sanitize": false, + "type": "string" + }, + { + "alias": "Tipset", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "chain.height.tipset", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [], + "measurement": "chain.height", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\", \"tipset\" FROM \"chain.height\" WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chain Table", + "transform": "timeseries_to_columns", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 4, + "y": 18 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "previous" + ], + "type": "fill" + } + ], + "measurement": "chain.pledge_collateral", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pledge Collateral", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "FIL", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Adr $tag_actor | Md $tag_method", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "actor" + ], + "type": "tag" + }, + { + "params": [ + "method" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.message_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Actor Messages Method", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Adr $tag_actor | Md $tag_method | Ex $tag_exitcode", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "actor" + ], + "type": "tag" + }, + { + "params": [ + "method" + ], + "type": "tag" + }, + { + "params": [ + "exitcode" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "chain.message_count", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Actor Messages Method With Exitcode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 20, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Chain", + "uid": "z6FtI92Zz", + "version": 4 +} \ No newline at end of file diff --git a/tools/stats/docker-compose.yml b/tools/stats/docker-compose.yml new file mode 100644 index 000000000..03d573b94 --- /dev/null +++ b/tools/stats/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3' + +services: + influxdb: + image: influxdb:latest + container_name: influxdb + environment: + - INFLUXDB_DB=lotus + ports: + - "8086:8086" + volumes: + - influxdb:/var/lib/influxdb + + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3000:3000" + links: + - influxdb + volumes: + - grafana:/var/lib/grafana + +volumes: + influxdb: + grafana: diff --git a/tools/stats/env.stats b/tools/stats/env.stats new file mode 100644 index 000000000..a76e7554a --- /dev/null +++ b/tools/stats/env.stats @@ -0,0 +1,3 @@ +export INFLUX_ADDR="http://localhost:8086" +export INFLUX_USER="" +export INFLUX_PASS="" diff --git a/tools/stats/main.go b/tools/stats/main.go new file mode 100644 index 000000000..f77ba941a --- /dev/null +++ b/tools/stats/main.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "time" +) + +const ( + INFLUX_ADDR = "INFLUX_ADDR" + INFLUX_USER = "INFLUX_USER" + INFLUX_PASS = "INFLUX_PASS" +) + +func main() { + var repo string = "~/.lotus" + var database string = "lotus" + var reset bool = false + var height int64 = 0 + + flag.StringVar(&repo, "repo", repo, "lotus repo path") + flag.StringVar(&database, "database", database, "influx database") + flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)") + flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering") + + flag.Parse() + + influxAddr := os.Getenv(INFLUX_ADDR) + influxUser := os.Getenv(INFLUX_USER) + influxPass := os.Getenv(INFLUX_PASS) + + ctx := context.Background() + + influx, err := InfluxClient(influxAddr, influxUser, influxPass) + if err != nil { + log.Fatal(err) + } + + if reset { + if err := ResetDatabase(influx, database); err != nil { + log.Fatal(err) + } + } + + if !reset && height == 0 { + h, err := GetLastRecordedHeight(influx, database) + if err != nil { + log.Print(err) + } + + height = h + } + + api, closer, err := GetFullNodeAPI(repo) + if err != nil { + log.Fatal(err) + } + defer closer() + + if err := WaitForSyncComplete(ctx, api); err != nil { + log.Fatal(err) + } + + tipsetsCh, err := GetTips(ctx, api, uint64(height)) + if err != nil { + log.Fatal(err) + } + + wq := NewInfluxWriteQueue(ctx, influx) + defer wq.Close() + + for tipset := range tipsetsCh { + pl := NewPointList() + height := tipset.Height() + + if err := RecordTipsetPoints(ctx, api, pl, tipset); err != nil { + log.Printf("Failed to record tipset at height %d: %w", height, err) + continue + } + + if err := RecordTipsetMessagesPoints(ctx, api, pl, tipset); err != nil { + log.Printf("Failed to record messages at height %d: %w", height, err) + continue + } + + if err := RecordTipsetStatePoints(ctx, api, pl, tipset); err != nil { + log.Printf("Failed to record state at height %d: %w", height, err) + continue + } + + // Instead of having to pass around a bunch of generic stuff we want for each point + // we will just add them at the end. + + tsHeight := fmt.Sprintf("%d", tipset.Height()) + tsTimestamp := time.Unix(int64(tipset.MinTimestamp()), int64(0)) + + nb, err := InfluxNewBatch() + if err != nil { + log.Fatal(err) + } + + for _, pt := range pl.Points() { + pt.AddTag("height", tsHeight) + pt.SetTime(tsTimestamp) + + nb.AddPoint(NewPointFrom(pt)) + } + + nb.SetDatabase(database) + + log.Printf("Writing %d points for height %d", len(nb.Points()), tipset.Height()) + + wq.AddBatch(nb) + } +} diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go new file mode 100644 index 000000000..6efc013ee --- /dev/null +++ b/tools/stats/metrics.go @@ -0,0 +1,241 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strings" + "time" + + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/build" + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/filecoin-project/go-lotus/chain/types" + + _ "github.com/influxdata/influxdb1-client" + models "github.com/influxdata/influxdb1-client/models" + client "github.com/influxdata/influxdb1-client/v2" +) + +type PointList struct { + points []models.Point +} + +func NewPointList() *PointList { + return &PointList{} +} + +func (pl *PointList) AddPoint(p models.Point) { + pl.points = append(pl.points, p) +} + +func (pl *PointList) Points() []models.Point { + return pl.points +} + +type InfluxWriteQueue struct { + influx client.Client + ch chan client.BatchPoints +} + +func NewInfluxWriteQueue(ctx context.Context, influx client.Client) *InfluxWriteQueue { + ch := make(chan client.BatchPoints, 128) + + maxRetries := 10 + + go func() { + main: + for { + select { + case <-ctx.Done(): + return + case batch := <-ch: + for i := 0; i < maxRetries; i++ { + if err := influx.Write(batch); err != nil { + log.Printf("Failed to write batch: %w", err) + time.Sleep(time.Second * 15) + continue + } + + continue main + } + + log.Printf("Dropping batch due to failure to write") + } + } + }() + + return &InfluxWriteQueue{ + ch: ch, + } +} + +func (i *InfluxWriteQueue) AddBatch(bp client.BatchPoints) { + i.ch <- bp +} + +func (i *InfluxWriteQueue) Close() { + close(i.ch) +} + +func InfluxClient(addr, user, pass string) (client.Client, error) { + return client.NewHTTPClient(client.HTTPConfig{ + Addr: addr, + Username: user, + Password: pass, + }) +} + +func InfluxNewBatch() (client.BatchPoints, error) { + return client.NewBatchPoints(client.BatchPointsConfig{}) +} + +func NewPoint(name string, value interface{}) models.Point { + pt, _ := models.NewPoint(name, models.Tags{}, map[string]interface{}{"value": value}, time.Now()) + return pt +} + +func NewPointFrom(p models.Point) *client.Point { + return client.NewPointFrom(p) +} + +func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { + cids := []string{} + for _, cid := range tipset.Cids() { + cids = append(cids, cid.String()) + } + + p := NewPoint("chain.height", int64(tipset.Height())) + p.AddTag("tipset", strings.Join(cids, " ")) + pl.AddPoint(p) + + p = NewPoint("chain.block_count", len(cids)) + pl.AddPoint(p) + + tsTime := time.Unix(int64(tipset.MinTimestamp()), int64(0)) + p = NewPoint("chain.blocktime", tsTime.Unix()) + pl.AddPoint(p) + + for _, blockheader := range tipset.Blocks() { + bs, err := blockheader.Serialize() + if err != nil { + return err + } + + p := NewPoint("chain.blockheader_size", len(bs)) + pl.AddPoint(p) + } + + return nil +} + +func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { + pc, err := api.StatePledgeCollateral(ctx, tipset) + if err != nil { + return err + } + + pcfil := types.BigDiv(pc, types.NewInt(uint64(build.FilecoinPrecision))) + p := NewPoint("chain.pledge_collateral", pcfil.Int64()) + pl.AddPoint(p) + + power, err := api.StateMinerPower(ctx, address.Address{}, tipset) + if err != nil { + return err + } + + p = NewPoint("chain.power", power.TotalPower.Int64()) + pl.AddPoint(p) + + miners, err := api.StateListMiners(ctx, tipset) + for _, miner := range miners { + power, err := api.StateMinerPower(ctx, miner, tipset) + if err != nil { + return err + } + + p = NewPoint("chain.miner_power", power.MinerPower.Int64()) + p.AddTag("miner", miner.String()) + pl.AddPoint(p) + } + + return nil +} + +func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { + cids := tipset.Cids() + if len(cids) == 0 { + return fmt.Errorf("no cids in tipset") + } + + msgs, err := api.ChainGetParentMessages(ctx, cids[0]) + if err != nil { + return err + } + + recp, err := api.ChainGetParentReceipts(ctx, cids[0]) + if err != nil { + return err + } + + for i, msg := range msgs { + p := NewPoint("chain.message_gasprice", msg.Message.GasPrice.Int64()) + pl.AddPoint(p) + + bs, err := msg.Message.Serialize() + if err != nil { + return err + } + + p = NewPoint("chain.message_size", len(bs)) + pl.AddPoint(p) + + actor, err := api.StateGetActor(ctx, msg.Message.To, tipset) + if err != nil { + return err + } + + p = NewPoint("chain.message_count", 1) + p.AddTag("actor", actor.Code.String()) + p.AddTag("method", fmt.Sprintf("%d", msg.Message.Method)) + p.AddTag("exitcode", fmt.Sprintf("%d", recp[i].ExitCode)) + pl.AddPoint(p) + + } + + return nil +} + +func ResetDatabase(influx client.Client, database string) error { + log.Print("Resetting database") + q := client.NewQuery(fmt.Sprintf(`DROP DATABASE "%s"; CREATE DATABASE "%s";`, database, database), "", "") + _, err := influx.Query(q) + return err +} + +func GetLastRecordedHeight(influx client.Client, database string) (int64, error) { + log.Print("Retrieving last record height") + q := client.NewQuery(`SELECT "value" FROM "chain.height" ORDER BY time DESC LIMIT 1`, database, "") + res, err := influx.Query(q) + if err != nil { + return 0, err + } + + if len(res.Results) == 0 { + return 0, fmt.Errorf("No results found for last recorded height") + } + + if len(res.Results[0].Series) == 0 { + return 0, fmt.Errorf("No results found for last recorded height") + } + + height, err := (res.Results[0].Series[0].Values[0][1].(json.Number)).Int64() + if err != nil { + return 0, err + } + + log.Printf("Last record height %d", height) + + return height, nil +} diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go new file mode 100644 index 000000000..48c58be54 --- /dev/null +++ b/tools/stats/rpc.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "log" + "net/http" + "time" + + "github.com/multiformats/go-multiaddr-net" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/api" + "github.com/filecoin-project/go-lotus/api/client" + "github.com/filecoin-project/go-lotus/chain" + "github.com/filecoin-project/go-lotus/chain/store" + "github.com/filecoin-project/go-lotus/chain/types" + "github.com/filecoin-project/go-lotus/lib/jsonrpc" + "github.com/filecoin-project/go-lotus/node/repo" +) + +func getAPI(path string) (string, http.Header, error) { + r, err := repo.NewFS(path) + if err != nil { + return "", nil, err + } + + ma, err := r.APIEndpoint() + if err != nil { + return "", nil, xerrors.Errorf("failed to get api endpoint: %w", err) + } + _, addr, err := manet.DialArgs(ma) + if err != nil { + return "", nil, err + } + var headers http.Header + token, err := r.APIToken() + if err != nil { + log.Printf("Couldn't load CLI token, capabilities may be limited: %w", err) + } else { + headers = http.Header{} + headers.Add("Authorization", "Bearer "+string(token)) + } + + return "ws://" + addr + "/rpc/v0", headers, nil +} + +func WaitForSyncComplete(ctx context.Context, napi api.FullNode) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(30 * time.Second): + state, err := napi.SyncState(ctx) + if err != nil { + return err + } + + log.Printf("Stage %s, Height %d", chain.SyncStageString(state.Stage), state.Height) + + if state.Stage == api.StageSyncComplete { + return nil + } + } + } +} + +func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan *types.TipSet, error) { + chmain := make(chan *types.TipSet) + + notif, err := api.ChainNotify(ctx) + if err != nil { + return nil, err + } + + go func() { + defer close(chmain) + + for { + select { + case changes := <-notif: + for _, change := range changes { + log.Printf("Head event { height:%d; type: %s }", change.Val.Height(), change.Type) + + switch change.Type { + case store.HCCurrent: + tipsets, err := loadTipsets(ctx, api, change.Val, lastHeight) + if err != nil { + log.Print(err) + return + } + + for _, tipset := range tipsets { + chmain <- tipset + } + case store.HCApply: + chmain <- change.Val + } + } + case <-ctx.Done(): + return + } + } + }() + + return chmain, nil +} + +func loadTipsets(ctx context.Context, api api.FullNode, curr *types.TipSet, lowestHeight uint64) ([]*types.TipSet, error) { + tipsets := []*types.TipSet{} + for { + if curr.Height() == 0 { + break + } + + if curr.Height() <= lowestHeight { + break + } + + log.Printf("Walking back { height:%d }", curr.Height()) + tipsets = append(tipsets, curr) + + ph := ParentTipsetHeight(curr) + if ph == 0 { + break + } + + prev, err := api.ChainGetTipSetByHeight(ctx, ph, curr) + if err != nil { + return tipsets, err + } + + curr = prev + } + + for i, j := 0, len(tipsets)-1; i < j; i, j = i+1, j-1 { + tipsets[i], tipsets[j] = tipsets[j], tipsets[i] + } + + return tipsets, nil +} + +func ParentTipsetHeight(tipset *types.TipSet) uint64 { + mtb := tipset.MinTicketBlock() + return tipset.Height() - uint64(len(mtb.Tickets)) - 1 +} + +func GetFullNodeAPI(repo string) (api.FullNode, jsonrpc.ClientCloser, error) { + addr, headers, err := getAPI(repo) + if err != nil { + return nil, nil, err + } + + return client.NewFullNodeRPC(addr, headers) +} diff --git a/tools/stats/setup.bash b/tools/stats/setup.bash new file mode 100755 index 000000000..e2812b93a --- /dev/null +++ b/tools/stats/setup.bash @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +GRAFANA_HOST="localhost:3000" + +curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF +{ + "name":"InfluxDB", + "type":"influxdb", + "database":"lotus", + "url": "http://influxdb:8086", + "basicAuth":false, + "access": "proxy" +} +EOF + +curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/dashboards/import -H 'Content-Type: text/json' --data-binary @- << EOF | jq -r "\"http://$GRAFANA_HOST\" + .importedUrl" +{ + "dashboard": $(cat ./chain.dashboard.json), + "overwrite": true, + "inputs": [ + { + "name": "DS_INFLUXDB", + "pluginId": "influxdb", + "type": "datasource", + "value": "InfluxDB" + } + ] +} +EOF From 09341d0440e5c4fe43d08f3acf28a857f6744b95 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 17 Oct 2019 14:36:08 -0700 Subject: [PATCH 2/4] Add check for liveness License: MIT Signed-off-by: Jakub Sztandera --- tools/stats/rpc.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go index 48c58be54..832f9249f 100644 --- a/tools/stats/rpc.go +++ b/tools/stats/rpc.go @@ -76,6 +76,8 @@ func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan * go func() { defer close(chmain) + ping := time.Tick(30 * time.Second) + for { select { case changes := <-notif: @@ -97,6 +99,18 @@ func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan * chmain <- change.Val } } + case <-ping: + log.Print("Running health check") + + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + if _, err := api.ID(cctx); err != nil { + log.Print("Health check failed") + return + } + + cancel() + + log.Print("Node online") case <-ctx.Done(): return } From 6d77eb2befc73a281c54f82f56b90ca28cbaa5f9 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 17 Oct 2019 17:20:36 -0700 Subject: [PATCH 3/4] Fix top miner charts License: MIT Signed-off-by: Jakub Sztandera --- tools/stats/chain.dashboard.json | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tools/stats/chain.dashboard.json b/tools/stats/chain.dashboard.json index 27ffe374d..9effbf85f 100644 --- a/tools/stats/chain.dashboard.json +++ b/tools/stats/chain.dashboard.json @@ -14,7 +14,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "6.5.0-pre" + "version": "6.4.2" }, { "type": "panel", @@ -1395,7 +1395,7 @@ "pageSize": null, "showHeader": true, "sort": { - "col": 2, + "col": 1, "desc": true }, "styles": [ @@ -1442,7 +1442,7 @@ "measurement": "chain.miner_power", "orderByTime": "ASC", "policy": "default", - "query": "SELECT last(\"value\") as \"power\" FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY \"miner\" SLIMIT 20", + "query": "SELECT top(\"value\", \"miner\", 20) as \"power\" FROM \"chain.miner_power\" WHERE $timeFilter", "rawQuery": true, "refId": "A", "resultFormat": "table", @@ -1481,6 +1481,7 @@ "y": 10 }, "id": 26, + "interval": "", "legend": { "alignAsTable": true, "avg": false, @@ -1539,7 +1540,7 @@ "orderByTime": "ASC", "policy": "default", "query": "SELECT mean(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time($__interval), \"miner\" fill(previous)", - "rawQuery": false, + "rawQuery": true, "refId": "A", "resultFormat": "time_series", "select": [ @@ -1551,12 +1552,14 @@ "type": "field" }, { - "params": [], - "type": "mean" + "params": [ + "\"miner\",20" + ], + "type": "top" } ] ], - "slimit": "20", + "slimit": "", "tags": [] } ], @@ -2114,5 +2117,5 @@ "timezone": "", "title": "Chain", "uid": "z6FtI92Zz", - "version": 4 + "version": 12 } \ No newline at end of file From 8ecbdc559a85bc87084f0903d3bc739cbd62fd71 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Oct 2019 20:51:35 +0900 Subject: [PATCH 4/4] Increase precision of pledge_collateral and add network actor balance License: MIT Signed-off-by: Jakub Sztandera --- go.sum | 6 ------ tools/stats/metrics.go | 27 +++++++++++++++++++++------ tools/stats/rpc.go | 14 +++++++------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/go.sum b/go.sum index be3b0270d..ec40b61b1 100644 --- a/go.sum +++ b/go.sum @@ -8,7 +8,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -20,7 +19,6 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -244,7 +242,6 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -440,7 +437,6 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -510,9 +506,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 6efc013ee..8d63f1c6d 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -5,13 +5,15 @@ import ( "encoding/json" "fmt" "log" + "math/big" "strings" "time" - "github.com/filecoin-project/go-lotus/api" - "github.com/filecoin-project/go-lotus/build" - "github.com/filecoin-project/go-lotus/chain/address" - "github.com/filecoin-project/go-lotus/chain/types" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" _ "github.com/influxdata/influxdb1-client" models "github.com/influxdata/influxdb1-client/models" @@ -136,8 +138,21 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis return err } - pcfil := types.BigDiv(pc, types.NewInt(uint64(build.FilecoinPrecision))) - p := NewPoint("chain.pledge_collateral", pcfil.Int64()) + attoFil := types.NewInt(build.FilecoinPrecision).Int + + pcFil := new(big.Rat).SetFrac(pc.Int, attoFil) + pcFilFloat, _ := pcFil.Float64() + p := NewPoint("chain.pledge_collateral", pcFilFloat) + pl.AddPoint(p) + + netBal, err := api.WalletBalance(ctx, actors.NetworkAddress) + if err != nil { + return err + } + + netBalFil := new(big.Rat).SetFrac(netBal.Int, attoFil) + netBalFilFloat, _ := netBalFil.Float64() + p = NewPoint("network.balance", netBalFilFloat) pl.AddPoint(p) power, err := api.StateMinerPower(ctx, address.Address{}, tipset) diff --git a/tools/stats/rpc.go b/tools/stats/rpc.go index 832f9249f..32943c16f 100644 --- a/tools/stats/rpc.go +++ b/tools/stats/rpc.go @@ -10,13 +10,13 @@ import ( "golang.org/x/xerrors" - "github.com/filecoin-project/go-lotus/api" - "github.com/filecoin-project/go-lotus/api/client" - "github.com/filecoin-project/go-lotus/chain" - "github.com/filecoin-project/go-lotus/chain/store" - "github.com/filecoin-project/go-lotus/chain/types" - "github.com/filecoin-project/go-lotus/lib/jsonrpc" - "github.com/filecoin-project/go-lotus/node/repo" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/jsonrpc" + "github.com/filecoin-project/lotus/node/repo" ) func getAPI(path string) (string, http.Header, error) {