cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Insert miner and network power data as gibibytes to avoid int64 overflows.

Browse Source
This commit is contained in:
Travis Person 2021-08-26 23:55:46 +00:00
parent 7c484cae13
commit 0e8abc1973
5 changed files with 317 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/lotus-stats/README.md
View File

@ -7,22 +7,24 @@
Influx configuration can be configured through env variables. Influx configuration can be configured through env variables.
``` ```
INFLUX_ADDR="http://localhost:8086" LOTUS_STATS_INFLUX_ADDR="http://localhost:8086"
INFLUX_USER="" LOTUS_STATS_INFLUX_USER=""
INFLUX_PASS="" LOTUS_STATS_INFLUX_PASS=""
``` ```
## Usage ## Usage
lotus-stats will be default look in `~/.lotus` to connect to a running daemon and resume collecting stats from last record block height. lotus-stats will look in `~/.lotus` to connect to a running daemon and resume collecting stats from last record block height.
For other usage see `./lotus-stats --help` For other usage see `./lotus-stats --help`
``` ```
go build -o lotus-stats *.go go build -o lotus-stats *.go
. env.stats && ./lotus-stats . env.stats && ./lotus-stats run
``` ```
For large networks there is an additional query in the `Top Miner Power` table, which can be toggled on to only show miners larger
than 1 PiB. This is a good option to enable to reduce the number of miners listed when viewing mainnet stats.
## Development ## Development
@ -37,3 +39,8 @@ docker-compose up -d
``` ```
The default username and password for grafana are both `admin`. The default username and password for grafana are both `admin`.
## Updating the dashboard
After importing the provided dashboard in `chain.dashboard.json`, you may make changes to the dashboard. To export
the dashboard to be commited back to the project, make sure the option "sharing externally" is toggled on.

340
cmd/lotus-stats/chain.dashboard.json
View File

@ -30,6 +30,12 @@
"id": "table-old", "id": "table-old",
"name": "Table (old)", "name": "Table (old)",
"version": "" "version": ""
},
{
"type": "panel",
"id": "text",
"name": "Text",
"version": ""
} }
], ],
"annotations": { "annotations": {
@ -47,11 +53,217 @@
}, },
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 1,
"id": null, "id": null,
"iteration": 1604018016916, "iteration": 1630020824868,
"links": [], "links": [],
"panels": [ "panels": [
{
"datasource": "$network",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 5,
"x": 0,
"y": 0
},
"id": 56,
"options": {
"content": "<div style=\"width: 200px; height: 50px;\" >\n <img style=\"margin-top:-20px;\" src=\"https://ipfs.io/ipfs/QmaW8ezM7AQfUb5CoSwcfETQk6jHAk259hFwm8Krji6AR7/Logo_Version01_BlueWhite.png\"/>\n</div>\n",
"mode": "html"
},
"pluginVersion": "7.3.0",
"targets": [
{
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"timeFrom": null,
"timeShift": null,
"title": "",
"transparent": true,
"type": "text"
},
{
"datasource": "$network",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 16,
"x": 5,
"y": 0
},
"id": 58,
"options": {
"content": "<div style=\"font-weight: 800; font-size: 1.5em;color:#a3cae1;margin-top:10px; display: none;\">\n The stats dashboard is undergoing some maintance. Please check back later.\n</div>",
"mode": "html"
},
"pluginVersion": "7.3.0",
"targets": [
{
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"timeFrom": null,
"timeShift": null,
"title": "",
"transparent": true,
"type": "text"
},
{
"datasource": "$network",
"fieldConfig": {
"defaults": {
"custom": {
"align": null,
"filterable": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 3,
"x": 21,
"y": 0
},
"id": 54,
"options": {
"content": "<div>\n <div>\n <ul style=\"list-style-type:none;margin-right:20px;padding-top:12px;overflow:hidden;\">\n <li style=\"display:inline\"><a href=\"https://lotu.sh\" target=\"_blank\" style=\"display:inline;\">Lotus Docs</a></li>\n <li style=\"display:inline\"><a href=\"https://filecoin.io/build/#community\" target=\"_blank\">Community</a></li>\n </ul>\n </div>\n</div>",
"mode": "html"
},
"pluginVersion": "7.3.0",
"targets": [
{
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"timeFrom": null,
"timeShift": null,
"title": "",
"transparent": true,
"type": "text"
},
{ {
"aliasColors": {}, "aliasColors": {},
"bars": true, "bars": true,
@ -72,7 +284,7 @@
"h": 9, "h": 9,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 0 "y": 3
}, },
"hiddenSeries": false, "hiddenSeries": false,
"hideTimeOverride": false, "hideTimeOverride": false,
@ -186,7 +398,7 @@
], ],
"hide": false, "hide": false,
"orderByTime": "ASC", "orderByTime": "ASC",
"policy": "defult", "policy": "default",
"query": "SELECT TRIPLE_EXPONENTIAL_MOVING_AVERAGE(sum(\"value\"), 40) FROM \"chain.election\" WHERE $timeFilter -$blockInterval*40 AND time < now() - $blockInterval*3 GROUP BY time($blockInterval) fill(0)", "query": "SELECT TRIPLE_EXPONENTIAL_MOVING_AVERAGE(sum(\"value\"), 40) FROM \"chain.election\" WHERE $timeFilter -$blockInterval*40 AND time < now() - $blockInterval*3 GROUP BY time($blockInterval) fill(0)",
"rawQuery": true, "rawQuery": true,
"refId": "B", "refId": "B",
@ -271,7 +483,7 @@
"h": 4, "h": 4,
"w": 8, "w": 8,
"x": 0, "x": 0,
"y": 9 "y": 12
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 22, "id": 22,
@ -396,7 +608,7 @@
"h": 4, "h": 4,
"w": 4, "w": 4,
"x": 8, "x": 8,
"y": 9 "y": 12
}, },
"id": 12, "id": 12,
"interval": null, "interval": null,
@ -493,7 +705,7 @@
}, },
"overrides": [] "overrides": []
}, },
"format": "bytes", "format": "gbytes",
"gauge": { "gauge": {
"maxValue": 100, "maxValue": 100,
"minValue": 0, "minValue": 0,
@ -505,7 +717,7 @@
"h": 4, "h": 4,
"w": 4, "w": 4,
"x": 12, "x": 12,
"y": 9 "y": 12
}, },
"id": 42, "id": 42,
"interval": "", "interval": "",
@ -626,7 +838,7 @@
"h": 4, "h": 4,
"w": 8, "w": 8,
"x": 16, "x": 16,
"y": 9 "y": 12
}, },
"id": 6, "id": 6,
"interval": null, "interval": null,
@ -741,7 +953,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 0, "x": 0,
"y": 13 "y": 16
}, },
"id": 4, "id": 4,
"interval": null, "interval": null,
@ -844,7 +1056,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 4, "x": 4,
"y": 13 "y": 16
}, },
"id": 14, "id": 14,
"interval": null, "interval": null,
@ -951,7 +1163,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 8, "x": 8,
"y": 13 "y": 16
}, },
"id": 32, "id": 32,
"interval": null, "interval": null,
@ -1071,7 +1283,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 12, "x": 12,
"y": 13 "y": 16
}, },
"id": 20, "id": 20,
"interval": null, "interval": null,
@ -1191,7 +1403,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 16, "x": 16,
"y": 13 "y": 16
}, },
"id": 8, "id": 8,
"interval": null, "interval": null,
@ -1311,7 +1523,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 20, "x": 20,
"y": 13 "y": 16
}, },
"id": 10, "id": 10,
"interval": null, "interval": null,
@ -1435,7 +1647,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 0, "x": 0,
"y": 16 "y": 19
}, },
"id": 16, "id": 16,
"interval": "", "interval": "",
@ -1536,7 +1748,7 @@
"h": 3, "h": 3,
"w": 16, "w": 16,
"x": 4, "x": 4,
"y": 16 "y": 19
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 2, "id": 2,
@ -1706,7 +1918,7 @@
"h": 3, "h": 3,
"w": 4, "w": 4,
"x": 20, "x": 20,
"y": 16 "y": 19
}, },
"id": 30, "id": 30,
"interval": null, "interval": null,
@ -1795,7 +2007,7 @@
"h": 21, "h": 21,
"w": 4, "w": 4,
"x": 0, "x": 0,
"y": 19 "y": 22
}, },
"id": 28, "id": 28,
"pageSize": null, "pageSize": null,
@ -1827,7 +2039,7 @@
"pattern": "power", "pattern": "power",
"thresholds": [], "thresholds": [],
"type": "number", "type": "number",
"unit": "bytes" "unit": "gbytes"
}, },
{ {
"alias": "", "alias": "",
@ -1894,7 +2106,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 4, "x": 4,
"y": 19 "y": 22
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 40, "id": 40,
@ -1979,6 +2191,44 @@
], ],
"slimit": "", "slimit": "",
"tags": [] "tags": []
},
{
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"hide": true,
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter AND value > 1125899906842624 GROUP BY time($__interval), \"miner\" fill(null)",
"rawQuery": true,
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
} }
], ],
"thresholds": [], "thresholds": [],
@ -2002,7 +2252,7 @@
"yaxes": [ "yaxes": [
{ {
"decimals": 2, "decimals": 2,
"format": "bytes", "format": "gbytes",
"label": "Power", "label": "Power",
"logBase": 1, "logBase": 1,
"max": "100", "max": "100",
@ -2037,7 +2287,7 @@
"h": 21, "h": 21,
"w": 8, "w": 8,
"x": 16, "x": 16,
"y": 19 "y": 22
}, },
"id": 18, "id": 18,
"pageSize": null, "pageSize": null,
@ -2142,7 +2392,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 4, "x": 4,
"y": 27 "y": 30
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 50, "id": 50,
@ -2182,7 +2432,7 @@
}, },
{ {
"params": [ "params": [
"null" "previous"
], ],
"type": "fill" "type": "fill"
} }
@ -2221,7 +2471,7 @@
}, },
{ {
"params": [ "params": [
"null" "previous"
], ],
"type": "fill" "type": "fill"
} }
@ -2260,7 +2510,7 @@
}, },
{ {
"params": [ "params": [
"null" "previous"
], ],
"type": "fill" "type": "fill"
} }
@ -2361,7 +2611,7 @@
"h": 6, "h": 6,
"w": 12, "w": 12,
"x": 4, "x": 4,
"y": 34 "y": 37
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 44, "id": 44,
@ -2491,7 +2741,7 @@
"h": 9, "h": 9,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 40 "y": 43
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 34, "id": 34,
@ -2632,7 +2882,7 @@
"h": 9, "h": 9,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 40 "y": 43
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 36, "id": 36,
@ -2785,7 +3035,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 49 "y": 52
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 48, "id": 48,
@ -2915,7 +3165,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 49 "y": 52
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 46, "id": 46,
@ -3046,7 +3296,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 57 "y": 60
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 51, "id": 51,
@ -3218,7 +3468,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 57 "y": 60
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 52, "id": 52,
@ -3417,8 +3667,8 @@
{ {
"current": { "current": {
"selected": false, "selected": false,
"text": "filecoin-ntwk-testnet", "text": "ntwk-localstats",
"value": "filecoin-ntwk-testnet" "value": "ntwk-localstats"
}, },
"error": null, "error": null,
"hide": 0, "hide": 0,
@ -3430,7 +3680,7 @@
"query": "influxdb", "query": "influxdb",
"queryValue": "", "queryValue": "",
"refresh": 1, "refresh": 1,
"regex": "/^filecoin-ntwk-/", "regex": "/^ntwk-/",
"skipUrlSync": false, "skipUrlSync": false,
"type": "datasource" "type": "datasource"
}, },
@ -3466,23 +3716,13 @@
"to": "now" "to": "now"
}, },
"timepicker": { "timepicker": {
"hidden": true,
"refresh_intervals": [ "refresh_intervals": [
"5s", "30s"
"10s",
"25s",
"30s",
"45s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
] ]
}, },
"timezone": "", "timezone": "",
"title": "Filecoin Chain Stats", "title": "Filecoin Chain Stats",
"uid": "z6FtI92Zz", "uid": "z6FtI92Zz",
"version": 4 "version": 2
} }

6
cmd/lotus-stats/env.stats
View File

@ -1,3 +1,3 @@
export INFLUX_ADDR="http://localhost:18086" export LOTUS_STATS_INFLUX_ADDR="http://localhost:18086"
export INFLUX_USER="" export LOTUS_STATS_INFLUX_USER=""
export INFLUX_PASS="" export LOTUS_STATS_INFLUX_PASS=""

8
cmd/lotus-stats/setup.bash
View File

@ -1,10 +1,10 @@
#!/usr/bin/env bash #!/usr/bin/env bash
GRAFANA_HOST="http://localhost:13000" GRAFANA_HOST="localhost:13000"
curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF curl -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF
{ {
"name":"filecoin-ntwk-localstats", "name":"ntwk-localstats",
"type":"influxdb", "type":"influxdb",
"database":"lotus", "database":"lotus",
"url": "http://influxdb:8086", "url": "http://influxdb:8086",
@ -13,7 +13,7 @@ curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type
} }
EOF EOF
curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/dashboards/import -H 'Content-Type: text/json' --data-binary @- << EOF | jq -r "\"http://$GRAFANA_HOST\" + .importedUrl" curl -XPOST http://admin:admin@$GRAFANA_HOST/api/dashboards/import -H 'Content-Type: text/json' --data-binary @- << EOF | jq -r "\"http://$GRAFANA_HOST\" + .importedUrl"
{ {
"dashboard": $(cat ./chain.dashboard.json), "dashboard": $(cat ./chain.dashboard.json),
"overwrite": true, "overwrite": true,

11
tools/stats/metrics.go
View File

@ -267,7 +267,11 @@ func RecordTipsetStatePoints(ctx context.Context, api v0api.FullNode, pl *PointL
return err return err
} }
p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) // We divide the power into gibibytes because 2^63 bytes is 8 exbibytes which is smaller than the Filecoin Mainnet.
// Dividing by a gibibyte gives us more room to work with. This will allow the dashboard to report network and miner
// sizes up to 8192 yobibytes.
gibi := types.NewInt(1024 * 1024 * 1024)
p = NewPoint("chain.power", types.BigDiv(totalPower.TotalPower.QualityAdjPower, gibi).Int64())
pl.AddPoint(p) pl.AddPoint(p)
powerActor, err := api.StateGetActor(ctx, power.Address, tipset.Key()) powerActor, err := api.StateGetActor(ctx, power.Address, tipset.Key())
@ -281,11 +285,12 @@ func RecordTipsetStatePoints(ctx context.Context, api v0api.FullNode, pl *PointL
} }
return powerActorState.ForEachClaim(func(addr address.Address, claim power.Claim) error { return powerActorState.ForEachClaim(func(addr address.Address, claim power.Claim) error {
if claim.QualityAdjPower.Int64() == 0 { // BigCmp returns 0 if values are equal
if types.BigCmp(claim.QualityAdjPower, types.NewInt(0)) == 0 {
return nil return nil
} }
p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) p = NewPoint("chain.miner_power", types.BigDiv(claim.QualityAdjPower, gibi).Int64())
p.AddTag("miner", addr.String()) p.AddTag("miner", addr.String())
pl.AddPoint(p) pl.AddPoint(p)