Merge pull request #403 from travisperson/tool/stats
Add chain stats tool
This commit is contained in:
Whyrusleeping
GitHub
commit
1641ee48a1
5
Makefile
5
Makefile
@ -104,6 +104,11 @@ fountain:
|
||||
go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain
|
||||
.PHONY: fountain
|
||||
|
||||
stats:
|
||||
rm -f stats
|
||||
go build -o stats ./tools/stats
|
||||
.PHONY: stats
|
||||
|
||||
clean:
|
||||
rm -rf $(CLEAN)
|
||||
-$(MAKE) -C $(BLS_PATH) clean
|
||||
|
||||
1
go.mod
1
go.mod
@ -16,6 +16,7 @@ require (
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
github.com/gorilla/websocket v1.4.0
|
||||
github.com/hashicorp/golang-lru v0.5.3
|
||||
github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e
|
||||
github.com/ipfs/go-bitswap v0.1.8
|
||||
github.com/ipfs/go-block-format v0.0.2
|
||||
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c
|
||||
|
||||
8
go.sum
8
go.sum
@ -8,7 +8,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
|
||||
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
|
||||
github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
|
||||
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
|
||||
@ -20,7 +19,6 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU
|
||||
github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo=
|
||||
github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
|
||||
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
@ -138,6 +136,8 @@ github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
|
||||
github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
|
||||
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e h1:txQltCyjXAqVVSZDArPEhUTg35hKwVIuXwtQo7eAMNQ=
|
||||
github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
|
||||
github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
@ -242,7 +242,6 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj
|
||||
github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
|
||||
github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
@ -438,7 +437,6 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA
|
||||
github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ=
|
||||
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg=
|
||||
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@ -508,9 +506,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
|
||||
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0=
|
||||
|
||||
1
tools/stats/.gitignore
vendored
Normal file
1
tools/stats/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
env.stats
|
||||
39
tools/stats/README.md
Normal file
39
tools/stats/README.md
Normal file
@ -0,0 +1,39 @@
|
||||
# Stats
|
||||
|
||||
Stats is a small tool to push chain information into influxdb
|
||||
|
||||
## Setup
|
||||
|
||||
Influx configuration can be configured through env variables.
|
||||
|
||||
```
|
||||
INFLUX_ADDR="http://localhost:8086"
|
||||
INFLUX_USER=""
|
||||
INFLUX_PASS=""
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Stats will be default look in `~/.lotus` to connect to a running daemon and resume collecting stats from last record block height.
|
||||
|
||||
For other usage see `./stats --help`
|
||||
|
||||
```
|
||||
go build -o stats *.go
|
||||
. env.stats && ./stats
|
||||
```
|
||||
|
||||
|
||||
## Development
|
||||
|
||||
Start grafana and influxdb containers and import the dashboard to grafana.
|
||||
The url of the imported dashboard will be returned.
|
||||
|
||||
If the script doesn't work, you can manually setup the datasource and import the dashboard.
|
||||
|
||||
```
|
||||
docker-compose up -d
|
||||
./setup.bash
|
||||
```
|
||||
|
||||
The default username and password for grafana are both `admin`.
|
||||
2121
tools/stats/chain.dashboard.json
Normal file
2121
tools/stats/chain.dashboard.json
Normal file
File diff suppressed because it is too large
Load Diff
26
tools/stats/docker-compose.yml
Normal file
26
tools/stats/docker-compose.yml
Normal file
@ -0,0 +1,26 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
influxdb:
|
||||
image: influxdb:latest
|
||||
container_name: influxdb
|
||||
environment:
|
||||
- INFLUXDB_DB=lotus
|
||||
ports:
|
||||
- "8086:8086"
|
||||
volumes:
|
||||
- influxdb:/var/lib/influxdb
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
links:
|
||||
- influxdb
|
||||
volumes:
|
||||
- grafana:/var/lib/grafana
|
||||
|
||||
volumes:
|
||||
influxdb:
|
||||
grafana:
|
||||
3
tools/stats/env.stats
Normal file
3
tools/stats/env.stats
Normal file
@ -0,0 +1,3 @@
|
||||
export INFLUX_ADDR="http://localhost:8086"
|
||||
export INFLUX_USER=""
|
||||
export INFLUX_PASS=""
|
||||
118
tools/stats/main.go
Normal file
118
tools/stats/main.go
Normal file
@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
INFLUX_ADDR = "INFLUX_ADDR"
|
||||
INFLUX_USER = "INFLUX_USER"
|
||||
INFLUX_PASS = "INFLUX_PASS"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var repo string = "~/.lotus"
|
||||
var database string = "lotus"
|
||||
var reset bool = false
|
||||
var height int64 = 0
|
||||
|
||||
flag.StringVar(&repo, "repo", repo, "lotus repo path")
|
||||
flag.StringVar(&database, "database", database, "influx database")
|
||||
flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)")
|
||||
flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
influxAddr := os.Getenv(INFLUX_ADDR)
|
||||
influxUser := os.Getenv(INFLUX_USER)
|
||||
influxPass := os.Getenv(INFLUX_PASS)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
influx, err := InfluxClient(influxAddr, influxUser, influxPass)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if reset {
|
||||
if err := ResetDatabase(influx, database); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !reset && height == 0 {
|
||||
h, err := GetLastRecordedHeight(influx, database)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
|
||||
height = h
|
||||
}
|
||||
|
||||
api, closer, err := GetFullNodeAPI(repo)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer closer()
|
||||
|
||||
if err := WaitForSyncComplete(ctx, api); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
tipsetsCh, err := GetTips(ctx, api, uint64(height))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
wq := NewInfluxWriteQueue(ctx, influx)
|
||||
defer wq.Close()
|
||||
|
||||
for tipset := range tipsetsCh {
|
||||
pl := NewPointList()
|
||||
height := tipset.Height()
|
||||
|
||||
if err := RecordTipsetPoints(ctx, api, pl, tipset); err != nil {
|
||||
log.Printf("Failed to record tipset at height %d: %w", height, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := RecordTipsetMessagesPoints(ctx, api, pl, tipset); err != nil {
|
||||
log.Printf("Failed to record messages at height %d: %w", height, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := RecordTipsetStatePoints(ctx, api, pl, tipset); err != nil {
|
||||
log.Printf("Failed to record state at height %d: %w", height, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Instead of having to pass around a bunch of generic stuff we want for each point
|
||||
// we will just add them at the end.
|
||||
|
||||
tsHeight := fmt.Sprintf("%d", tipset.Height())
|
||||
tsTimestamp := time.Unix(int64(tipset.MinTimestamp()), int64(0))
|
||||
|
||||
nb, err := InfluxNewBatch()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, pt := range pl.Points() {
|
||||
pt.AddTag("height", tsHeight)
|
||||
pt.SetTime(tsTimestamp)
|
||||
|
||||
nb.AddPoint(NewPointFrom(pt))
|
||||
}
|
||||
|
||||
nb.SetDatabase(database)
|
||||
|
||||
log.Printf("Writing %d points for height %d", len(nb.Points()), tipset.Height())
|
||||
|
||||
wq.AddBatch(nb)
|
||||
}
|
||||
}
|
||||
256
tools/stats/metrics.go
Normal file
256
tools/stats/metrics.go
Normal file
@ -0,0 +1,256 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/actors"
|
||||
"github.com/filecoin-project/lotus/chain/address"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
|
||||
_ "github.com/influxdata/influxdb1-client"
|
||||
models "github.com/influxdata/influxdb1-client/models"
|
||||
client "github.com/influxdata/influxdb1-client/v2"
|
||||
)
|
||||
|
||||
type PointList struct {
|
||||
points []models.Point
|
||||
}
|
||||
|
||||
func NewPointList() *PointList {
|
||||
return &PointList{}
|
||||
}
|
||||
|
||||
func (pl *PointList) AddPoint(p models.Point) {
|
||||
pl.points = append(pl.points, p)
|
||||
}
|
||||
|
||||
func (pl *PointList) Points() []models.Point {
|
||||
return pl.points
|
||||
}
|
||||
|
||||
type InfluxWriteQueue struct {
|
||||
influx client.Client
|
||||
ch chan client.BatchPoints
|
||||
}
|
||||
|
||||
func NewInfluxWriteQueue(ctx context.Context, influx client.Client) *InfluxWriteQueue {
|
||||
ch := make(chan client.BatchPoints, 128)
|
||||
|
||||
maxRetries := 10
|
||||
|
||||
go func() {
|
||||
main:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case batch := <-ch:
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
if err := influx.Write(batch); err != nil {
|
||||
log.Printf("Failed to write batch: %w", err)
|
||||
time.Sleep(time.Second * 15)
|
||||
continue
|
||||
}
|
||||
|
||||
continue main
|
||||
}
|
||||
|
||||
log.Printf("Dropping batch due to failure to write")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return &InfluxWriteQueue{
|
||||
ch: ch,
|
||||
}
|
||||
}
|
||||
|
||||
func (i *InfluxWriteQueue) AddBatch(bp client.BatchPoints) {
|
||||
i.ch <- bp
|
||||
}
|
||||
|
||||
func (i *InfluxWriteQueue) Close() {
|
||||
close(i.ch)
|
||||
}
|
||||
|
||||
func InfluxClient(addr, user, pass string) (client.Client, error) {
|
||||
return client.NewHTTPClient(client.HTTPConfig{
|
||||
Addr: addr,
|
||||
Username: user,
|
||||
Password: pass,
|
||||
})
|
||||
}
|
||||
|
||||
func InfluxNewBatch() (client.BatchPoints, error) {
|
||||
return client.NewBatchPoints(client.BatchPointsConfig{})
|
||||
}
|
||||
|
||||
func NewPoint(name string, value interface{}) models.Point {
|
||||
pt, _ := models.NewPoint(name, models.Tags{}, map[string]interface{}{"value": value}, time.Now())
|
||||
return pt
|
||||
}
|
||||
|
||||
func NewPointFrom(p models.Point) *client.Point {
|
||||
return client.NewPointFrom(p)
|
||||
}
|
||||
|
||||
func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error {
|
||||
cids := []string{}
|
||||
for _, cid := range tipset.Cids() {
|
||||
cids = append(cids, cid.String())
|
||||
}
|
||||
|
||||
p := NewPoint("chain.height", int64(tipset.Height()))
|
||||
p.AddTag("tipset", strings.Join(cids, " "))
|
||||
pl.AddPoint(p)
|
||||
|
||||
p = NewPoint("chain.block_count", len(cids))
|
||||
pl.AddPoint(p)
|
||||
|
||||
tsTime := time.Unix(int64(tipset.MinTimestamp()), int64(0))
|
||||
p = NewPoint("chain.blocktime", tsTime.Unix())
|
||||
pl.AddPoint(p)
|
||||
|
||||
for _, blockheader := range tipset.Blocks() {
|
||||
bs, err := blockheader.Serialize()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p := NewPoint("chain.blockheader_size", len(bs))
|
||||
pl.AddPoint(p)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error {
|
||||
pc, err := api.StatePledgeCollateral(ctx, tipset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attoFil := types.NewInt(build.FilecoinPrecision).Int
|
||||
|
||||
pcFil := new(big.Rat).SetFrac(pc.Int, attoFil)
|
||||
pcFilFloat, _ := pcFil.Float64()
|
||||
p := NewPoint("chain.pledge_collateral", pcFilFloat)
|
||||
pl.AddPoint(p)
|
||||
|
||||
netBal, err := api.WalletBalance(ctx, actors.NetworkAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
netBalFil := new(big.Rat).SetFrac(netBal.Int, attoFil)
|
||||
netBalFilFloat, _ := netBalFil.Float64()
|
||||
p = NewPoint("network.balance", netBalFilFloat)
|
||||
pl.AddPoint(p)
|
||||
|
||||
power, err := api.StateMinerPower(ctx, address.Address{}, tipset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p = NewPoint("chain.power", power.TotalPower.Int64())
|
||||
pl.AddPoint(p)
|
||||
|
||||
miners, err := api.StateListMiners(ctx, tipset)
|
||||
for _, miner := range miners {
|
||||
power, err := api.StateMinerPower(ctx, miner, tipset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p = NewPoint("chain.miner_power", power.MinerPower.Int64())
|
||||
p.AddTag("miner", miner.String())
|
||||
pl.AddPoint(p)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error {
|
||||
cids := tipset.Cids()
|
||||
if len(cids) == 0 {
|
||||
return fmt.Errorf("no cids in tipset")
|
||||
}
|
||||
|
||||
msgs, err := api.ChainGetParentMessages(ctx, cids[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
recp, err := api.ChainGetParentReceipts(ctx, cids[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, msg := range msgs {
|
||||
p := NewPoint("chain.message_gasprice", msg.Message.GasPrice.Int64())
|
||||
pl.AddPoint(p)
|
||||
|
||||
bs, err := msg.Message.Serialize()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p = NewPoint("chain.message_size", len(bs))
|
||||
pl.AddPoint(p)
|
||||
|
||||
actor, err := api.StateGetActor(ctx, msg.Message.To, tipset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p = NewPoint("chain.message_count", 1)
|
||||
p.AddTag("actor", actor.Code.String())
|
||||
p.AddTag("method", fmt.Sprintf("%d", msg.Message.Method))
|
||||
p.AddTag("exitcode", fmt.Sprintf("%d", recp[i].ExitCode))
|
||||
pl.AddPoint(p)
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ResetDatabase(influx client.Client, database string) error {
|
||||
log.Print("Resetting database")
|
||||
q := client.NewQuery(fmt.Sprintf(`DROP DATABASE "%s"; CREATE DATABASE "%s";`, database, database), "", "")
|
||||
_, err := influx.Query(q)
|
||||
return err
|
||||
}
|
||||
|
||||
func GetLastRecordedHeight(influx client.Client, database string) (int64, error) {
|
||||
log.Print("Retrieving last record height")
|
||||
q := client.NewQuery(`SELECT "value" FROM "chain.height" ORDER BY time DESC LIMIT 1`, database, "")
|
||||
res, err := influx.Query(q)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if len(res.Results) == 0 {
|
||||
return 0, fmt.Errorf("No results found for last recorded height")
|
||||
}
|
||||
|
||||
if len(res.Results[0].Series) == 0 {
|
||||
return 0, fmt.Errorf("No results found for last recorded height")
|
||||
}
|
||||
|
||||
height, err := (res.Results[0].Series[0].Values[0][1].(json.Number)).Int64()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
log.Printf("Last record height %d", height)
|
||||
|
||||
return height, nil
|
||||
}
|
||||
169
tools/stats/rpc.go
Normal file
169
tools/stats/rpc.go
Normal file
@ -0,0 +1,169 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/multiformats/go-multiaddr-net"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/api/client"
|
||||
"github.com/filecoin-project/lotus/chain"
|
||||
"github.com/filecoin-project/lotus/chain/store"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/jsonrpc"
|
||||
"github.com/filecoin-project/lotus/node/repo"
|
||||
)
|
||||
|
||||
func getAPI(path string) (string, http.Header, error) {
|
||||
r, err := repo.NewFS(path)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
ma, err := r.APIEndpoint()
|
||||
if err != nil {
|
||||
return "", nil, xerrors.Errorf("failed to get api endpoint: %w", err)
|
||||
}
|
||||
_, addr, err := manet.DialArgs(ma)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var headers http.Header
|
||||
token, err := r.APIToken()
|
||||
if err != nil {
|
||||
log.Printf("Couldn't load CLI token, capabilities may be limited: %w", err)
|
||||
} else {
|
||||
headers = http.Header{}
|
||||
headers.Add("Authorization", "Bearer "+string(token))
|
||||
}
|
||||
|
||||
return "ws://" + addr + "/rpc/v0", headers, nil
|
||||
}
|
||||
|
||||
func WaitForSyncComplete(ctx context.Context, napi api.FullNode) error {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(30 * time.Second):
|
||||
state, err := napi.SyncState(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Stage %s, Height %d", chain.SyncStageString(state.Stage), state.Height)
|
||||
|
||||
if state.Stage == api.StageSyncComplete {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func GetTips(ctx context.Context, api api.FullNode, lastHeight uint64) (<-chan *types.TipSet, error) {
|
||||
chmain := make(chan *types.TipSet)
|
||||
|
||||
notif, err := api.ChainNotify(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(chmain)
|
||||
|
||||
ping := time.Tick(30 * time.Second)
|
||||
|
||||
for {
|
||||
select {
|
||||
case changes := <-notif:
|
||||
for _, change := range changes {
|
||||
log.Printf("Head event { height:%d; type: %s }", change.Val.Height(), change.Type)
|
||||
|
||||
switch change.Type {
|
||||
case store.HCCurrent:
|
||||
tipsets, err := loadTipsets(ctx, api, change.Val, lastHeight)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, tipset := range tipsets {
|
||||
chmain <- tipset
|
||||
}
|
||||
case store.HCApply:
|
||||
chmain <- change.Val
|
||||
}
|
||||
}
|
||||
case <-ping:
|
||||
log.Print("Running health check")
|
||||
|
||||
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
if _, err := api.ID(cctx); err != nil {
|
||||
log.Print("Health check failed")
|
||||
return
|
||||
}
|
||||
|
||||
cancel()
|
||||
|
||||
log.Print("Node online")
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return chmain, nil
|
||||
}
|
||||
|
||||
func loadTipsets(ctx context.Context, api api.FullNode, curr *types.TipSet, lowestHeight uint64) ([]*types.TipSet, error) {
|
||||
tipsets := []*types.TipSet{}
|
||||
for {
|
||||
if curr.Height() == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
if curr.Height() <= lowestHeight {
|
||||
break
|
||||
}
|
||||
|
||||
log.Printf("Walking back { height:%d }", curr.Height())
|
||||
tipsets = append(tipsets, curr)
|
||||
|
||||
ph := ParentTipsetHeight(curr)
|
||||
if ph == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
prev, err := api.ChainGetTipSetByHeight(ctx, ph, curr)
|
||||
if err != nil {
|
||||
return tipsets, err
|
||||
}
|
||||
|
||||
curr = prev
|
||||
}
|
||||
|
||||
for i, j := 0, len(tipsets)-1; i < j; i, j = i+1, j-1 {
|
||||
tipsets[i], tipsets[j] = tipsets[j], tipsets[i]
|
||||
}
|
||||
|
||||
return tipsets, nil
|
||||
}
|
||||
|
||||
func ParentTipsetHeight(tipset *types.TipSet) uint64 {
|
||||
mtb := tipset.MinTicketBlock()
|
||||
return tipset.Height() - uint64(len(mtb.Tickets)) - 1
|
||||
}
|
||||
|
||||
func GetFullNodeAPI(repo string) (api.FullNode, jsonrpc.ClientCloser, error) {
|
||||
addr, headers, err := getAPI(repo)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return client.NewFullNodeRPC(addr, headers)
|
||||
}
|
||||
29
tools/stats/setup.bash
Executable file
29
tools/stats/setup.bash
Executable file
@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
GRAFANA_HOST="localhost:3000"
|
||||
|
||||
curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF
|
||||
{
|
||||
"name":"InfluxDB",
|
||||
"type":"influxdb",
|
||||
"database":"lotus",
|
||||
"url": "http://influxdb:8086",
|
||||
"basicAuth":false,
|
||||
"access": "proxy"
|
||||
}
|
||||
EOF
|
||||
|
||||
curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/dashboards/import -H 'Content-Type: text/json' --data-binary @- << EOF | jq -r "\"http://$GRAFANA_HOST\" + .importedUrl"
|
||||
{
|
||||
"dashboard": $(cat ./chain.dashboard.json),
|
||||
"overwrite": true,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "DS_INFLUXDB",
|
||||
"pluginId": "influxdb",
|
||||
"type": "datasource",
|
||||
"value": "InfluxDB"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
Loading…
Reference in New Issue
Block a user