Merge pull request #4718 from filecoin-project/feat/stats-update

Update lotus-stats with a richer cli

Makefile

@@ -179,7 +179,7 @@ BINS+=lotus-bench
 
 lotus-stats:
 	rm -f lotus-stats
-	go build -o lotus-stats ./cmd/lotus-stats
+	go build $(GOFLAGS) -o lotus-stats ./cmd/lotus-stats
 	go run github.com/GeertJohan/go.rice/rice append --exec lotus-stats -i ./build
 .PHONY: lotus-stats
 BINS+=lotus-stats

cmd/lotus-stats/docker-compose.yml

@@ -4,10 +4,10 @@ services:
   influxdb:
     image: influxdb:latest
     container_name: influxdb
+    ports:
+      - "18086:8086"
     environment:
       - INFLUXDB_DB=lotus
-    ports:
-      - "8086:8086"
     volumes:
       - influxdb:/var/lib/influxdb
 
@@ -15,7 +15,7 @@ services:
     image: grafana/grafana:latest
     container_name: grafana
     ports:
-      - "3000:3000"
+      - "13000:3000"
     links:
       - influxdb
     volumes:

cmd/lotus-stats/env.stats

@@ -1,3 +1,3 @@
-export INFLUX_ADDR="http://localhost:8086"
+export INFLUX_ADDR="http://localhost:18086"
 export INFLUX_USER=""
 export INFLUX_PASS=""

cmd/lotus-stats/main.go

@@ -2,71 +2,158 @@ package main
 
 import (
 	"context"
-	"flag"
 	"os"
 
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/tools/stats"
+
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
 )
 
 var log = logging.Logger("stats")
 
-const (
-	influxAddrEnvVar = "INFLUX_ADDR"
-	influxUserEnvVar = "INFLUX_USER"
-	influxPassEnvVar = "INFLUX_PASS"
-)
-
 func main() {
-	var repo string = "~/.lotus"
+	local := []*cli.Command{
-	var database string = "lotus"
+		runCmd,
-	var reset bool = false
+		versionCmd,
-	var nosync bool = false
-	var height int64 = 0
-	var headlag int = 3
-
-	flag.StringVar(&repo, "repo", repo, "lotus repo path")
-	flag.StringVar(&database, "database", database, "influx database")
-	flag.Int64Var(&height, "height", height, "block height to start syncing from (0 will resume)")
-	flag.IntVar(&headlag, "head-lag", headlag, "number of head events to hold to protect against small reorgs")
-	flag.BoolVar(&reset, "reset", reset, "truncate database before starting stats gathering")
-	flag.BoolVar(&nosync, "nosync", nosync, "skip waiting for sync")
-
-	flag.Parse()
-
-	ctx := context.Background()
-
-	influx, err := stats.InfluxClient(os.Getenv(influxAddrEnvVar), os.Getenv(influxUserEnvVar), os.Getenv(influxPassEnvVar))
-	if err != nil {
-		log.Fatal(err)
 	}
 
-	if reset {
+	app := &cli.App{
-		if err := stats.ResetDatabase(influx, database); err != nil {
+		Name:    "lotus-stats",
-			log.Fatal(err)
+		Usage:   "Collect basic information about a filecoin network using lotus",
-		}
+		Version: build.UserVersion(),
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "lotus-path",
+				EnvVars: []string{"LOTUS_PATH"},
+				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
+			},
+			&cli.StringFlag{
+				Name:    "log-level",
+				EnvVars: []string{"LOTUS_STATS_LOG_LEVEL"},
+				Value:   "info",
+			},
+		},
+		Before: func(cctx *cli.Context) error {
+			return logging.SetLogLevel("stats", cctx.String("log-level"))
+		},
+		Commands: local,
 	}
 
-	if !reset && height == 0 {
+	if err := app.Run(os.Args); err != nil {
-		h, err := stats.GetLastRecordedHeight(influx, database)
+		log.Errorw("exit in error", "err", err)
-		if err != nil {
+		os.Exit(1)
-			log.Info(err)
+		return
-		}
-
-		height = h
 	}
-
+}
-	api, closer, err := stats.GetFullNodeAPI(ctx, repo)
+
-	if err != nil {
+var versionCmd = &cli.Command{
-		log.Fatal(err)
+	Name:  "version",
-	}
+	Usage: "Print version",
-	defer closer()
+	Action: func(cctx *cli.Context) error {
-
+		cli.VersionPrinter(cctx)
-	if !nosync {
+		return nil
-		if err := stats.WaitForSyncComplete(ctx, api); err != nil {
+	},
-			log.Fatal(err)
+}
-		}
+
-	}
+var runCmd = &cli.Command{
-
+	Name:  "run",
-	stats.Collect(ctx, api, influx, database, height, headlag)
+	Usage: "",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "influx-database",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_DATABASE"},
+			Usage:   "influx database",
+			Value:   "",
+		},
+		&cli.StringFlag{
+			Name:    "influx-hostname",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_HOSTNAME"},
+			Value:   "http://localhost:8086",
+			Usage:   "influx hostname",
+		},
+		&cli.StringFlag{
+			Name:    "influx-username",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_USERNAME"},
+			Usage:   "influx username",
+			Value:   "",
+		},
+		&cli.StringFlag{
+			Name:    "influx-password",
+			EnvVars: []string{"LOTUS_STATS_INFLUX_PASSWORD"},
+			Usage:   "influx password",
+			Value:   "",
+		},
+		&cli.IntFlag{
+			Name:    "height",
+			EnvVars: []string{"LOTUS_STATS_HEIGHT"},
+			Usage:   "tipset height to start processing from",
+			Value:   0,
+		},
+		&cli.IntFlag{
+			Name:    "head-lag",
+			EnvVars: []string{"LOTUS_STATS_HEAD_LAG"},
+			Usage:   "the number of tipsets to delay processing on to smooth chain reorgs",
+			Value:   int(build.MessageConfidence),
+		},
+		&cli.BoolFlag{
+			Name:    "no-sync",
+			EnvVars: []string{"LOTUS_STATS_NO_SYNC"},
+			Usage:   "do not wait for chain sync to complete",
+			Value:   false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := context.Background()
+
+		resetFlag := cctx.Bool("reset")
+		noSyncFlag := cctx.Bool("no-sync")
+		heightFlag := cctx.Int("height")
+		headLagFlag := cctx.Int("head-lag")
+
+		influxAddrFlag := cctx.String("influx-addr")
+		influxUserFlag := cctx.String("influx-user")
+		influxPassFlag := cctx.String("influx-pass")
+		influxDatabaseFlag := cctx.String("influx-database")
+
+		influx, err := stats.InfluxClient(influxAddrFlag, influxUserFlag, influxPassFlag)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if resetFlag {
+			if err := stats.ResetDatabase(influx, influxDatabaseFlag); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		height := int64(heightFlag)
+
+		if !resetFlag && height == 0 {
+			h, err := stats.GetLastRecordedHeight(influx, influxDatabaseFlag)
+			if err != nil {
+				log.Info(err)
+			}
+
+			height = h
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		if !noSyncFlag {
+			if err := stats.WaitForSyncComplete(ctx, api); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		stats.Collect(ctx, api, influx, influxDatabaseFlag, height, headLagFlag)
+
+		return nil
+	},
 }

cmd/lotus-stats/setup.bash

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 
-GRAFANA_HOST="localhost:3000"
+GRAFANA_HOST="http://localhost:13000"
 
 curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF
 {
-  "name":"InfluxDB",
+  "name":"filecoin-ntwk-localstats",
   "type":"influxdb",
   "database":"lotus",
   "url": "http://influxdb:8086",