From efebea7a186d6b78f7808210545f4b3565796934 Mon Sep 17 00:00:00 2001 From: Ilnur Galiev Date: Mon, 19 Oct 2020 15:07:19 +0300 Subject: [PATCH] add prometheus metrics --- cmd/root.go | 26 ++++++ go.mod | 1 + go.sum | 8 ++ pkg/prom/db_stats_collector.go | 143 +++++++++++++++++++++++++++++++++ pkg/prom/prom.go | 27 +++++++ pkg/prom/serve.go | 31 +++++++ 6 files changed, 236 insertions(+) create mode 100644 pkg/prom/db_stats_collector.go create mode 100644 pkg/prom/prom.go create mode 100644 pkg/prom/serve.go diff --git a/cmd/root.go b/cmd/root.go index fdcd87d2..608c9ebd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -24,6 +24,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/vulcanize/ipld-eth-server/pkg/prom" ) var ( @@ -62,6 +63,19 @@ func initFuncs(cmd *cobra.Command, args []string) { if err := logLevel(); err != nil { log.Fatal("Could not set log level: ", err) } + + if viper.GetBool("metrics") { + prom.Init() + } + + if viper.GetBool("http") { + addr := fmt.Sprintf( + "%s:%s", + viper.GetString("http.addr"), + viper.GetString("http.port"), + ) + prom.Serve(addr) + } } func logLevel() error { @@ -93,6 +107,12 @@ func init() { rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic") + rootCmd.PersistentFlags().Bool("metrics", false, "enable metrics") + + rootCmd.PersistentFlags().Bool("http", false, "enable http service") + rootCmd.PersistentFlags().String("http-addr", "127.0.0.1", "http host") + rootCmd.PersistentFlags().String("http-port", "8080", "http port") + viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile")) viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) @@ -100,6 +120,12 @@ func init() { viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user")) viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password")) viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) + + viper.BindPFlag("metrics", rootCmd.PersistentFlags().Lookup("metrics")) + + viper.BindPFlag("http", rootCmd.PersistentFlags().Lookup("http")) + viper.BindPFlag("http.addr", rootCmd.PersistentFlags().Lookup("http-addr")) + viper.BindPFlag("http.port", rootCmd.PersistentFlags().Lookup("http-port")) } func initConfig() { diff --git a/go.mod b/go.mod index 028423f3..00b84a37 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/multiformats/go-multihash v0.0.13 github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.10.1 + github.com/prometheus/client_golang v1.5.1 github.com/sirupsen/logrus v1.6.0 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.0 diff --git a/go.sum b/go.sum index 054a0e26..a1503247 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= @@ -703,6 +704,7 @@ github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuuj github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= @@ -827,21 +829,25 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= @@ -953,6 +959,8 @@ github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5 h1:U+BqhjRLR22e9OEm8cgW github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ= github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha h1:+XVaC7TsA0K278YWpfqdrNxwgC6hY6fBaN8w2/e1Lts= github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha/go.mod h1:SuMBscFfcBHYlQuzDDd4by+R0S3gaAFjrOU+uQfAefE= +github.com/vulcanize/ipld-eth-indexer v0.4.0-alpha h1:DXZ/hF65uegvh70YZTcPM6InMUxGkWxee6awyLcrhDg= +github.com/vulcanize/ipld-eth-indexer v0.4.0-alpha/go.mod h1:SuMBscFfcBHYlQuzDDd4by+R0S3gaAFjrOU+uQfAefE= github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha h1:Y7j0Hw1jgVVOg+eUGUr7OgH+gOBID0DwbsfZV1KoL7I= github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha/go.mod h1:OuqE4r2LGWAtDVx3s1yaAzDcwy+LEAqrWaE1L8UfrGY= github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= diff --git a/pkg/prom/db_stats_collector.go b/pkg/prom/db_stats_collector.go new file mode 100644 index 00000000..0b225899 --- /dev/null +++ b/pkg/prom/db_stats_collector.go @@ -0,0 +1,143 @@ +package prom + +import ( + "database/sql" + + "github.com/prometheus/client_golang/prometheus" +) + +const subsystem = "connections" + +// DBStatsGetter is an interface that gets sql.DBStats. +type DBStatsGetter interface { + Stats() sql.DBStats +} + +// DBStatsCollector implements the prometheus.Collector interface. +type DBStatsCollector struct { + sg DBStatsGetter + + // descriptions of exported metrics + maxOpenDesc *prometheus.Desc + openDesc *prometheus.Desc + inUseDesc *prometheus.Desc + idleDesc *prometheus.Desc + waitedForDesc *prometheus.Desc + blockedSecondsDesc *prometheus.Desc + closedMaxIdleDesc *prometheus.Desc + closedMaxLifetimeDesc *prometheus.Desc +} + +// NewDBStatsCollector creates a new DBStatsCollector. +func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector { + labels := prometheus.Labels{"db_name": dbName} + return &DBStatsCollector{ + sg: sg, + maxOpenDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "max_open"), + "Maximum number of open connections to the database.", + nil, + labels, + ), + openDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "open"), + "The number of established connections both in use and idle.", + nil, + labels, + ), + inUseDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "in_use"), + "The number of connections currently in use.", + nil, + labels, + ), + idleDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "idle"), + "The number of idle connections.", + nil, + labels, + ), + waitedForDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "waited_for"), + "The total number of connections waited for.", + nil, + labels, + ), + blockedSecondsDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "blocked_seconds"), + "The total time blocked waiting for a new connection.", + nil, + labels, + ), + closedMaxIdleDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "closed_max_idle"), + "The total number of connections closed due to SetMaxIdleConns.", + nil, + labels, + ), + closedMaxLifetimeDesc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "closed_max_lifetime"), + "The total number of connections closed due to SetConnMaxLifetime.", + nil, + labels, + ), + } +} + +// Describe implements the prometheus.Collector interface. +func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.maxOpenDesc + ch <- c.openDesc + ch <- c.inUseDesc + ch <- c.idleDesc + ch <- c.waitedForDesc + ch <- c.blockedSecondsDesc + ch <- c.closedMaxIdleDesc + ch <- c.closedMaxLifetimeDesc +} + +// Collect implements the prometheus.Collector interface. +func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) { + stats := c.sg.Stats() + + ch <- prometheus.MustNewConstMetric( + c.maxOpenDesc, + prometheus.GaugeValue, + float64(stats.MaxOpenConnections), + ) + ch <- prometheus.MustNewConstMetric( + c.openDesc, + prometheus.GaugeValue, + float64(stats.OpenConnections), + ) + ch <- prometheus.MustNewConstMetric( + c.inUseDesc, + prometheus.GaugeValue, + float64(stats.InUse), + ) + ch <- prometheus.MustNewConstMetric( + c.idleDesc, + prometheus.GaugeValue, + float64(stats.Idle), + ) + ch <- prometheus.MustNewConstMetric( + c.waitedForDesc, + prometheus.CounterValue, + float64(stats.WaitCount), + ) + ch <- prometheus.MustNewConstMetric( + c.blockedSecondsDesc, + prometheus.CounterValue, + stats.WaitDuration.Seconds(), + ) + ch <- prometheus.MustNewConstMetric( + c.closedMaxIdleDesc, + prometheus.CounterValue, + float64(stats.MaxIdleClosed), + ) + ch <- prometheus.MustNewConstMetric( + c.closedMaxLifetimeDesc, + prometheus.CounterValue, + float64(stats.MaxLifetimeClosed), + ) +} diff --git a/pkg/prom/prom.go b/pkg/prom/prom.go new file mode 100644 index 00000000..ebf95c86 --- /dev/null +++ b/pkg/prom/prom.go @@ -0,0 +1,27 @@ +package prom + +import ( + "github.com/jmoiron/sqlx" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace = "ipld_eth_server" + statsSubsystem = "stats" +) + +var ( + metrics bool +) + +// Init module initialization +func Init() { + metrics = true +} + +// RegisterDBCollector create metric colletor for given connection +func RegisterDBCollector(name string, db *sqlx.DB) { + if metrics { + prometheus.Register(NewDBStatsCollector(name, db)) + } +} diff --git a/pkg/prom/serve.go b/pkg/prom/serve.go new file mode 100644 index 00000000..058d1772 --- /dev/null +++ b/pkg/prom/serve.go @@ -0,0 +1,31 @@ +package prom + +import ( + "errors" + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" +) + +var errPromHTTP = errors.New("can't start http server for prometheus") + +// Serve start listening http +func Serve(addr string) *http.Server { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + srv := http.Server{ + Addr: addr, + Handler: mux, + } + go func() { + if err := srv.ListenAndServe(); err != nil { + logrus. + WithError(err). + WithField("module", "prom"). + WithField("addr", addr). + Fatal(errPromHTTP) + } + }() + return &srv +}