prom pkg + monitoring

This commit is contained in:
i-norden 2021-10-21 11:57:44 -05:00
parent 6a912eef41
commit 0f81dc32d2
4 changed files with 418 additions and 0 deletions

View File

@ -0,0 +1,7 @@
global:
scrape_interval: 10s
scrape_configs:
- job_name: 'eth-statediiff-service'
static_configs:
- targets: ['localhost:8100']

View File

@ -0,0 +1,162 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"database/sql"
"github.com/prometheus/client_golang/prometheus"
)
const (
namespace = "eth-statediff-service"
subsystem = "connections"
)
// DBStatsGetter is an interface that gets sql.DBStats.
type DBStatsGetter interface {
Stats() sql.DBStats
}
// DBStatsCollector implements the prometheus.Collector interface.
type DBStatsCollector struct {
sg DBStatsGetter
// descriptions of exported metrics
maxOpenDesc *prometheus.Desc
openDesc *prometheus.Desc
inUseDesc *prometheus.Desc
idleDesc *prometheus.Desc
waitedForDesc *prometheus.Desc
blockedSecondsDesc *prometheus.Desc
closedMaxIdleDesc *prometheus.Desc
closedMaxLifetimeDesc *prometheus.Desc
}
// NewDBStatsCollector creates a new DBStatsCollector.
func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
labels := prometheus.Labels{"db_name": dbName}
return &DBStatsCollector{
sg: sg,
maxOpenDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "max_open"),
"Maximum number of open connections to the database.",
nil,
labels,
),
openDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "open"),
"The number of established connections both in use and idle.",
nil,
labels,
),
inUseDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "in_use"),
"The number of connections currently in use.",
nil,
labels,
),
idleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "idle"),
"The number of idle connections.",
nil,
labels,
),
waitedForDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "waited_for"),
"The total number of connections waited for.",
nil,
labels,
),
blockedSecondsDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "blocked_seconds"),
"The total time blocked waiting for a new connection.",
nil,
labels,
),
closedMaxIdleDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "closed_max_idle"),
"The total number of connections closed due to SetMaxIdleConns.",
nil,
labels,
),
closedMaxLifetimeDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "closed_max_lifetime"),
"The total number of connections closed due to SetConnMaxLifetime.",
nil,
labels,
),
}
}
// Describe implements the prometheus.Collector interface.
func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.maxOpenDesc
ch <- c.openDesc
ch <- c.inUseDesc
ch <- c.idleDesc
ch <- c.waitedForDesc
ch <- c.blockedSecondsDesc
ch <- c.closedMaxIdleDesc
ch <- c.closedMaxLifetimeDesc
}
// Collect implements the prometheus.Collector interface.
func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
stats := c.sg.Stats()
ch <- prometheus.MustNewConstMetric(
c.maxOpenDesc,
prometheus.GaugeValue,
float64(stats.MaxOpenConnections),
)
ch <- prometheus.MustNewConstMetric(
c.openDesc,
prometheus.GaugeValue,
float64(stats.OpenConnections),
)
ch <- prometheus.MustNewConstMetric(
c.inUseDesc,
prometheus.GaugeValue,
float64(stats.InUse),
)
ch <- prometheus.MustNewConstMetric(
c.idleDesc,
prometheus.GaugeValue,
float64(stats.Idle),
)
ch <- prometheus.MustNewConstMetric(
c.waitedForDesc,
prometheus.CounterValue,
float64(stats.WaitCount),
)
ch <- prometheus.MustNewConstMetric(
c.blockedSecondsDesc,
prometheus.CounterValue,
stats.WaitDuration.Seconds(),
)
ch <- prometheus.MustNewConstMetric(
c.closedMaxIdleDesc,
prometheus.CounterValue,
float64(stats.MaxIdleClosed),
)
ch <- prometheus.MustNewConstMetric(
c.closedMaxLifetimeDesc,
prometheus.CounterValue,
float64(stats.MaxLifetimeClosed),
)
}

209
pkg/prom/prom.go Normal file
View File

@ -0,0 +1,209 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"time"
"github.com/jmoiron/sqlx"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
const statsSubsystem = "stats"
var (
metrics bool
receipts prometheus.Counter
transactions prometheus.Counter
blocks prometheus.Counter
logs prometheus.Counter
accessListEntries prometheus.Counter
lenPayloadChan prometheus.Gauge
tPayloadDecode prometheus.Histogram
tFreePostgres prometheus.Histogram
tPostgresCommit prometheus.Histogram
tHeaderProcessing prometheus.Histogram
tUncleProcessing prometheus.Histogram
tTxAndRecProcessing prometheus.Histogram
tStateAndStoreProcessing prometheus.Histogram
tCodeAndCodeHashProcessing prometheus.Histogram
)
// Init module initialization
func Init() {
metrics = true
blocks = promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "blocks",
Help: "The total number of processed blocks",
})
transactions = promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "transactions",
Help: "The total number of processed transactions",
})
receipts = promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "receipts",
Help: "The total number of processed receipts",
})
logs = promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "logs",
Help: "The total number of processed logs",
})
accessListEntries = promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "access_list_entries",
Help: "The total number of processed access list entries",
})
lenPayloadChan = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "len_payload_chan",
Help: "Current length of publishPayload",
})
tPayloadDecode = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_payload_decode",
Help: "Payload decoding time",
})
tFreePostgres = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_free_postgres",
Help: "Time spent waiting for free postgres tx",
})
tPostgresCommit = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_postgres_commit",
Help: "Postgres transaction commit duration",
})
tHeaderProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_header_processing",
Help: "Header processing time",
})
tUncleProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_uncle_processing",
Help: "Uncle processing time",
})
tTxAndRecProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_tx_receipt_processing",
Help: "Tx and receipt processing time",
})
tStateAndStoreProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_state_store_processing",
Help: "State and storage processing time",
})
tCodeAndCodeHashProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: statsSubsystem,
Name: "t_code_codehash_processing",
Help: "Code and codehash processing time",
})
}
// RegisterDBCollector create metric colletor for given connection
func RegisterDBCollector(name string, db *sqlx.DB) {
if metrics {
prometheus.Register(NewDBStatsCollector(name, db))
}
}
// BlockInc block counter increment
func BlockInc() {
if metrics {
blocks.Inc()
}
}
// TransactionInc transaction counter increment
func TransactionInc() {
if metrics {
transactions.Inc()
}
}
// ReceiptInc receipt counter increment
func ReceiptInc() {
if metrics {
receipts.Inc()
}
}
// LogInc log counter increment
func LogInc() {
if metrics {
logs.Inc()
}
}
// AccessListElementInc access list element counter increment
func AccessListElementInc() {
if metrics {
accessListEntries.Inc()
}
}
// SetLenPayloadChan set chan length
func SetLenPayloadChan(ln int) {
if metrics {
lenPayloadChan.Set(float64(ln))
}
}
// SetTimeMetric time metric observation
func SetTimeMetric(name string, t time.Duration) {
if !metrics {
return
}
tAsF64 := t.Seconds()
switch name {
case "t_payload_decode":
tPayloadDecode.Observe(tAsF64)
case "t_free_postgres":
tFreePostgres.Observe(tAsF64)
case "t_postgres_commit":
tPostgresCommit.Observe(tAsF64)
case "t_header_processing":
tHeaderProcessing.Observe(tAsF64)
case "t_uncle_processing":
tUncleProcessing.Observe(tAsF64)
case "t_tx_receipt_processing":
tTxAndRecProcessing.Observe(tAsF64)
case "t_state_store_processing":
tStateAndStoreProcessing.Observe(tAsF64)
case "t_code_codehash_processing":
tCodeAndCodeHashProcessing.Observe(tAsF64)
}
}

40
pkg/prom/web.go Normal file
View File

@ -0,0 +1,40 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package prom
import (
"net/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
)
// Listen start listening http
func Listen(addr string) *http.Server {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
srv := http.Server{
Addr: addr,
Handler: mux,
}
go func() {
if err := srv.ListenAndServe(); err != nil {
logrus.Fatal(err)
}
}()
return &srv
}