2018-02-23 09:56:08 +00:00
|
|
|
// Go port of Coda Hale's Metrics library
|
2015-07-07 00:54:22 +00:00
|
|
|
//
|
2018-02-23 09:56:08 +00:00
|
|
|
// <https://github.com/rcrowley/go-metrics>
|
2015-07-07 00:54:22 +00:00
|
|
|
//
|
2018-02-23 09:56:08 +00:00
|
|
|
// Coda Hale's original work: <https://github.com/codahale/metrics>
|
2015-06-27 15:12:58 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
import (
|
2015-06-29 13:11:01 +00:00
|
|
|
"os"
|
2015-06-27 15:12:58 +00:00
|
|
|
"runtime"
|
2015-06-29 13:11:01 +00:00
|
|
|
"strings"
|
2015-06-27 15:12:58 +00:00
|
|
|
"time"
|
|
|
|
|
2017-02-22 12:10:07 +00:00
|
|
|
"github.com/ethereum/go-ethereum/log"
|
2015-06-27 15:12:58 +00:00
|
|
|
)
|
|
|
|
|
2018-02-23 09:56:08 +00:00
|
|
|
// Enabled is checked by the constructor functions for all of the
|
2019-03-25 08:01:18 +00:00
|
|
|
// standard metrics. If it is true, the metric returned is a stub.
|
2018-02-23 09:56:08 +00:00
|
|
|
//
|
|
|
|
// This global kill-switch helps quantify the observer effect and makes
|
|
|
|
// for less cluttered pprof profiles.
|
2019-02-18 11:37:31 +00:00
|
|
|
var Enabled = false
|
2018-02-23 09:56:08 +00:00
|
|
|
|
2019-03-25 08:01:18 +00:00
|
|
|
// EnabledExpensive is a soft-flag meant for external packages to check if costly
|
|
|
|
// metrics gathering is allowed or not. The goal is to separate standard metrics
|
|
|
|
// for health monitoring and debug metrics that might impact runtime performance.
|
|
|
|
var EnabledExpensive = false
|
|
|
|
|
|
|
|
// enablerFlags is the CLI flag names to use to enable metrics collections.
|
|
|
|
var enablerFlags = []string{"metrics", "dashboard"}
|
|
|
|
|
|
|
|
// expensiveEnablerFlags is the CLI flag names to use to enable metrics collections.
|
|
|
|
var expensiveEnablerFlags = []string{"metrics.expensive"}
|
2015-06-29 13:11:01 +00:00
|
|
|
|
|
|
|
// Init enables or disables the metrics system. Since we need this to run before
|
|
|
|
// any other code gets to create meters and timers, we'll actually do an ugly hack
|
|
|
|
// and peek into the command line args for the metrics flag.
|
|
|
|
func init() {
|
|
|
|
for _, arg := range os.Args {
|
2019-03-25 08:01:18 +00:00
|
|
|
flag := strings.TrimLeft(arg, "-")
|
|
|
|
|
|
|
|
for _, enabler := range enablerFlags {
|
|
|
|
if !Enabled && flag == enabler {
|
|
|
|
log.Info("Enabling metrics collection")
|
|
|
|
Enabled = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, enabler := range expensiveEnablerFlags {
|
2019-03-25 08:40:46 +00:00
|
|
|
if !EnabledExpensive && flag == enabler {
|
2019-03-25 08:01:18 +00:00
|
|
|
log.Info("Enabling expensive metrics collection")
|
|
|
|
EnabledExpensive = true
|
|
|
|
}
|
2015-06-29 13:11:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-27 15:12:58 +00:00
|
|
|
// CollectProcessMetrics periodically collects various metrics about the running
|
|
|
|
// process.
|
|
|
|
func CollectProcessMetrics(refresh time.Duration) {
|
2015-06-29 13:11:01 +00:00
|
|
|
// Short circuit if the metrics system is disabled
|
2015-07-02 11:13:46 +00:00
|
|
|
if !Enabled {
|
2015-06-29 13:11:01 +00:00
|
|
|
return
|
|
|
|
}
|
2019-06-10 11:21:02 +00:00
|
|
|
refreshFreq := int64(refresh / time.Second)
|
|
|
|
|
2015-06-27 15:12:58 +00:00
|
|
|
// Create the various data collectors
|
2019-06-10 11:21:02 +00:00
|
|
|
cpuStats := make([]*CPUStats, 2)
|
2015-06-27 15:12:58 +00:00
|
|
|
memstats := make([]*runtime.MemStats, 2)
|
|
|
|
diskstats := make([]*DiskStats, 2)
|
|
|
|
for i := 0; i < len(memstats); i++ {
|
2019-06-10 11:21:02 +00:00
|
|
|
cpuStats[i] = new(CPUStats)
|
2015-06-27 15:12:58 +00:00
|
|
|
memstats[i] = new(runtime.MemStats)
|
|
|
|
diskstats[i] = new(DiskStats)
|
|
|
|
}
|
|
|
|
// Define the various metrics to collect
|
2019-06-10 11:21:02 +00:00
|
|
|
cpuSysLoad := GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry)
|
|
|
|
cpuSysWait := GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry)
|
|
|
|
cpuProcLoad := GetOrRegisterGauge("system/cpu/procload", DefaultRegistry)
|
|
|
|
|
|
|
|
memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
|
2018-02-23 09:56:08 +00:00
|
|
|
memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
|
|
|
|
memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
|
2019-06-10 11:21:02 +00:00
|
|
|
memHeld := GetOrRegisterGauge("system/memory/held", DefaultRegistry)
|
|
|
|
memUsed := GetOrRegisterGauge("system/memory/used", DefaultRegistry)
|
2015-06-27 15:12:58 +00:00
|
|
|
|
2018-02-23 09:56:08 +00:00
|
|
|
var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
|
2018-07-09 12:11:49 +00:00
|
|
|
var diskReadBytesCounter, diskWriteBytesCounter Counter
|
2015-06-27 15:12:58 +00:00
|
|
|
if err := ReadDiskStats(diskstats[0]); err == nil {
|
2018-02-23 09:56:08 +00:00
|
|
|
diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
|
|
|
|
diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
|
2018-07-09 12:11:49 +00:00
|
|
|
diskReadBytesCounter = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry)
|
2018-02-23 09:56:08 +00:00
|
|
|
diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
|
|
|
|
diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
|
2018-07-09 12:11:49 +00:00
|
|
|
diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry)
|
2015-06-27 15:12:58 +00:00
|
|
|
} else {
|
2017-03-03 09:41:52 +00:00
|
|
|
log.Debug("Failed to read disk metrics", "err", err)
|
2015-06-27 15:12:58 +00:00
|
|
|
}
|
|
|
|
// Iterate loading the different stats and updating the meters
|
|
|
|
for i := 1; ; i++ {
|
2018-06-12 11:02:36 +00:00
|
|
|
location1 := i % 2
|
|
|
|
location2 := (i - 1) % 2
|
2015-06-27 15:12:58 +00:00
|
|
|
|
2019-06-10 11:21:02 +00:00
|
|
|
ReadCPUStats(cpuStats[location1])
|
|
|
|
cpuSysLoad.Update((cpuStats[location1].GlobalTime - cpuStats[location2].GlobalTime) / refreshFreq)
|
|
|
|
cpuSysWait.Update((cpuStats[location1].GlobalWait - cpuStats[location2].GlobalWait) / refreshFreq)
|
|
|
|
cpuProcLoad.Update((cpuStats[location1].LocalTime - cpuStats[location2].LocalTime) / refreshFreq)
|
|
|
|
|
2018-06-11 11:45:25 +00:00
|
|
|
runtime.ReadMemStats(memstats[location1])
|
2019-06-10 11:21:02 +00:00
|
|
|
memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
|
2018-06-11 11:45:25 +00:00
|
|
|
memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs))
|
|
|
|
memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees))
|
2019-06-10 11:21:02 +00:00
|
|
|
memHeld.Update(int64(memstats[location1].HeapSys - memstats[location1].HeapReleased))
|
|
|
|
memUsed.Update(int64(memstats[location1].Alloc))
|
2018-06-11 11:45:25 +00:00
|
|
|
|
|
|
|
if ReadDiskStats(diskstats[location1]) == nil {
|
|
|
|
diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount)
|
|
|
|
diskReadBytes.Mark(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
|
|
|
|
diskWrites.Mark(diskstats[location1].WriteCount - diskstats[location2].WriteCount)
|
|
|
|
diskWriteBytes.Mark(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
|
2018-07-09 12:11:49 +00:00
|
|
|
|
|
|
|
diskReadBytesCounter.Inc(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes)
|
|
|
|
diskWriteBytesCounter.Inc(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes)
|
2015-06-27 15:12:58 +00:00
|
|
|
}
|
|
|
|
time.Sleep(refresh)
|
|
|
|
}
|
|
|
|
}
|