From 656ece06e588da63bd948ae309c297ffe09ffb90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= <raul@protocol.ai>
Date: Mon, 2 Nov 2020 17:56:49 +0000
Subject: [PATCH] fix metrics wiring.

Some components like go-ds-measure, go-ipfs-blockstore and go-bitswap
expose metrics via ipfs/go-metrics-interface, but Lotus never injects
the Prometheus exporter (ipfs/go-metrics-prometheus). Therefore, those
metrics never surface in instrumentation.

Instead, Lotus uses OpenCensus directly.

This commit injects the Prometheus exporter for go-metrics-interface, and
instructs the OpenCensus Prometheus exporter to use the DefaultRegistry.

This has the effect of exposing blending the metrics of both metrics
libraries.

With this patch, the datastore, cache utilisation, and bitswap metrics
are now exported via the /debug/metrics endpoint.

This commit also fixes an issue where the metrics scope was empty, making
go-metrics-interface default to "<no-scope>". Angle brackets are inadmissible
characters for Prometheus, so it was refusing to export the affected metrics.
(These were the ARC cache metrics.)
---
 cmd/lotus/daemon.go |  9 ++++++++-
 cmd/lotus/rpc.go    | 10 ++++++++++
 go.mod              |  1 +
 node/builder.go     |  7 ++++++-
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go
index 4ff63be11..7d078407a 100644
--- a/cmd/lotus/daemon.go
+++ b/cmd/lotus/daemon.go
@@ -16,6 +16,7 @@ import (
 	"strings"
 
 	paramfetch "github.com/filecoin-project/go-paramfetch"
+	metricsprom "github.com/ipfs/go-metrics-prometheus"
 	"github.com/mitchellh/go-homedir"
 	"github.com/multiformats/go-multiaddr"
 	"github.com/urfave/cli/v2"
@@ -262,8 +263,14 @@ var DaemonCmd = &cli.Command{
 			liteModeDeps = node.Override(new(api.GatewayAPI), gapi)
 		}
 
-		var api api.FullNode
+		// some libraries like ipfs/go-ds-measure and ipfs/go-ipfs-blockstore
+		// use ipfs/go-metrics-interface. This injects a Prometheus exporter
+		// for those. Metrics are exported to the default registry.
+		if err := metricsprom.Inject(); err != nil {
+			log.Warnf("unable to inject prometheus ipfs/go-metrics exporter; some metrics will be unavailable; err: %s", err)
+		}
 
+		var api api.FullNode
 		stop, err := node.New(ctx,
 			node.FullAPI(&api, node.Lite(isLite)),
 
diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go
index 4f68ac85a..f2c59b615 100644
--- a/cmd/lotus/rpc.go
+++ b/cmd/lotus/rpc.go
@@ -14,6 +14,7 @@ import (
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
+	promclient "github.com/prometheus/client_golang/prometheus"
 	"go.opencensus.io/tag"
 	"golang.org/x/xerrors"
 
@@ -49,7 +50,16 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shut
 
 	http.Handle("/rest/v0/import", importAH)
 
+	// Prometheus globals are exposed as interfaces, but the prometheus
+	// OpenCensus exporter expects a concrete *Registry. The concrete type of
+	// the globals are actually *Registry, so we downcast them, staying
+	// defensive in case things change under the hood.
+	registry, ok := promclient.DefaultRegisterer.(*promclient.Registry)
+	if !ok {
+		log.Warnf("failed to export default prometheus registry; some metrics will be unavailable; unexpected type: %T", promclient.DefaultRegisterer)
+	}
 	exporter, err := prometheus.NewExporter(prometheus.Options{
+		Registry:  registry,
 		Namespace: "lotus",
 	})
 	if err != nil {
diff --git a/go.mod b/go.mod
index 723640930..c4787ac9f 100644
--- a/go.mod
+++ b/go.mod
@@ -81,6 +81,7 @@ require (
 	github.com/ipfs/go-log v1.0.4
 	github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
 	github.com/ipfs/go-merkledag v0.3.2
+	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
 	github.com/ipfs/go-path v0.0.7
 	github.com/ipfs/go-unixfs v0.2.4
diff --git a/node/builder.go b/node/builder.go
index 05409df04..d797ebb2f 100644
--- a/node/builder.go
+++ b/node/builder.go
@@ -5,6 +5,8 @@ import (
 	"errors"
 	"time"
 
+	metricsi "github.com/ipfs/go-metrics-interface"
+
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/exchange"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -167,7 +169,10 @@ func defaults() []Option {
 		Override(new(journal.DisabledEvents), journal.EnvDisabledEvents),
 		Override(new(journal.Journal), modules.OpenFilesystemJournal),
 
-		Override(new(helpers.MetricsCtx), context.Background),
+		Override(new(helpers.MetricsCtx), func() context.Context {
+			return metricsi.CtxScope(context.Background(), "lotus")
+		}),
+
 		Override(new(record.Validator), modules.RecordValidator),
 		Override(new(dtypes.Bootstrapper), dtypes.Bootstrapper(false)),
 		Override(new(dtypes.ShutdownChan), make(chan struct{})),