From 9bf9843ff6b5bbb50189a2c21e15ef5d22f85259 Mon Sep 17 00:00:00 2001 From: Cory Schwartz Date: Wed, 20 Apr 2022 19:48:41 -0700 Subject: [PATCH] add metrics --- cmd/lotus-gateway/main.go | 2 +- gateway/node.go | 8 ++++---- metrics/metrics.go | 11 +++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-gateway/main.go b/cmd/lotus-gateway/main.go index 7e4bc22d0..87dea40d2 100644 --- a/cmd/lotus-gateway/main.go +++ b/cmd/lotus-gateway/main.go @@ -149,7 +149,7 @@ var runCmd = &cli.Command{ // Register all metric views if err := view.Register( - metrics.ChainNodeViews..., + metrics.GatewayNodeViews..., ); err != nil { log.Fatalf("Cannot register the view: %v", err) } diff --git a/gateway/node.go b/gateway/node.go index e157645d4..dfeaea237 100644 --- a/gateway/node.go +++ b/gateway/node.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ipfs/go-cid" + "go.opencensus.io/stats" "golang.org/x/time/rate" "golang.org/x/xerrors" @@ -22,6 +23,7 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/node/impl/full" ) @@ -165,12 +167,10 @@ func (gw *Node) checkTimestamp(at time.Time) error { func (gw *Node) limit(ctx context.Context, tokens int) error { ctx2, cancel := context.WithTimeout(ctx, gw.rateLimitTimeout) defer cancel() - if !gw.rateLimiter.AllowN(time.Now(), tokens) { - return fmt.Errorf("server busy") - } err := gw.rateLimiter.WaitN(ctx2, tokens) if err != nil { - return fmt.Errorf("server busy, cannot complete before timeout %w", err) + stats.Record(ctx, metrics.RateLimitCount.M(1)) + return fmt.Errorf("server busy. %w", err) } return nil } diff --git a/metrics/metrics.go b/metrics/metrics.go index 2bbba88d3..8a4c3aa31 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -167,6 +167,9 @@ var ( RcmgrBlockSvcPeer = stats.Int64("rcmgr/block_svc", "Number of blocked blocked streams attached to a service for a specific peer", stats.UnitDimensionless) RcmgrAllowMem = stats.Int64("rcmgr/allow_mem", "Number of allowed memory reservations", stats.UnitDimensionless) RcmgrBlockMem = stats.Int64("rcmgr/block_mem", "Number of blocked memory reservations", stats.UnitDimensionless) + + // gateway rate limit + RateLimitCount = stats.Int64("ratelimit/limited", "rate limited connections", stats.UnitDimensionless) ) var ( @@ -599,6 +602,10 @@ var ( Measure: RcmgrBlockMem, Aggregation: view.Count(), } + RateLimitedView = &view.View{ + Measure: RateLimitCount, + Aggregation: view.Count(), + } ) // DefaultViews is an array of OpenCensus views for metric gathering purposes @@ -711,6 +718,10 @@ var MinerNodeViews = append([]*view.View{ DagStorePRSeekForwardBytesView, }, DefaultViews...) +var GatewayNodeViews = append([]*view.View{ + RateLimitedView, +}, ChainNodeViews...) + // SinceInMilliseconds returns the duration of time since the provide time as a float64. func SinceInMilliseconds(startTime time.Time) float64 { return float64(time.Since(startTime).Nanoseconds()) / 1e6