266 lines
8.4 KiB
Go
266 lines
8.4 KiB
Go
package lp2p
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math/bits"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
|
|
logging "github.com/ipfs/go-log/v2"
|
|
"github.com/libp2p/go-libp2p"
|
|
"github.com/libp2p/go-libp2p/core/network"
|
|
"github.com/libp2p/go-libp2p/core/peer"
|
|
"github.com/libp2p/go-libp2p/core/protocol"
|
|
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"go.opencensus.io/stats"
|
|
"go.opencensus.io/tag"
|
|
"go.uber.org/fx"
|
|
|
|
"github.com/filecoin-project/lotus/metrics"
|
|
"github.com/filecoin-project/lotus/node/repo"
|
|
)
|
|
|
|
var rcmgrMetricsOnce sync.Once
|
|
|
|
func ResourceManager(connMgrHi uint) func(lc fx.Lifecycle, repo repo.LockedRepo) (network.ResourceManager, error) {
|
|
return func(lc fx.Lifecycle, repo repo.LockedRepo) (network.ResourceManager, error) {
|
|
isFullNode := repo.RepoType().Type() == "FullNode"
|
|
envvar := os.Getenv("LOTUS_RCMGR")
|
|
if (isFullNode && envvar == "0") || // only set NullResourceManager if envvar is explicitly "0"
|
|
(!isFullNode && envvar != "1") { // set NullResourceManager *unless* envvar is explicitly "1"
|
|
log.Info("libp2p resource manager is disabled")
|
|
return &network.NullResourceManager{}, nil
|
|
}
|
|
|
|
log.Info("libp2p resource manager is enabled")
|
|
// enable debug logs for rcmgr
|
|
logging.SetLogLevel("rcmgr", "debug")
|
|
|
|
// Adjust default defaultLimits
|
|
// - give it more memory, up to 4G, min of 1G
|
|
// - if maxconns are too high, adjust Conn/FD/Stream defaultLimits
|
|
defaultLimits := rcmgr.DefaultLimits
|
|
|
|
// TODO: also set appropriate default limits for lotus protocols
|
|
libp2p.SetDefaultServiceLimits(&defaultLimits)
|
|
|
|
// Minimum 1GB of memory
|
|
defaultLimits.SystemBaseLimit.Memory = 1 << 30
|
|
// For every extra 1GB of memory we have available, increase our limit by 1GiB
|
|
defaultLimits.SystemLimitIncrease.Memory = 1 << 30
|
|
defaultLimitConfig := defaultLimits.AutoScale()
|
|
|
|
changes := rcmgr.PartialLimitConfig{}
|
|
|
|
if defaultLimitConfig.ToPartialLimitConfig().System.Memory > 4<<30 {
|
|
// Cap our memory limit
|
|
changes.System.Memory = 4 << 30
|
|
}
|
|
|
|
maxconns := int(connMgrHi)
|
|
if rcmgr.LimitVal(2*maxconns) > defaultLimitConfig.ToPartialLimitConfig().System.ConnsInbound {
|
|
// adjust conns to 2x to allow for two conns per peer (TCP+QUIC)
|
|
changes.System.ConnsInbound = rcmgr.LimitVal(logScale(2 * maxconns))
|
|
changes.System.ConnsOutbound = rcmgr.LimitVal(logScale(2 * maxconns))
|
|
changes.System.Conns = rcmgr.LimitVal(logScale(4 * maxconns))
|
|
|
|
changes.System.StreamsInbound = rcmgr.LimitVal(logScale(16 * maxconns))
|
|
changes.System.StreamsOutbound = rcmgr.LimitVal(logScale(64 * maxconns))
|
|
changes.System.Streams = rcmgr.LimitVal(logScale(64 * maxconns))
|
|
|
|
if rcmgr.LimitVal(2*maxconns) > defaultLimitConfig.ToPartialLimitConfig().System.FD {
|
|
changes.System.FD = rcmgr.LimitVal(logScale(2 * maxconns))
|
|
}
|
|
|
|
changes.ServiceDefault.StreamsInbound = rcmgr.LimitVal(logScale(8 * maxconns))
|
|
changes.ServiceDefault.StreamsOutbound = rcmgr.LimitVal(logScale(32 * maxconns))
|
|
changes.ServiceDefault.Streams = rcmgr.LimitVal(logScale(32 * maxconns))
|
|
|
|
changes.ProtocolDefault.StreamsInbound = rcmgr.LimitVal(logScale(8 * maxconns))
|
|
changes.ProtocolDefault.StreamsOutbound = rcmgr.LimitVal(logScale(32 * maxconns))
|
|
changes.ProtocolDefault.Streams = rcmgr.LimitVal(logScale(32 * maxconns))
|
|
|
|
log.Info("adjusted default resource manager limits")
|
|
}
|
|
|
|
changedLimitConfig := changes.Build(defaultLimitConfig)
|
|
// initialize
|
|
var limiter rcmgr.Limiter
|
|
var opts []rcmgr.Option
|
|
|
|
repoPath := repo.Path()
|
|
|
|
// create limiter -- parse $repo/limits.json if exists
|
|
limitsFile := filepath.Join(repoPath, "limits.json")
|
|
limitsIn, err := os.Open(limitsFile)
|
|
switch {
|
|
case err == nil:
|
|
defer limitsIn.Close() //nolint:errcheck
|
|
limiter, err = rcmgr.NewLimiterFromJSON(limitsIn, changedLimitConfig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error parsing limit file: %w", err)
|
|
}
|
|
|
|
case errors.Is(err, os.ErrNotExist):
|
|
limiter = rcmgr.NewFixedLimiter(changedLimitConfig)
|
|
|
|
default:
|
|
return nil, err
|
|
}
|
|
|
|
str, err := rcmgr.NewStatsTraceReporter()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating resource manager stats reporter: %w", err)
|
|
}
|
|
|
|
rcmgrMetricsOnce.Do(func() {
|
|
rcmgr.MustRegisterWith(prometheus.DefaultRegisterer)
|
|
})
|
|
|
|
// Metrics
|
|
opts = append(opts, rcmgr.WithMetrics(rcmgrMetrics{}), rcmgr.WithTraceReporter(str))
|
|
|
|
if os.Getenv("LOTUS_DEBUG_RCMGR") != "" {
|
|
debugPath := filepath.Join(repoPath, "debug")
|
|
if err := os.MkdirAll(debugPath, 0755); err != nil {
|
|
return nil, fmt.Errorf("error creating debug directory: %w", err)
|
|
}
|
|
traceFile := filepath.Join(debugPath, "rcmgr.json.gz")
|
|
opts = append(opts, rcmgr.WithTrace(traceFile))
|
|
}
|
|
|
|
mgr, err := rcmgr.NewResourceManager(limiter, opts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating resource manager: %w", err)
|
|
}
|
|
|
|
lc.Append(fx.Hook{
|
|
OnStop: func(_ context.Context) error {
|
|
return mgr.Close()
|
|
}})
|
|
|
|
return mgr, nil
|
|
}
|
|
}
|
|
|
|
func logScale(val int) int {
|
|
bitlen := bits.Len(uint(val))
|
|
return 1 << bitlen
|
|
}
|
|
|
|
func ResourceManagerOption(mgr network.ResourceManager) Libp2pOpts {
|
|
return Libp2pOpts{
|
|
Opts: []libp2p.Option{libp2p.ResourceManager(mgr)},
|
|
}
|
|
}
|
|
|
|
type rcmgrMetrics struct{}
|
|
|
|
func (r rcmgrMetrics) AllowConn(dir network.Direction, usefd bool) {
|
|
ctx := context.Background()
|
|
if dir == network.DirInbound {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "inbound"))
|
|
} else {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "outbound"))
|
|
}
|
|
if usefd {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.UseFD, "true"))
|
|
} else {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.UseFD, "false"))
|
|
}
|
|
stats.Record(ctx, metrics.RcmgrAllowConn.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockConn(dir network.Direction, usefd bool) {
|
|
ctx := context.Background()
|
|
if dir == network.DirInbound {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "inbound"))
|
|
} else {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "outbound"))
|
|
}
|
|
if usefd {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.UseFD, "true"))
|
|
} else {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.UseFD, "false"))
|
|
}
|
|
stats.Record(ctx, metrics.RcmgrBlockConn.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) AllowStream(p peer.ID, dir network.Direction) {
|
|
ctx := context.Background()
|
|
if dir == network.DirInbound {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "inbound"))
|
|
} else {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "outbound"))
|
|
}
|
|
stats.Record(ctx, metrics.RcmgrAllowStream.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockStream(p peer.ID, dir network.Direction) {
|
|
ctx := context.Background()
|
|
if dir == network.DirInbound {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "inbound"))
|
|
} else {
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Direction, "outbound"))
|
|
}
|
|
stats.Record(ctx, metrics.RcmgrBlockStream.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) AllowPeer(p peer.ID) {
|
|
ctx := context.Background()
|
|
stats.Record(ctx, metrics.RcmgrAllowPeer.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockPeer(p peer.ID) {
|
|
ctx := context.Background()
|
|
stats.Record(ctx, metrics.RcmgrBlockPeer.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) AllowProtocol(proto protocol.ID) {
|
|
ctx := context.Background()
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.ProtocolID, string(proto)))
|
|
stats.Record(ctx, metrics.RcmgrAllowProto.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockProtocol(proto protocol.ID) {
|
|
ctx := context.Background()
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.ProtocolID, string(proto)))
|
|
stats.Record(ctx, metrics.RcmgrBlockProto.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockProtocolPeer(proto protocol.ID, p peer.ID) {
|
|
ctx := context.Background()
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.ProtocolID, string(proto)))
|
|
stats.Record(ctx, metrics.RcmgrBlockProtoPeer.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) AllowService(svc string) {
|
|
ctx := context.Background()
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.ServiceID, svc))
|
|
stats.Record(ctx, metrics.RcmgrAllowSvc.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockService(svc string) {
|
|
ctx := context.Background()
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.ServiceID, svc))
|
|
stats.Record(ctx, metrics.RcmgrBlockSvc.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockServicePeer(svc string, p peer.ID) {
|
|
ctx := context.Background()
|
|
ctx, _ = tag.New(ctx, tag.Upsert(metrics.ServiceID, svc))
|
|
stats.Record(ctx, metrics.RcmgrBlockSvcPeer.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) AllowMemory(size int) {
|
|
stats.Record(context.Background(), metrics.RcmgrAllowMem.M(1))
|
|
}
|
|
|
|
func (r rcmgrMetrics) BlockMemory(size int) {
|
|
stats.Record(context.Background(), metrics.RcmgrBlockMem.M(1))
|
|
}
|