integrate memory watchdog; impose limits on badger caches.

node/builder.go

@@ -111,8 +111,10 @@ const (
 	// the system starts, so that it's available for all other components.
 	InitJournalKey = invoke(iota)
 
-	// libp2p
+	// System processes.
+	InitMemoryWatchdog
 
+	// libp2p
 	PstoreAddSelfKeysKey
 	StartListeningKey
 	BootstrapKey
@@ -174,6 +176,8 @@ func defaults() []Option {
 		Override(new(journal.DisabledEvents), journal.EnvDisabledEvents),
 		Override(new(journal.Journal), modules.OpenFilesystemJournal),
 
+		Override(InitMemoryWatchdog, modules.MemoryWatchdog),
+
 		Override(new(helpers.MetricsCtx), func() context.Context {
 			return metricsi.CtxScope(context.Background(), "lotus")
 		}),

node/modules/core.go

@@ -6,12 +6,14 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"time"
 
 	"github.com/gbrlsnchs/jwt/v3"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/peerstore"
 	record "github.com/libp2p/go-libp2p-record"
+	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
@@ -24,9 +26,19 @@ import (
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/lotus/system"
+	"github.com/raulk/go-watchdog"
 )
 
-var log = logging.Logger("modules")
+const (
+	JWTSecretName   = "auth-jwt-private" //nolint:gosec
+	KTJwtHmacSecret = "jwt-hmac-secret"  //nolint:gosec
+)
+
+var (
+	log         = logging.Logger("modules")
+	logWatchdog = logging.Logger("watchdog")
+)
 
 type Genesis func() (*types.BlockHeader, error)
 
@@ -37,8 +49,44 @@ func RecordValidator(ps peerstore.Peerstore) record.Validator {
 	}
 }
 
-const JWTSecretName = "auth-jwt-private"  //nolint:gosec
-const KTJwtHmacSecret = "jwt-hmac-secret" //nolint:gosec
+// MemoryWatchdog starts the memory watchdog, applying the computed resource
+// constraints.
+func MemoryWatchdog(lc fx.Lifecycle) {
+	cfg := watchdog.MemConfig{
+		Resolution: 10 * time.Second,
+		Policy: &watchdog.WatermarkPolicy{
+			Watermarks:         []float64{0.50, 0.60, 0.70, 0.85, 0.90, 0.925, 0.95},
+			EmergencyWatermark: 0.95,
+			Silence:            45 * time.Second, // avoid forced GC within 45 seconds of previous GC.
+		},
+		Logger: logWatchdog,
+	}
+
+	// if user has set max heap limit, apply it. Otherwise, fall back to total
+	// system memory constraint.
+	if maxHeap := system.ResourceConstraints.MaxHeapMem; maxHeap != 0 {
+		log.Infof("memory watchdog will apply max heap constraint: %d bytes", maxHeap)
+		cfg.Limit = maxHeap
+		cfg.Scope = watchdog.ScopeHeap
+	} else {
+		log.Infof("max heap size not provided; memory watchdog will apply total system memory constraint: %d bytes", system.ResourceConstraints.TotalSystemMem)
+		cfg.Limit = system.ResourceConstraints.TotalSystemMem
+		cfg.Scope = watchdog.ScopeSystem
+	}
+
+	err, stop := watchdog.Memory(cfg)
+	if err != nil {
+		log.Warnf("failed to instantiate memory watchdog: %s", err)
+		return
+	}
+
+	lc.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			stop()
+			return nil
+		},
+	})
+}
 
 type JwtPayload struct {
 	Allow []auth.Permission

node/repo/blockstore_opts.go

@@ -1,6 +1,9 @@
 package repo
 
-import badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
+import (
+	badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
+	"github.com/filecoin-project/lotus/system"
+)
 
 // BadgerBlockstoreOptions returns the badger options to apply for the provided
 // domain.
@@ -41,10 +44,21 @@ func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool
 	// Default table size is already 64MiB. This is here to make it explicit.
 	opts.MaxTableSize = 64 << 20
 
-	// IndexCacheSize is 1GiB. If we don't set an index cache size, badger will
-	// retain all indices from all tables _in memory_. This is quite counter
-	// intuitive, but it's true. See badger/table.Table#initIndex.
-	opts.IndexCacheSize = 1 << 30
+	// If we don't set an index cache size, badger will retain all indices from
+	// all tables _in memory_. This is quite counter intuitive, but it's true.
+	// See badger/table.Table#initIndex.
+	//
+	// We vary the cache size depending on the configured system limits, taking
+	// up to 20% of the configured limit, with a min of 256MiB, and a max
+	// of 1GiB.
+	var icachesize int64
+	switch icachesize = int64(float64(system.ResourceConstraints.EffectiveMemLimit) * 0.20); {
+	case icachesize < 256<<20: // 256MiB.
+		icachesize = 256 << 20
+	case icachesize > 1<<30: // 1GiB.
+		icachesize = 1 << 30
+	}
+	opts.IndexCacheSize = icachesize
 
 	// NOTE: The chain blockstore doesn't require any GC (blocks are never
 	// deleted). This will change if we move to a tiered blockstore.