alerting: Address review

This commit is contained in:
Łukasz Magiera 2021-08-26 15:59:54 +02:00
parent 667836fbe5
commit 1ba427f638
11 changed files with 33 additions and 12 deletions

6
build/limits.go Normal file
View File

@ -0,0 +1,6 @@
package build
var (
DefaultFDLimit uint64 = 16 << 10
MinerFDLimit uint64 = 100_000
)

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -3,11 +3,14 @@ package ulimit
// from go-ipfs // from go-ipfs
import ( import (
"errors"
"fmt" "fmt"
"os" "os"
"strconv" "strconv"
"syscall" "syscall"
"github.com/filecoin-project/lotus/build"
logging "github.com/ipfs/go-log/v2" logging "github.com/ipfs/go-log/v2"
) )
@ -17,7 +20,7 @@ var (
supportsFDManagement = false supportsFDManagement = false
// getlimit returns the soft and hard limits of file descriptors counts // getlimit returns the soft and hard limits of file descriptors counts
GetLimit func() (uint64, uint64, error) getLimit func() (uint64, uint64, error)
// set limit sets the soft and hard limits of file descriptors counts // set limit sets the soft and hard limits of file descriptors counts
setLimit func(uint64, uint64) error setLimit func(uint64, uint64) error
) )
@ -25,8 +28,15 @@ var (
// minimum file descriptor limit before we complain // minimum file descriptor limit before we complain
const minFds = 2048 const minFds = 2048
// default max file descriptor limit. var ErrUnsupported = errors.New("unsupported")
const maxFds = 16 << 10
func GetLimit() (uint64, uint64, error) {
if getLimit == nil {
return 0, 0, ErrUnsupported
}
return getLimit()
}
// userMaxFDs returns the value of LOTUS_FD_MAX // userMaxFDs returns the value of LOTUS_FD_MAX
func userMaxFDs() uint64 { func userMaxFDs() uint64 {
@ -55,7 +65,7 @@ func ManageFdLimit() (changed bool, newLimit uint64, err error) {
return false, 0, nil return false, 0, nil
} }
targetLimit := uint64(maxFds) targetLimit := build.DefaultFDLimit
userLimit := userMaxFDs() userLimit := userMaxFDs()
if userLimit > 0 { if userLimit > 0 {
targetLimit = userLimit targetLimit = userLimit

View File

@ -11,7 +11,7 @@ import (
func init() { func init() {
supportsFDManagement = true supportsFDManagement = true
GetLimit = freebsdGetLimit getLimit = freebsdGetLimit
setLimit = freebsdSetLimit setLimit = freebsdSetLimit
} }

View File

@ -8,6 +8,8 @@ import (
"strings" "strings"
"syscall" "syscall"
"testing" "testing"
"github.com/filecoin-project/lotus/build"
) )
func TestManageFdLimit(t *testing.T) { func TestManageFdLimit(t *testing.T) {
@ -16,7 +18,7 @@ func TestManageFdLimit(t *testing.T) {
t.Errorf("Cannot manage file descriptors") t.Errorf("Cannot manage file descriptors")
} }
if maxFds != uint64(16<<10) { if build.DefaultFDLimit != uint64(16<<10) {
t.Errorf("Maximum file descriptors default value changed") t.Errorf("Maximum file descriptors default value changed")
} }
} }

View File

@ -8,7 +8,7 @@ import (
func init() { func init() {
supportsFDManagement = true supportsFDManagement = true
GetLimit = unixGetLimit getLimit = unixGetLimit
setLimit = unixSetLimit setLimit = unixSetLimit
} }

View File

@ -28,6 +28,7 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"golang.org/x/xerrors" "golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/beacon" "github.com/filecoin-project/lotus/chain/beacon"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/extern/sector-storage/stores"
@ -152,7 +153,7 @@ func defaults() []Option {
Override(new(journal.Journal), modules.OpenFilesystemJournal), Override(new(journal.Journal), modules.OpenFilesystemJournal),
Override(new(*alerting.Alerting), alerting.NewAlertingSystem), Override(new(*alerting.Alerting), alerting.NewAlertingSystem),
Override(CheckFDLimit, modules.CheckFdLimit(16<<10)), Override(CheckFDLimit, modules.CheckFdLimit(build.DefaultFDLimit)),
Override(new(system.MemoryConstraints), modules.MemoryConstraints), Override(new(system.MemoryConstraints), modules.MemoryConstraints),
Override(InitMemoryWatchdog, modules.MemoryWatchdog), Override(InitMemoryWatchdog, modules.MemoryWatchdog),

View File

@ -15,6 +15,7 @@ import (
storage2 "github.com/filecoin-project/specs-storage/storage" storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/gen/slashfilter" "github.com/filecoin-project/lotus/chain/gen/slashfilter"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
@ -74,7 +75,7 @@ func ConfigStorageMiner(c interface{}) Option {
return Options( return Options(
ConfigCommon(&cfg.Common, enableLibp2pNode), ConfigCommon(&cfg.Common, enableLibp2pNode),
Override(CheckFDLimit, modules.CheckFdLimit(100_000)), // recommend at least 100k FD limit to miners Override(CheckFDLimit, modules.CheckFdLimit(build.MinerFDLimit)), // recommend at least 100k FD limit to miners
Override(new(api.MinerSubsystems), modules.ExtractEnabledMinerSubsystems(cfg.Subsystems)), Override(new(api.MinerSubsystems), modules.ExtractEnabledMinerSubsystems(cfg.Subsystems)),
Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))),

View File

@ -7,13 +7,14 @@ import (
func CheckFdLimit(min uint64) func(al *alerting.Alerting) { func CheckFdLimit(min uint64) func(al *alerting.Alerting) {
return func(al *alerting.Alerting) { return func(al *alerting.Alerting) {
if ulimit.GetLimit == nil { soft, _, err := ulimit.GetLimit()
if err == ulimit.ErrUnsupported {
log.Warn("FD limit monitoring not available")
return return
} }
alert := al.AddAlertType("process", "fd-limit") alert := al.AddAlertType("process", "fd-limit")
soft, _, err := ulimit.GetLimit()
if err != nil { if err != nil {
al.Raise(alert, map[string]string{ al.Raise(alert, map[string]string{
"message": "failed to get FD limit", "message": "failed to get FD limit",