From df9aa0e3576fe95c089d7c07a02fc66cae461a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 22 Mar 2020 22:08:22 +0100 Subject: [PATCH] storagemitner: manage ulimit on start --- cmd/lotus-storage-miner/run.go | 5 ++ go.mod | 1 + lib/ulimit/ulimit.go | 116 +++++++++++++++++++++++++++++++++ lib/ulimit/ulimit_freebsd.go | 36 ++++++++++ lib/ulimit/ulimit_test.go | 84 ++++++++++++++++++++++++ lib/ulimit/ulimit_unix.go | 27 ++++++++ 6 files changed, 269 insertions(+) create mode 100644 lib/ulimit/ulimit.go create mode 100644 lib/ulimit/ulimit_freebsd.go create mode 100644 lib/ulimit/ulimit_test.go create mode 100644 lib/ulimit/ulimit_unix.go diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 1454d8b42..897e60f27 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -20,6 +20,7 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/auth" "github.com/filecoin-project/lotus/lib/jsonrpc" + "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -57,6 +58,10 @@ var runCmd = &cli.Command{ return err } + if _, _, err := ulimit.ManageFdLimit(); err != nil { + log.Errorf("setting file descriptor limit: %s", err) + } + if v.APIVersion != build.APIVersion { return xerrors.Errorf("lotus-daemon API version doesn't match: local: %s", api.Version{APIVersion: build.APIVersion}) } diff --git a/go.mod b/go.mod index 3b16a5b64..2c99a3f5b 100644 --- a/go.mod +++ b/go.mod @@ -102,6 +102,7 @@ require ( go.uber.org/goleak v1.0.0 // indirect go.uber.org/multierr v1.4.0 go.uber.org/zap v1.13.0 + golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 diff --git a/lib/ulimit/ulimit.go b/lib/ulimit/ulimit.go new file mode 100644 index 000000000..5686a0f9a --- /dev/null +++ b/lib/ulimit/ulimit.go @@ -0,0 +1,116 @@ +package ulimit + +// from go-ipfs + +import ( + "fmt" + "os" + "strconv" + "syscall" + + logging "github.com/ipfs/go-log" +) + +var log = logging.Logger("ulimit") + +var ( + supportsFDManagement = false + + // getlimit returns the soft and hard limits of file descriptors counts + getLimit func() (uint64, uint64, error) + // set limit sets the soft and hard limits of file descriptors counts + setLimit func(uint64, uint64) error +) + +// minimum file descriptor limit before we complain +const minFds = 2048 + +// default max file descriptor limit. +const maxFds = 16 << 10 + +// userMaxFDs returns the value of IPFS_FD_MAX +func userMaxFDs() uint64 { + // check if the IPFS_FD_MAX is set up and if it does + // not have a valid fds number notify the user + if val := os.Getenv("IPFS_FD_MAX"); val != "" { + fds, err := strconv.ParseUint(val, 10, 64) + if err != nil { + log.Errorf("bad value for IPFS_FD_MAX: %s", err) + return 0 + } + return fds + } + return 0 +} + +// ManageFdLimit raise the current max file descriptor count +// of the process based on the IPFS_FD_MAX value +func ManageFdLimit() (changed bool, newLimit uint64, err error) { + if !supportsFDManagement { + return false, 0, nil + } + + targetLimit := uint64(maxFds) + userLimit := userMaxFDs() + if userLimit > 0 { + targetLimit = userLimit + } + + soft, hard, err := getLimit() + if err != nil { + return false, 0, err + } + + if targetLimit <= soft { + return false, 0, nil + } + + // the soft limit is the value that the kernel enforces for the + // corresponding resource + // the hard limit acts as a ceiling for the soft limit + // an unprivileged process may only set it's soft limit to a + // alue in the range from 0 up to the hard limit + err = setLimit(targetLimit, targetLimit) + switch err { + case nil: + newLimit = targetLimit + case syscall.EPERM: + // lower limit if necessary. + if targetLimit > hard { + targetLimit = hard + } + + // the process does not have permission so we should only + // set the soft value + err = setLimit(targetLimit, hard) + if err != nil { + err = fmt.Errorf("error setting ulimit wihout hard limit: %s", err) + break + } + newLimit = targetLimit + + // Warn on lowered limit. + + if newLimit < userLimit { + err = fmt.Errorf( + "failed to raise ulimit to IPFS_FD_MAX (%d): set to %d", + userLimit, + newLimit, + ) + break + } + + if userLimit == 0 && newLimit < minFds { + err = fmt.Errorf( + "failed to raise ulimit to minimum %d: set to %d", + minFds, + newLimit, + ) + break + } + default: + err = fmt.Errorf("error setting: ulimit: %s", err) + } + + return newLimit > 0, newLimit, err +} diff --git a/lib/ulimit/ulimit_freebsd.go b/lib/ulimit/ulimit_freebsd.go new file mode 100644 index 000000000..7e50436f3 --- /dev/null +++ b/lib/ulimit/ulimit_freebsd.go @@ -0,0 +1,36 @@ +// +build freebsd + +package ulimit + +import ( + "errors" + "math" + + unix "golang.org/x/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = freebsdGetLimit + setLimit = freebsdSetLimit +} + +func freebsdGetLimit() (uint64, uint64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + if (rlimit.Cur < 0) || (rlimit.Max < 0) { + return 0, 0, errors.New("invalid rlimits") + } + return uint64(rlimit.Cur), uint64(rlimit.Max), err +} + +func freebsdSetLimit(soft uint64, max uint64) error { + if (soft > math.MaxInt64) || (max > math.MaxInt64) { + return errors.New("invalid rlimits") + } + rlimit := unix.Rlimit{ + Cur: int64(soft), + Max: int64(max), + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} diff --git a/lib/ulimit/ulimit_test.go b/lib/ulimit/ulimit_test.go new file mode 100644 index 000000000..b09f957a4 --- /dev/null +++ b/lib/ulimit/ulimit_test.go @@ -0,0 +1,84 @@ +// +build !windows + +package ulimit + +import ( + "fmt" + "os" + "strings" + "syscall" + "testing" +) + +func TestManageFdLimit(t *testing.T) { + t.Log("Testing file descriptor count") + if _, _, err := ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptors") + } + + if maxFds != uint64(16 << 10) { + t.Errorf("Maximum file descriptors default value changed") + } +} + +func TestManageInvalidNFds(t *testing.T) { + t.Logf("Testing file descriptor invalidity") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max + rlimit.Cur + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + t.Logf("setting ulimit to %d, max %d, cur %d", value, rlimit.Max, rlimit.Cur) + + if changed, new, err := ManageFdLimit(); err == nil { + t.Errorf("ManageFdLimit should return an error: changed %t, new: %d", changed, new) + } else if err != nil { + flag := strings.Contains(err.Error(), + "failed to raise ulimit to IPFS_FD_MAX") + if !flag { + t.Error("ManageFdLimit returned unexpected error", err) + } + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} + +func TestManageFdLimitWithEnvSet(t *testing.T) { + t.Logf("Testing file descriptor manager with IPFS_FD_MAX set") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max - rlimit.Cur + 1 + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + if _, _, err = ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptor count") + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} \ No newline at end of file diff --git a/lib/ulimit/ulimit_unix.go b/lib/ulimit/ulimit_unix.go new file mode 100644 index 000000000..e1e90c7eb --- /dev/null +++ b/lib/ulimit/ulimit_unix.go @@ -0,0 +1,27 @@ +// +build darwin linux netbsd openbsd + +package ulimit + +import ( +unix "golang.org/x/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = unixGetLimit + setLimit = unixSetLimit +} + +func unixGetLimit() (uint64, uint64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + return rlimit.Cur, rlimit.Max, err +} + +func unixSetLimit(soft uint64, max uint64) error { + rlimit := unix.Rlimit{ + Cur: soft, + Max: max, + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +}