storagemitner: manage ulimit on start

cmd/lotus-storage-miner/run.go

@@ -20,6 +20,7 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/lib/auth"
 	"github.com/filecoin-project/lotus/lib/jsonrpc"
+	"github.com/filecoin-project/lotus/lib/ulimit"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/impl"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -57,6 +58,10 @@ var runCmd = &cli.Command{
 			return err
 		}
 
+		if _, _, err := ulimit.ManageFdLimit(); err != nil {
+			log.Errorf("setting file descriptor limit: %s", err)
+		}
+
 		if v.APIVersion != build.APIVersion {
 			return xerrors.Errorf("lotus-daemon API version doesn't match: local: %s", api.Version{APIVersion: build.APIVersion})
 		}

go.mod

@@ -102,6 +102,7 @@ require (
 	go.uber.org/goleak v1.0.0 // indirect
 	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.13.0
+	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8

lib/ulimit/ulimit.go

@@ -0,0 +1,116 @@
+package ulimit
+
+// from go-ipfs
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"syscall"
+
+	logging "github.com/ipfs/go-log"
+)
+
+var log = logging.Logger("ulimit")
+
+var (
+	supportsFDManagement = false
+
+	// getlimit returns the soft and hard limits of file descriptors counts
+	getLimit func() (uint64, uint64, error)
+	// set limit sets the soft and hard limits of file descriptors counts
+	setLimit func(uint64, uint64) error
+)
+
+// minimum file descriptor limit before we complain
+const minFds = 2048
+
+// default max file descriptor limit.
+const maxFds = 16 << 10
+
+// userMaxFDs returns the value of IPFS_FD_MAX
+func userMaxFDs() uint64 {
+	// check if the IPFS_FD_MAX is set up and if it does
+	// not have a valid fds number notify the user
+	if val := os.Getenv("IPFS_FD_MAX"); val != "" {
+		fds, err := strconv.ParseUint(val, 10, 64)
+		if err != nil {
+			log.Errorf("bad value for IPFS_FD_MAX: %s", err)
+			return 0
+		}
+		return fds
+	}
+	return 0
+}
+
+// ManageFdLimit raise the current max file descriptor count
+// of the process based on the IPFS_FD_MAX value
+func ManageFdLimit() (changed bool, newLimit uint64, err error) {
+	if !supportsFDManagement {
+		return false, 0, nil
+	}
+
+	targetLimit := uint64(maxFds)
+	userLimit := userMaxFDs()
+	if userLimit > 0 {
+		targetLimit = userLimit
+	}
+
+	soft, hard, err := getLimit()
+	if err != nil {
+		return false, 0, err
+	}
+
+	if targetLimit <= soft {
+		return false, 0, nil
+	}
+
+	// the soft limit is the value that the kernel enforces for the
+	// corresponding resource
+	// the hard limit acts as a ceiling for the soft limit
+	// an unprivileged process may only set it's soft limit to a
+	// alue in the range from 0 up to the hard limit
+	err = setLimit(targetLimit, targetLimit)
+	switch err {
+	case nil:
+		newLimit = targetLimit
+	case syscall.EPERM:
+		// lower limit if necessary.
+		if targetLimit > hard {
+			targetLimit = hard
+		}
+
+		// the process does not have permission so we should only
+		// set the soft value
+		err = setLimit(targetLimit, hard)
+		if err != nil {
+			err = fmt.Errorf("error setting ulimit wihout hard limit: %s", err)
+			break
+		}
+		newLimit = targetLimit
+
+		// Warn on lowered limit.
+
+		if newLimit < userLimit {
+			err = fmt.Errorf(
+				"failed to raise ulimit to IPFS_FD_MAX (%d): set to %d",
+				userLimit,
+				newLimit,
+			)
+			break
+		}
+
+		if userLimit == 0 && newLimit < minFds {
+			err = fmt.Errorf(
+				"failed to raise ulimit to minimum %d: set to %d",
+				minFds,
+				newLimit,
+			)
+			break
+		}
+	default:
+		err = fmt.Errorf("error setting: ulimit: %s", err)
+	}
+
+	return newLimit > 0, newLimit, err
+}

lib/ulimit/ulimit_freebsd.go

@@ -0,0 +1,36 @@
+// +build freebsd
+
+package ulimit
+
+import (
+	"errors"
+	"math"
+
+	unix "golang.org/x/sys/unix"
+)
+
+func init() {
+	supportsFDManagement = true
+	getLimit = freebsdGetLimit
+	setLimit = freebsdSetLimit
+}
+
+func freebsdGetLimit() (uint64, uint64, error) {
+	rlimit := unix.Rlimit{}
+	err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
+	if (rlimit.Cur < 0) || (rlimit.Max < 0) {
+		return 0, 0, errors.New("invalid rlimits")
+	}
+	return uint64(rlimit.Cur), uint64(rlimit.Max), err
+}
+
+func freebsdSetLimit(soft uint64, max uint64) error {
+	if (soft > math.MaxInt64) || (max > math.MaxInt64) {
+		return errors.New("invalid rlimits")
+	}
+	rlimit := unix.Rlimit{
+		Cur: int64(soft),
+		Max: int64(max),
+	}
+	return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
+}

lib/ulimit/ulimit_test.go

@@ -0,0 +1,84 @@
+// +build !windows
+
+package ulimit
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+func TestManageFdLimit(t *testing.T) {
+	t.Log("Testing file descriptor count")
+	if _, _, err := ManageFdLimit(); err != nil {
+		t.Errorf("Cannot manage file descriptors")
+	}
+
+	if maxFds != uint64(16 << 10) {
+		t.Errorf("Maximum file descriptors default value changed")
+	}
+}
+
+func TestManageInvalidNFds(t *testing.T) {
+	t.Logf("Testing file descriptor invalidity")
+	var err error
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+
+	rlimit := syscall.Rlimit{}
+	if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
+		t.Fatal("Cannot get the file descriptor count")
+	}
+
+	value := rlimit.Max + rlimit.Cur
+	if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
+		t.Fatal("Cannot set the IPFS_FD_MAX env variable")
+	}
+
+	t.Logf("setting ulimit to %d, max %d, cur %d", value, rlimit.Max, rlimit.Cur)
+
+	if changed, new, err := ManageFdLimit(); err == nil {
+		t.Errorf("ManageFdLimit should return an error: changed %t, new: %d", changed, new)
+	} else if err != nil {
+		flag := strings.Contains(err.Error(),
+			"failed to raise ulimit to IPFS_FD_MAX")
+		if !flag {
+			t.Error("ManageFdLimit returned unexpected error", err)
+		}
+	}
+
+	// unset all previous operations
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+}
+
+func TestManageFdLimitWithEnvSet(t *testing.T) {
+	t.Logf("Testing file descriptor manager with IPFS_FD_MAX set")
+	var err error
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+
+	rlimit := syscall.Rlimit{}
+	if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
+		t.Fatal("Cannot get the file descriptor count")
+	}
+
+	value := rlimit.Max - rlimit.Cur + 1
+	if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
+		t.Fatal("Cannot set the IPFS_FD_MAX env variable")
+	}
+
+	if _, _, err = ManageFdLimit(); err != nil {
+		t.Errorf("Cannot manage file descriptor count")
+	}
+
+	// unset all previous operations
+	if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
+		t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
+	}
+}

lib/ulimit/ulimit_unix.go

@@ -0,0 +1,27 @@
+// +build darwin linux netbsd openbsd
+
+package ulimit
+
+import (
+unix "golang.org/x/sys/unix"
+)
+
+func init() {
+	supportsFDManagement = true
+	getLimit = unixGetLimit
+	setLimit = unixSetLimit
+}
+
+func unixGetLimit() (uint64, uint64, error) {
+	rlimit := unix.Rlimit{}
+	err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
+	return rlimit.Cur, rlimit.Max, err
+}
+
+func unixSetLimit(soft uint64, max uint64) error {
+	rlimit := unix.Rlimit{
+		Cur: soft,
+		Max: max,
+	}
+	return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
+}