Merge pull request #5624 from filecoin-project/feat/miner-storage-limit

Configurable storage path storage limit

cmd/lotus-seal-worker/storage.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/docker/go-units"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
@@ -46,6 +47,10 @@ var storageAttachCmd = &cli.Command{
 			Name:  "store",
 			Usage: "(for init) use path for long-term storage",
 		},
+		&cli.StringFlag{
+			Name:  "max-storage",
+			Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
@@ -79,11 +84,20 @@ var storageAttachCmd = &cli.Command{
 				return err
 			}
 
+			var maxStor int64
+			if cctx.IsSet("max-storage") {
+				maxStor, err = units.RAMInBytes(cctx.String("max-storage"))
+				if err != nil {
+					return xerrors.Errorf("parsing max-storage: %w", err)
+				}
+			}
+
 			cfg := &stores.LocalStorageMeta{
-				ID:       stores.ID(uuid.New().String()),
+				ID:         stores.ID(uuid.New().String()),
-				Weight:   cctx.Uint64("weight"),
+				Weight:     cctx.Uint64("weight"),
-				CanSeal:  cctx.Bool("seal"),
+				CanSeal:    cctx.Bool("seal"),
-				CanStore: cctx.Bool("store"),
+				CanStore:   cctx.Bool("store"),
+				MaxStorage: uint64(maxStor),
 			}
 
 			if !(cfg.CanStore || cfg.CanSeal) {

cmd/lotus-storage-miner/storage.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/docker/go-units"
 	"github.com/fatih/color"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
@@ -88,6 +89,10 @@ over time
 			Name:  "store",
 			Usage: "(for init) use path for long-term storage",
 		},
+		&cli.StringFlag{
+			Name:  "max-storage",
+			Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
@@ -121,11 +126,20 @@ over time
 				return err
 			}
 
+			var maxStor int64
+			if cctx.IsSet("max-storage") {
+				maxStor, err = units.RAMInBytes(cctx.String("max-storage"))
+				if err != nil {
+					return xerrors.Errorf("parsing max-storage: %w", err)
+				}
+			}
+
 			cfg := &stores.LocalStorageMeta{
-				ID:       stores.ID(uuid.New().String()),
+				ID:         stores.ID(uuid.New().String()),
-				Weight:   cctx.Uint64("weight"),
+				Weight:     cctx.Uint64("weight"),
-				CanSeal:  cctx.Bool("seal"),
+				CanSeal:    cctx.Bool("seal"),
-				CanStore: cctx.Bool("store"),
+				CanStore:   cctx.Bool("store"),
+				MaxStorage: uint64(maxStor),
 			}
 
 			if !(cfg.CanStore || cfg.CanSeal) {
@@ -220,26 +234,66 @@ var storageListCmd = &cli.Command{
 			}
 			ping := time.Now().Sub(pingStart)
 
-			usedPercent := (st.Capacity - st.Available) * 100 / st.Capacity
+			safeRepeat := func(s string, count int) string {
-
+				if count < 0 {
-			percCol := color.FgGreen
+					return ""
-			switch {
+				}
-			case usedPercent > 98:
+				return strings.Repeat(s, count)
-				percCol = color.FgRed
-			case usedPercent > 90:
-				percCol = color.FgYellow
 			}
 
 			var barCols = int64(50)
-			set := (st.Capacity - st.Available) * barCols / st.Capacity
-			used := (st.Capacity - (st.Available + st.Reserved)) * barCols / st.Capacity
-			reserved := set - used
-			bar := strings.Repeat("#", int(used)) + strings.Repeat("*", int(reserved)) + strings.Repeat(" ", int(barCols-set))
 
-			fmt.Printf("\t[%s] %s/%s %s\n", color.New(percCol).Sprint(bar),
+			// filesystem use bar
-				types.SizeStr(types.NewInt(uint64(st.Capacity-st.Available))),
+			{
-				types.SizeStr(types.NewInt(uint64(st.Capacity))),
+				usedPercent := (st.Capacity - st.FSAvailable) * 100 / st.Capacity
-				color.New(percCol).Sprintf("%d%%", usedPercent))
+
+				percCol := color.FgGreen
+				switch {
+				case usedPercent > 98:
+					percCol = color.FgRed
+				case usedPercent > 90:
+					percCol = color.FgYellow
+				}
+
+				set := (st.Capacity - st.FSAvailable) * barCols / st.Capacity
+				used := (st.Capacity - (st.FSAvailable + st.Reserved)) * barCols / st.Capacity
+				reserved := set - used
+				bar := safeRepeat("#", int(used)) + safeRepeat("*", int(reserved)) + safeRepeat(" ", int(barCols-set))
+
+				desc := ""
+				if st.Max > 0 {
+					desc = " (filesystem)"
+				}
+
+				fmt.Printf("\t[%s] %s/%s %s%s\n", color.New(percCol).Sprint(bar),
+					types.SizeStr(types.NewInt(uint64(st.Capacity-st.FSAvailable))),
+					types.SizeStr(types.NewInt(uint64(st.Capacity))),
+					color.New(percCol).Sprintf("%d%%", usedPercent), desc)
+			}
+
+			// optional configured limit bar
+			if st.Max > 0 {
+				usedPercent := st.Used * 100 / st.Max
+
+				percCol := color.FgGreen
+				switch {
+				case usedPercent > 98:
+					percCol = color.FgRed
+				case usedPercent > 90:
+					percCol = color.FgYellow
+				}
+
+				set := st.Used * barCols / st.Max
+				used := (st.Used + st.Reserved) * barCols / st.Max
+				reserved := set - used
+				bar := safeRepeat("#", int(used)) + safeRepeat("*", int(reserved)) + safeRepeat(" ", int(barCols-set))
+
+				fmt.Printf("\t[%s] %s/%s %s (limit)\n", color.New(percCol).Sprint(bar),
+					types.SizeStr(types.NewInt(uint64(st.Used))),
+					types.SizeStr(types.NewInt(uint64(st.Max))),
+					color.New(percCol).Sprintf("%d%%", usedPercent))
+			}
+
 			fmt.Printf("\t%s; %s; %s; Reserved: %s\n",
 				color.YellowString("Unsealed: %d", cnt[0]),
 				color.GreenString("Sealed: %d", cnt[1]),

documentation/en/api-methods-miner.md

@@ -1812,13 +1812,17 @@ Inputs:
     "ID": "76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8",
     "URLs": null,
     "Weight": 42,
+    "MaxStorage": 42,
     "CanSeal": true,
     "CanStore": true
   },
   {
     "Capacity": 9,
     "Available": 9,
-    "Reserved": 9
+    "FSAvailable": 9,
+    "Reserved": 9,
+    "Max": 9,
+    "Used": 9
   }
 ]
 ```
@@ -1918,6 +1922,7 @@ Response:
   "ID": "76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8",
   "URLs": null,
   "Weight": 42,
+  "MaxStorage": 42,
   "CanSeal": true,
   "CanStore": true
 }
@@ -1989,7 +1994,10 @@ Inputs:
     "Stat": {
       "Capacity": 9,
       "Available": 9,
-      "Reserved": 9
+      "FSAvailable": 9,
+      "Reserved": 9,
+      "Max": 9,
+      "Used": 9
     },
     "Err": "string value"
   }
@@ -2015,7 +2023,10 @@ Response:
 {
   "Capacity": 9,
   "Available": 9,
-  "Reserved": 9
+  "FSAvailable": 9,
+  "Reserved": 9,
+  "Max": 9,
+  "Used": 9
 }
 ```
 

extern/sector-storage/fsutil/statfs.go

@@ -1,7 +1,12 @@
 package fsutil
 
 type FsStat struct {
-	Capacity  int64
+	Capacity    int64
-	Available int64 // Available to use for sector storage
+	Available   int64 // Available to use for sector storage
-	Reserved  int64
+	FSAvailable int64 // Available in the filesystem
+	Reserved    int64
+
+	// non-zero when storage has configured MaxStorage
+	Max  int64
+	Used int64
 }

extern/sector-storage/fsutil/statfs_unix.go

@@ -15,7 +15,9 @@ func Statfs(path string) (FsStat, error) {
 	// force int64 to handle platform specific differences
 	//nolint:unconvert
 	return FsStat{
-		Capacity:  int64(stat.Blocks) * int64(stat.Bsize),
+		Capacity: int64(stat.Blocks) * int64(stat.Bsize),
-		Available: int64(stat.Bavail) * int64(stat.Bsize),
+
+		Available:   int64(stat.Bavail) * int64(stat.Bsize),
+		FSAvailable: int64(stat.Bavail) * int64(stat.Bsize),
 	}, nil
 }

extern/sector-storage/fsutil/statfs_windows.go

@@ -22,7 +22,8 @@ func Statfs(volumePath string) (FsStat, error) {
 		uintptr(unsafe.Pointer(&availBytes)))
 
 	return FsStat{
-		Capacity:  totalBytes,
+		Capacity:    totalBytes,
-		Available: availBytes,
+		Available:   availBytes,
+		FSAvailable: availBytes,
 	}, nil
 }

extern/sector-storage/sched_test.go

@@ -154,9 +154,10 @@ func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name str
 			CanSeal:  path.CanSeal,
 			CanStore: path.CanStore,
 		}, fsutil.FsStat{
-			Capacity:  1 << 40,
+			Capacity:    1 << 40,
-			Available: 1 << 40,
+			Available:   1 << 40,
-			Reserved:  3,
+			FSAvailable: 1 << 40,
+			Reserved:    3,
 		})
 		require.NoError(t, err)
 	}

extern/sector-storage/stores/index.go

@@ -26,9 +26,10 @@ var SkippedHeartbeatThresh = HeartbeatInterval * 5
 type ID string
 
 type StorageInfo struct {
-	ID     ID
+	ID         ID
-	URLs   []string // TODO: Support non-http transports
+	URLs       []string // TODO: Support non-http transports
-	Weight uint64
+	Weight     uint64
+	MaxStorage uint64
 
 	CanSeal  bool
 	CanStore bool
@@ -156,6 +157,7 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsS
 		}
 
 		i.stores[si.ID].info.Weight = si.Weight
+		i.stores[si.ID].info.MaxStorage = si.MaxStorage
 		i.stores[si.ID].info.CanSeal = si.CanSeal
 		i.stores[si.ID].info.CanStore = si.CanStore
 

extern/sector-storage/stores/local.go

@@ -42,6 +42,10 @@ type LocalStorageMeta struct {
 
 	// Finalized sectors that will be proved over time will be stored here
 	CanStore bool
+
+	// MaxStorage specifies the maximum number of bytes to use for sector storage
+	// (0 = unlimited)
+	MaxStorage uint64
 }
 
 // StorageConfig .lotusstorage/storage.json
@@ -77,7 +81,8 @@ type Local struct {
 }
 
 type path struct {
-	local string // absolute local path
+	local      string // absolute local path
+	maxStorage uint64
 
 	reserved     int64
 	reservations map[abi.SectorID]storiface.SectorFileType
@@ -127,6 +132,25 @@ func (p *path) stat(ls LocalStorage) (fsutil.FsStat, error) {
 		stat.Available = 0
 	}
 
+	if p.maxStorage > 0 {
+		used, err := ls.DiskUsage(p.local)
+		if err != nil {
+			return fsutil.FsStat{}, err
+		}
+
+		stat.Max = int64(p.maxStorage)
+		stat.Used = used
+
+		avail := int64(p.maxStorage) - used
+		if uint64(used) > p.maxStorage {
+			avail = 0
+		}
+
+		if avail < stat.Available {
+			stat.Available = avail
+		}
+	}
+
 	return stat, err
 }
 
@@ -164,6 +188,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
 	out := &path{
 		local: p,
 
+		maxStorage:   meta.MaxStorage,
 		reserved:     0,
 		reservations: map[abi.SectorID]storiface.SectorFileType{},
 	}
@@ -174,11 +199,12 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
 	}
 
 	err = st.index.StorageAttach(ctx, StorageInfo{
-		ID:       meta.ID,
+		ID:         meta.ID,
-		URLs:     st.urls,
+		URLs:       st.urls,
-		Weight:   meta.Weight,
+		Weight:     meta.Weight,
-		CanSeal:  meta.CanSeal,
+		MaxStorage: meta.MaxStorage,
-		CanStore: meta.CanStore,
+		CanSeal:    meta.CanSeal,
+		CanStore:   meta.CanStore,
 	}, fst)
 	if err != nil {
 		return xerrors.Errorf("declaring storage in index: %w", err)
@@ -237,11 +263,12 @@ func (st *Local) Redeclare(ctx context.Context) error {
 		}
 
 		err = st.index.StorageAttach(ctx, StorageInfo{
-			ID:       id,
+			ID:         id,
-			URLs:     st.urls,
+			URLs:       st.urls,
-			Weight:   meta.Weight,
+			Weight:     meta.Weight,
-			CanSeal:  meta.CanSeal,
+			MaxStorage: meta.MaxStorage,
-			CanStore: meta.CanStore,
+			CanSeal:    meta.CanSeal,
+			CanStore:   meta.CanStore,
 		}, fst)
 		if err != nil {
 			return xerrors.Errorf("redeclaring storage in index: %w", err)

extern/sector-storage/stores/local_test.go

@@ -36,8 +36,9 @@ func (t *TestingLocalStorage) SetStorage(f func(*StorageConfig)) error {
 
 func (t *TestingLocalStorage) Stat(path string) (fsutil.FsStat, error) {
 	return fsutil.FsStat{
-		Capacity:  pathSize,
+		Capacity:    pathSize,
-		Available: pathSize,
+		Available:   pathSize,
+		FSAvailable: pathSize,
 	}, nil
 }