cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Configurable storage path storage limit

Browse Source
This commit is contained in:
Łukasz Magiera 2021-02-18 16:44:34 +01:00
parent 61eb91ff37
commit 37b1fb97fb
8 changed files with 121 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
cmd/lotus-storage-miner/storage.go
View File

@ -220,26 +220,66 @@ var storageListCmd = &cli.Command{
}
ping := time.Now().Sub(pingStart)
usedPercent := (st.Capacity - st.Available) * 100 / st.Capacity
percCol := color.FgGreen
switch {
case usedPercent > 98:
percCol = color.FgRed
case usedPercent > 90:
percCol = color.FgYellow
safeRepeat := func(s string, count int) string {
if count < 0 {
return ""
}
return strings.Repeat(s, count)
}
var barCols = int64(50)
set := (st.Capacity - st.Available) * barCols / st.Capacity
used := (st.Capacity - (st.Available + st.Reserved)) * barCols / st.Capacity
reserved := set - used
bar := strings.Repeat("#", int(used)) + strings.Repeat("*", int(reserved)) + strings.Repeat(" ", int(barCols-set))
fmt.Printf("\t[%s] %s/%s %s\n", color.New(percCol).Sprint(bar),
types.SizeStr(types.NewInt(uint64(st.Capacity-st.Available))),
types.SizeStr(types.NewInt(uint64(st.Capacity))),
color.New(percCol).Sprintf("%d%%", usedPercent))
// filesystem use bar
{
usedPercent := (st.Capacity - st.FSAvailable) * 100 / st.Capacity
percCol := color.FgGreen
switch {
case usedPercent > 98:
percCol = color.FgRed
case usedPercent > 90:
percCol = color.FgYellow
}
set := (st.Capacity - st.FSAvailable) * barCols / st.Capacity
used := (st.Capacity - (st.FSAvailable + st.Reserved)) * barCols / st.Capacity
reserved := set - used
bar := safeRepeat("#", int(used)) + safeRepeat("*", int(reserved)) + safeRepeat(" ", int(barCols-set))
desc := ""
if st.Max > 0 {
desc = " (filesystem)"
}
fmt.Printf("\t[%s] %s/%s %s%s\n", color.New(percCol).Sprint(bar),
types.SizeStr(types.NewInt(uint64(st.Capacity-st.FSAvailable))),
types.SizeStr(types.NewInt(uint64(st.Capacity))),
color.New(percCol).Sprintf("%d%%", usedPercent), desc)
}
// optional configured limit bar
if st.Max > 0 {
usedPercent := st.Used * 100 / st.Max
percCol := color.FgGreen
switch {
case usedPercent > 98:
percCol = color.FgRed
case usedPercent > 90:
percCol = color.FgYellow
}
set := st.Used * barCols / st.Max
used := (st.Used + st.Reserved) * barCols / st.Max
reserved := set - used
bar := safeRepeat("#", int(used)) + safeRepeat("*", int(reserved)) + safeRepeat(" ", int(barCols-set))
fmt.Printf("\t[%s] %s/%s %s (limit)\n", color.New(percCol).Sprint(bar),
types.SizeStr(types.NewInt(uint64(st.Used))),
types.SizeStr(types.NewInt(uint64(st.Max))),
color.New(percCol).Sprintf("%d%%", usedPercent))
}
fmt.Printf("\t%s; %s; %s; Reserved: %s\n",
color.YellowString("Unsealed: %d", cnt[0]),
color.GreenString("Sealed: %d", cnt[1]),

11
extern/sector-storage/fsutil/statfs.go vendored
View File

@ -1,7 +1,12 @@
package fsutil
type FsStat struct {
Capacity int64
Available int64 // Available to use for sector storage
Reserved int64
Capacity int64
Available int64 // Available to use for sector storage
FSAvailable int64 // Available in the filesystem
Reserved int64
// non-zero when storage has configured MaxStorage
Max int64
Used int64
}

6
extern/sector-storage/fsutil/statfs_unix.go vendored
View File

@ -15,7 +15,9 @@ func Statfs(path string) (FsStat, error) {
// force int64 to handle platform specific differences
//nolint:unconvert
return FsStat{
Capacity: int64(stat.Blocks) * int64(stat.Bsize),
Available: int64(stat.Bavail) * int64(stat.Bsize),
Capacity: int64(stat.Blocks) * int64(stat.Bsize),
Available: int64(stat.Bavail) * int64(stat.Bsize),
FSAvailable: int64(stat.Bavail) * int64(stat.Bsize),
}, nil
}

5
extern/sector-storage/fsutil/statfs_windows.go vendored
View File

@ -22,7 +22,8 @@ func Statfs(volumePath string) (FsStat, error) {
uintptr(unsafe.Pointer(&availBytes)))
return FsStat{
Capacity: totalBytes,
Available: availBytes,
Capacity: totalBytes,
Available: availBytes,
FSAvailable: availBytes,
}, nil
}

7
extern/sector-storage/sched_test.go vendored
View File

@ -154,9 +154,10 @@ func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name str
CanSeal: path.CanSeal,
CanStore: path.CanStore,
}, fsutil.FsStat{
Capacity: 1 << 40,
Available: 1 << 40,
Reserved: 3,
Capacity: 1 << 40,
Available: 1 << 40,
FSAvailable: 1 << 40,
Reserved: 3,
})
require.NoError(t, err)
}

8
extern/sector-storage/stores/index.go vendored
View File

@ -26,9 +26,10 @@ var SkippedHeartbeatThresh = HeartbeatInterval * 5
type ID string
type StorageInfo struct {
ID ID
URLs []string // TODO: Support non-http transports
Weight uint64
ID ID
URLs []string // TODO: Support non-http transports
Weight uint64
MaxStorage uint64
CanSeal bool
CanStore bool
@ -156,6 +157,7 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsS
}
i.stores[si.ID].info.Weight = si.Weight
i.stores[si.ID].info.MaxStorage = si.MaxStorage
i.stores[si.ID].info.CanSeal = si.CanSeal
i.stores[si.ID].info.CanStore = si.CanStore

49
extern/sector-storage/stores/local.go vendored
View File

@ -42,6 +42,10 @@ type LocalStorageMeta struct {
// Finalized sectors that will be proved over time will be stored here
CanStore bool
// MaxStorage specifies the maximum number of bytes to use for sector storage
// (0 = unlimited)
MaxStorage uint64
}
// StorageConfig .lotusstorage/storage.json
@ -77,7 +81,8 @@ type Local struct {
}
type path struct {
local string // absolute local path
local string // absolute local path
maxStorage uint64
reserved int64
reservations map[abi.SectorID]storiface.SectorFileType
@ -127,6 +132,25 @@ func (p *path) stat(ls LocalStorage) (fsutil.FsStat, error) {
stat.Available = 0
}
if p.maxStorage > 0 {
used, err := ls.DiskUsage(p.local)
if err != nil {
return fsutil.FsStat{}, err
}
stat.Max = int64(p.maxStorage)
stat.Used = used
avail := int64(p.maxStorage) - used
if uint64(used) > p.maxStorage {
avail = 0
}
if avail < stat.Available {
stat.Available = avail
}
}
return stat, err
}
@ -164,6 +188,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
out := &path{
local: p,
maxStorage: meta.MaxStorage,
reserved: 0,
reservations: map[abi.SectorID]storiface.SectorFileType{},
}
@ -174,11 +199,12 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
}
err = st.index.StorageAttach(ctx, StorageInfo{
ID: meta.ID,
URLs: st.urls,
Weight: meta.Weight,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
ID: meta.ID,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
}, fst)
if err != nil {
return xerrors.Errorf("declaring storage in index: %w", err)
@ -237,11 +263,12 @@ func (st *Local) Redeclare(ctx context.Context) error {
}
err = st.index.StorageAttach(ctx, StorageInfo{
ID: id,
URLs: st.urls,
Weight: meta.Weight,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
ID: id,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
}, fst)
if err != nil {
return xerrors.Errorf("redeclaring storage in index: %w", err)

5
extern/sector-storage/stores/local_test.go vendored
View File

@ -36,8 +36,9 @@ func (t *TestingLocalStorage) SetStorage(f func(*StorageConfig)) error {
func (t *TestingLocalStorage) Stat(path string) (fsutil.FsStat, error) {
return fsutil.FsStat{
Capacity: pathSize,
Available: pathSize,
Capacity: pathSize,
Available: pathSize,
FSAvailable: pathSize,
}, nil
}