cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

storage: Path type filters

Browse Source
This commit is contained in:
Łukasz Magiera 2022-07-01 18:02:10 +02:00
parent e7c8082956
commit 6ac5c16d2b
12 changed files with 286 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
api/api_storage.go
View File

@ -146,18 +146,29 @@ type StorageMiner interface {
SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
SealingAbort(ctx context.Context, call storiface.CallID) error //perm:admin
// SectorIndex
StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error //perm:admin
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error //perm:admin
StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
// paths.SectorIndex
StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error //perm:admin
StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error) //perm:admin
StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error //perm:admin
StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error //perm:admin
// StorageFindSector returns list of paths where the specified sector files exist.
//
// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
// - Paths which have sector files locally (don't require fetching) will be listed first.
// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
// file types are specified, each type will be considered individually, and a union of all paths
// which can accommodate each file type will be returned.
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
// Paths with more weight and more % of free space are preferred.
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

39
documentation/en/api-v0-methods-miner.md
View File

@ -3271,7 +3271,7 @@ Inputs:
Response: `{}`
### StorageAttach
SectorIndex
paths.SectorIndex
Perms: admin
@ -3293,6 +3293,12 @@ Inputs:
],
"AllowTo": [
"string value"
],
"AllowTypes": [
"string value"
],
"DenyTypes": [
"string value"
]
},
{
@ -3328,6 +3334,9 @@ Response:
```
### StorageBestAlloc
StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
Paths with more weight and more % of free space are preferred.
Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
Perms: admin
@ -3358,6 +3367,12 @@ Response:
],
"AllowTo": [
"string value"
],
"AllowTypes": [
"string value"
],
"DenyTypes": [
"string value"
]
}
]
@ -3403,6 +3418,14 @@ Inputs:
Response: `{}`
### StorageFindSector
StorageFindSector returns list of paths where the specified sector files exist.
If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
- Paths which have sector files locally (don't require fetching) will be listed first.
- Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
- Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
file types are specified, each type will be considered individually, and a union of all paths
which can accommodate each file type will be returned.
Perms: admin
@ -3434,7 +3457,13 @@ Response:
"Weight": 42,
"CanSeal": true,
"CanStore": true,
"Primary": true
"Primary": true,
"AllowTypes": [
"string value"
],
"DenyTypes": [
"string value"
]
}
]
```
@ -3502,6 +3531,12 @@ Response:
],
"AllowTo": [
"string value"
],
"AllowTypes": [
"string value"
],
"DenyTypes": [
"string value"
]
}
```

29
storage/paths/index.go
View File

@ -107,6 +107,22 @@ func (i *Index) StorageList(ctx context.Context) (map[storiface.ID][]storiface.D
}
func (i *Index) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error {
for i, typ := range si.AllowTypes {
_, err := storiface.TypeFromString(typ)
if err != nil {
// No need no hard-fail here, just warn the user
// (note that even with all-invalid entries we'll deny all types, so nothing unexpected should enter the path)
log.Errorw("bad path type in AllowTypes", "path", si.ID, "idx", i, "type", typ, "error", err)
}
}
for i, typ := range si.DenyTypes {
_, err := storiface.TypeFromString(typ)
if err != nil {
// No need no hard-fail here, just warn the user
log.Errorw("bad path type in DenyTypes", "path", si.ID, "idx", i, "type", typ, "error", err)
}
}
i.lk.Lock()
defer i.lk.Unlock()
@ -136,6 +152,8 @@ func (i *Index) StorageAttach(ctx context.Context, si storiface.StorageInfo, st
i.stores[si.ID].info.CanStore = si.CanStore
i.stores[si.ID].info.Groups = si.Groups
i.stores[si.ID].info.AllowTo = si.AllowTo
i.stores[si.ID].info.AllowTypes = si.AllowTypes
i.stores[si.ID].info.DenyTypes = si.DenyTypes
return nil
}
@ -312,6 +330,9 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
CanStore: st.info.CanStore,
Primary: isprimary[id],
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
})
}
@ -345,6 +366,11 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
continue
}
if !ft.AnyAllowed(st.info.AllowTypes, st.info.DenyTypes) {
log.Debugf("not selecting on %s, not allowed by file type filters", st.info.ID)
continue
}
if allowTo != nil {
allow := false
for _, group := range st.info.Groups {
@ -383,6 +409,9 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storif
CanStore: st.info.CanStore,
Primary: false,
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
})
}
}

4
storage/paths/local.go
View File

@ -506,6 +506,10 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
if !fileType.Allowed(si.AllowTypes, si.DenyTypes) {
continue
}
// TODO: Check free space
best = p.sectorPath(sid.ID, fileType)

11
storage/paths/remote.go
View File

@ -140,20 +140,21 @@ func (r *Remote) AcquireSector(ctx context.Context, s storiface.SectorRef, exist
}
}
apaths, ids, err := r.local.AcquireSector(ctx, s, storiface.FTNone, toFetch, pathType, op)
// get a list of paths to fetch data into. Note: file type filters will apply inside this call.
fetchPaths, ids, err := r.local.AcquireSector(ctx, s, storiface.FTNone, toFetch, pathType, op)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("allocate local sector for fetching: %w", err)
}
odt := storiface.FSOverheadSeal
overheadTable := storiface.FSOverheadSeal
if pathType == storiface.PathStorage {
odt = storiface.FsOverheadFinalized
overheadTable = storiface.FsOverheadFinalized
}
// If any path types weren't found in local storage, try fetching them
// First reserve storage
releaseStorage, err := r.local.Reserve(ctx, s, toFetch, ids, odt)
releaseStorage, err := r.local.Reserve(ctx, s, toFetch, ids, overheadTable)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("reserving storage space: %w", err)
}
@ -168,7 +169,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s storiface.SectorRef, exist
continue
}
dest := storiface.PathByType(apaths, fileType)
dest := storiface.PathByType(fetchPaths, fileType)
storageID := storiface.PathByType(ids, fileType)
url, err := r.acquireFromRemote(ctx, s.ID, fileType, dest)

11
storage/sealer/selector_alloc.go
View File

@ -55,13 +55,20 @@ func (s *allocSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi
return false, false, xerrors.Errorf("finding best alloc storage: %w", err)
}
requested := s.alloc
for _, info := range best {
if _, ok := have[info.ID]; ok {
return true, false, nil
requested = requested.SubAllowed(info.AllowTypes, info.AllowTypes)
// got all paths
if requested == storiface.FTNone {
break
}
}
}
return false, false, nil
return requested == storiface.FTNone, false, nil
}
func (s *allocSelector) Cmp(ctx context.Context, task sealtasks.TaskType, a, b *WorkerHandle) (bool, error) {

22
storage/sealer/selector_existing.go
View File

@ -15,7 +15,7 @@ import (
type existingSelector struct {
index paths.SectorIndex
sector abi.SectorID
alloc storiface.SectorFileType
fileType storiface.SectorFileType
allowFetch bool
}
@ -23,7 +23,7 @@ func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc sto
return &existingSelector{
index: index,
sector: sector,
alloc: alloc,
fileType: alloc,
allowFetch: allowFetch,
}
}
@ -52,18 +52,30 @@ func (s *existingSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt
return false, false, xerrors.Errorf("getting sector size: %w", err)
}
best, err := s.index.StorageFindSector(ctx, s.sector, s.alloc, ssize, s.allowFetch)
best, err := s.index.StorageFindSector(ctx, s.sector, s.fileType, ssize, s.allowFetch)
if err != nil {
return false, false, xerrors.Errorf("finding best storage: %w", err)
}
requested := s.fileType
for _, info := range best {
if _, ok := have[info.ID]; ok {
return true, false, nil
// we're not putting new sector files anywhere
if !s.allowFetch {
return true, false, nil
}
requested = requested.SubAllowed(info.AllowTypes, info.AllowTypes)
// got all paths
if requested == storiface.FTNone {
break
}
}
}
return false, false, nil
return requested == storiface.FTNone, false, nil
}
func (s *existingSelector) Cmp(ctx context.Context, task sealtasks.TaskType, a, b *WorkerHandle) (bool, error) {

14
storage/sealer/selector_move.go
View File

@ -72,7 +72,8 @@ func (s *moveSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.
return false, false, xerrors.Errorf("finding best dest storage: %w", err)
}
var ok bool
var ok, pref bool
requested := s.alloc
for _, info := range best {
if n, has := workerPaths[info.ID]; has {
@ -83,12 +84,19 @@ func (s *moveSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.
// either a no-op because the sector is already in the correct path,
// or the move a local move.
if n > 0 {
return true, true, nil
pref = true
}
requested = requested.SubAllowed(info.AllowTypes, info.AllowTypes)
// got all paths
if requested == storiface.FTNone {
break
}
}
}
return ok && s.allowRemote, false, nil
return ok && s.allowRemote, pref, nil
}
func (s *moveSelector) Cmp(ctx context.Context, task sealtasks.TaskType, a, b *WorkerHandle) (bool, error) {

66
storage/sealer/storiface/filetype.go
View File

@ -46,6 +46,23 @@ var FsOverheadFinalized = map[SectorFileType]int{
type SectorFileType int
func TypeFromString(s string) (SectorFileType, error) {
switch s {
case "unsealed":
return FTUnsealed, nil
case "sealed":
return FTSealed, nil
case "cache":
return FTCache, nil
case "update":
return FTUpdate, nil
case "update-cache":
return FTUpdateCache, nil
default:
return 0, xerrors.Errorf("unknown sector file type '%s'", s)
}
}
func (t SectorFileType) String() string {
switch t {
case FTUnsealed:
@ -63,6 +80,18 @@ func (t SectorFileType) String() string {
}
}
func (t SectorFileType) Strings() []string {
var out []string
for _, fileType := range PathTypes {
if fileType&t == 0 {
continue
}
out = append(out, fileType.String())
}
return out
}
func (t SectorFileType) Has(singleType SectorFileType) bool {
return t&singleType == singleType
}
@ -85,6 +114,43 @@ func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
return need, nil
}
func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType {
var denyMask SectorFileType // 1s deny
if len(allowTypes) > 0 {
denyMask = ^denyMask
for _, allowType := range allowTypes {
pt, err := TypeFromString(allowType)
if err != nil {
// we've told the user about this already, don't spam logs and ignore
continue
}
denyMask = denyMask & (^pt) // unset allowed types
}
}
for _, denyType := range denyTypes {
pt, err := TypeFromString(denyType)
if err != nil {
// we've told the user about this already, don't spam logs and ignore
continue
}
denyMask |= pt
}
return t & denyMask
}
func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) != t
}
func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) == 0
}
func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
var need uint64
for _, pathType := range PathTypes {

38
storage/sealer/storiface/filetype_test.go Normal file
View File

@ -0,0 +1,38 @@
package storiface
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestFileTypeAllow(t *testing.T) {
// no filters = allow all
require.True(t, FTCache.Allowed(nil, nil))
// allow allows matching type
require.True(t, FTCache.Allowed((FTCache).Strings(), nil))
// deny denies matching type
require.False(t, FTCache.Allowed(nil, (FTCache).Strings()))
// deny has precedence over allow
require.False(t, FTCache.Allowed((FTCache).Strings(), (FTCache).Strings()))
// deny allows non-matching types
require.True(t, FTUnsealed.Allowed(nil, (FTCache).Strings()))
// allow denies non-matching types
require.False(t, FTUnsealed.Allowed((FTCache).Strings(), nil))
}
func TestFileTypeAnyAllow(t *testing.T) {
// no filters = allow all
require.True(t, FTCache.AnyAllowed(nil, nil))
// one denied
require.False(t, FTCache.AnyAllowed(nil, (FTCache).Strings()))
// one denied, one allowed = allowed
require.True(t, (FTCache|FTUpdateCache).AnyAllowed(nil, (FTCache).Strings()))
}

52
storage/sealer/storiface/index.go
View File

@ -36,16 +36,55 @@ func ParseIDList(s string) IDList {
type Group = string
type StorageInfo struct {
ID ID
URLs []string // TODO: Support non-http transports
Weight uint64
// ID is the UUID of the storage path
ID ID
// URLs for remote access
URLs []string // TODO: Support non-http transports
// Storage path weight; higher number means that the path will be preferred more often
Weight uint64
// MaxStorage is the number of bytes allowed to be used by files in the
// storage path
MaxStorage uint64
CanSeal bool
// CanStore is true when the path is allowed to be used for io-intensive
// sealing operations
CanSeal bool
// CanStore is true when the path is allowed to be used for long-term storage
CanStore bool
Groups []Group
// Groups is the list of path groups this path belongs to
Groups []Group
// AllowTo is the list of paths to which data from this path can be moved to
AllowTo []Group
// AllowTypes lists sector file types which are allowed to be put into this
// path. If empty, all file types are allowed.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
AllowTypes []string
// DenyTypes lists sector file types which aren't allowed to be put into this
// path.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
}
type HealthReport struct {
@ -63,6 +102,9 @@ type SectorStorageInfo struct {
CanStore bool
Primary bool
AllowTypes []string
DenyTypes []string
}
type Decl struct {