cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

add miner fliter

Browse Source
This commit is contained in:
LexLuthr 2024-04-01 22:19:00 +04:00 committed by Łukasz Magiera
parent a99d8c8791
commit 2951d038a8
23 changed files with 834 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/api_storage.go
View File

@ -200,11 +200,11 @@ type StorageMiner interface {
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
// Paths with more weight and more % of free space are preferred.
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error) //perm:admin
StorageLocal(ctx context.Context) (map[storiface.ID]string, error) //perm:admin
StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin

8
api/proxy_gen.go
View File

@ -1185,7 +1185,7 @@ type StorageMinerMethods struct {
StorageAuthVerify func(p0 context.Context, p1 string) ([]auth.Permission, error) `perm:"read"`
StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) `perm:"admin"`
StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) `perm:"admin"`
StorageDeclareSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error `perm:"admin"`
@ -6988,14 +6988,14 @@ func (s *StorageMinerStub) StorageAuthVerify(p0 context.Context, p1 string) ([]a
return *new([]auth.Permission), ErrNotSupported
}
func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {
if s.Internal.StorageBestAlloc == nil {
return *new([]storiface.StorageInfo), ErrNotSupported
}
return s.Internal.StorageBestAlloc(p0, p1, p2, p3)
return s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)
}
func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {
return *new([]storiface.StorageInfo), ErrNotSupported
}

91
build/openrpc/miner.json
View File

@ -8897,11 +8897,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
@ -8920,6 +8932,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
@ -9078,7 +9096,7 @@
},
{
"name": "Filecoin.StorageBestAlloc",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3)\n}\n```",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)\n}\n```",
"summary": "StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.\nPaths with more weight and more % of free space are preferred.\nNote: This method doesn't filter paths based on AllowTypes/DenyTypes.\n",
"paramStructure": "by-position",
"params": [
@ -9130,6 +9148,23 @@
},
"required": true,
"deprecated": false
},
{
"name": "p4",
"description": "abi.ActorID",
"summary": "",
"schema": {
"title": "number",
"description": "Number is a number",
"examples": [
1000
],
"type": [
"number"
]
},
"required": true,
"deprecated": false
}
],
"result": {
@ -9159,6 +9194,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -9167,6 +9208,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
@ -9185,6 +9232,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
@ -9620,6 +9673,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -9628,6 +9687,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTypes": {
"items": {
"type": "string"
@ -9646,6 +9711,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
@ -9833,11 +9904,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
@ -9856,6 +9939,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"

18
cmd/curio/tasks/tasks.go
View File

@ -8,6 +8,8 @@ import (
"github.com/samber/lo"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/cmd/curio/deps"
curio "github.com/filecoin-project/lotus/curiosrc"
"github.com/filecoin-project/lotus/curiosrc/chainsched"
@ -37,6 +39,20 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
si := dependencies.Si
var activeTasks []harmonytask.TaskInterface
// Get all miner address from config
var miners []address.Address
for _, ad := range cfg.Addresses {
ad := ad
for _, m := range ad.MinerAddresses {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("failed to parse the miner address: %w", err)
}
miners = append(miners, maddr)
}
}
sender, sendTask := message.NewSender(full, full, db)
activeTasks = append(activeTasks, sendTask)
@ -76,7 +92,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
{
// Piece handling
if cfg.Subsystems.EnableParkPiece {
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks)
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks, miners)
cleanupPieceTask := piece.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0)
activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask)
}

2
curiosrc/ffi/task_storage.go
View File

@ -68,7 +68,7 @@ func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (Se
func (t *TaskStorage) HasCapacity() bool {
ctx := context.Background()
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType)
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType, abi.ActorID(0))
if err != nil {
log.Errorf("finding best alloc in HasCapacity: %+v", err)
return false

5
curiosrc/piece/task_park_piece.go
View File

@ -9,6 +9,8 @@ import (
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/curiosrc/ffi"
"github.com/filecoin-project/lotus/curiosrc/seal"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
@ -32,13 +34,14 @@ type ParkPieceTask struct {
max int
}
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int) *ParkPieceTask {
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int, miners []address.Address) *ParkPieceTask {
pt := &ParkPieceTask{
db: db,
sc: sc,
max: max,
}
go pt.pollPieceTasks(context.Background())
return pt
}

8
curiosrc/web/hapi/simpleinfo.go
View File

@ -393,6 +393,8 @@ type machineInfo struct {
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string
@ -458,7 +460,7 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
}
// query storage info
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, allow_miners, deny_miners, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
if err != nil {
return nil, err
}
@ -481,13 +483,15 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string
UsedPercent float64
ReservedPercent float64
}
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.AllowMiners, &s.DenyMiners, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
return nil, err
}

12
documentation/en/api-v0-methods-curio.md
View File

@ -179,6 +179,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -218,6 +224,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```

27
documentation/en/api-v0-methods-miner.md
View File

@ -3735,6 +3735,12 @@ Inputs:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
},
{
@ -3782,7 +3788,8 @@ Inputs:
[
1,
34359738368,
"sealing"
"sealing",
1000
]
```
@ -3809,6 +3816,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -3928,6 +3941,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -4004,6 +4023,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```

1
lib/harmony/harmonydb/sql/20240401-storage-segregation.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS allow_miners varchar, ADD COLUMN IF NOT EXISTS deny_miners varchar; -- comma separated list of miner addresses

205
storage/paths/db_index.go
View File

@ -15,6 +15,7 @@ import (
"go.opencensus.io/tag"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/journal/alerting"
@ -200,7 +201,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
currUrls = union(currUrls, si.URLs)
_, err = tx.Exec(
"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, last_heartbeat=NOW() WHERE storage_id=$10",
"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, allow_miners=$10, deny_miners=$11, last_heartbeat=NOW() WHERE storage_id=$12",
strings.Join(currUrls, ","),
si.Weight,
si.MaxStorage,
@ -210,6 +211,8 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
strings.Join(si.AllowTo, ","),
strings.Join(si.AllowTypes, ","),
strings.Join(si.DenyTypes, ","),
strings.Join(si.AllowMiners, ","),
strings.Join(si.DenyMiners, ","),
si.ID)
if err != nil {
return false, xerrors.Errorf("storage attach UPDATE fails: %v", err)
@ -221,7 +224,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
// Insert storage id
_, err = tx.Exec(
"INSERT INTO storage_path "+
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW())",
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), $16, $17)",
si.ID,
strings.Join(si.URLs, ","),
si.Weight,
@ -236,7 +239,9 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
st.Available,
st.FSAvailable,
st.Reserved,
st.Used)
st.Used,
strings.Join(si.AllowMiners, ","),
strings.Join(si.DenyMiners, ","))
if err != nil {
return false, xerrors.Errorf("StorageAttach insert fails: %v", err)
}
@ -532,14 +537,16 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
// 7. Storage path is part of the groups which are allowed from the storage paths which already hold the sector
var rows []struct {
StorageId string
Urls string
Weight uint64
CanSeal bool
CanStore bool
Groups string
AllowTypes string
DenyTypes string
StorageId string
Urls string
Weight uint64
CanSeal bool
CanStore bool
Groups string
AllowTypes string
DenyTypes string
AllowMiners string
DenyMiners string
}
err = dbi.harmonyDB.Select(ctx, &rows,
`SELECT storage_id,
@ -549,7 +556,9 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
can_store,
groups,
allow_types,
deny_types
deny_types,
allow_miners,
deny_miners
FROM storage_path
WHERE can_seal=true
and available >= $1
@ -570,6 +579,53 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
continue
}
if len(row.AllowMiners) > 0 {
found := false
allowMiners := splitString(row.AllowMiners)
for _, m := range allowMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if !found {
log.Debugf("not selecting on %s, not allowed by allow miners filters", row.StorageId)
continue
}
}
if len(row.DenyMiners) > 0 {
found := false
denyMiners := splitString(row.DenyMiners)
for _, m := range denyMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if found {
log.Debugf("not selecting on %s, not allowed by deny miners filters", row.StorageId)
continue
}
}
if allowList != nil {
groups := splitString(row.Groups)
allow := false
@ -618,19 +674,21 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {
var qResults []struct {
Urls string
Weight uint64
MaxStorage uint64
CanSeal bool
CanStore bool
Groups string
AllowTo string
AllowTypes string
DenyTypes string
Urls string
Weight uint64
MaxStorage uint64
CanSeal bool
CanStore bool
Groups string
AllowTo string
AllowTypes string
DenyTypes string
AllowMiners string
DenyMiners string
}
err := dbi.harmonyDB.Select(ctx, &qResults,
"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types "+
"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, allow_miners, deny_miners "+
"FROM storage_path WHERE storage_id=$1", string(id))
if err != nil {
return storiface.StorageInfo{}, xerrors.Errorf("StorageInfo query fails: %v", err)
@ -647,11 +705,13 @@ func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface
sinfo.AllowTo = splitString(qResults[0].AllowTo)
sinfo.AllowTypes = splitString(qResults[0].AllowTypes)
sinfo.DenyTypes = splitString(qResults[0].DenyTypes)
sinfo.AllowMiners = splitString(qResults[0].AllowMiners)
sinfo.DenyMiners = splitString(qResults[0].DenyMiners)
return sinfo, nil
}
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) {
var err error
var spaceReq uint64
switch pathType {
@ -667,16 +727,18 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
}
var rows []struct {
StorageId string
Urls string
Weight uint64
MaxStorage uint64
CanSeal bool
CanStore bool
Groups string
AllowTo string
AllowTypes string
DenyTypes string
StorageId string
Urls string
Weight uint64
MaxStorage uint64
CanSeal bool
CanStore bool
Groups string
AllowTo string
AllowTypes string
DenyTypes string
AllowMiners string
DenyMiners string
}
err = dbi.harmonyDB.Select(ctx, &rows,
@ -689,7 +751,9 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
groups,
allow_to,
allow_types,
deny_types
deny_types,
allow_miners,
deny_miners
FROM storage_path
WHERE available >= $1
and NOW()-($2 * INTERVAL '1 second') < last_heartbeat
@ -700,24 +764,77 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
SkippedHeartbeatThresh.Seconds(),
pathType == storiface.PathSealing,
pathType == storiface.PathStorage,
miner.String(),
)
if err != nil {
return nil, xerrors.Errorf("Querying for best storage sectors fails with err %w: ", err)
}
var result []storiface.StorageInfo
for _, row := range rows {
// Matching with 0 as a workaround to avoid having minerID
// present when calling TaskStorage.HasCapacity()
if !(miner == abi.ActorID(0)) {
if len(row.AllowMiners) > 0 {
found := false
allowMiners := splitString(row.AllowMiners)
for _, m := range allowMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if !found {
continue
}
}
if len(row.DenyMiners) > 0 {
found := false
denyMiners := splitString(row.DenyMiners)
for _, m := range denyMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if found {
continue
}
}
}
result = append(result, storiface.StorageInfo{
ID: storiface.ID(row.StorageId),
URLs: splitString(row.Urls),
Weight: row.Weight,
MaxStorage: row.MaxStorage,
CanSeal: row.CanSeal,
CanStore: row.CanStore,
Groups: splitString(row.Groups),
AllowTo: splitString(row.AllowTo),
AllowTypes: splitString(row.AllowTypes),
DenyTypes: splitString(row.DenyTypes),
ID: storiface.ID(row.StorageId),
URLs: splitString(row.Urls),
Weight: row.Weight,
MaxStorage: row.MaxStorage,
CanSeal: row.CanSeal,
CanStore: row.CanStore,
Groups: splitString(row.Groups),
AllowTo: splitString(row.AllowTo),
AllowTypes: splitString(row.AllowTypes),
DenyTypes: splitString(row.DenyTypes),
AllowMiners: splitString(row.AllowMiners),
DenyMiners: splitString(row.DenyMiners),
})
}

153
storage/paths/index.go
View File

@ -14,6 +14,7 @@ import (
"go.opencensus.io/tag"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
@ -38,7 +39,7 @@ type SectorIndex interface { // part of storage-miner api
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)
// atomically acquire locks on all sector file types. close ctx to unlock
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error
@ -61,6 +62,7 @@ type storageEntry struct {
heartbeatErr error
}
// MemIndex represents an in-memory index of storage sectors and storage entries.
type MemIndex struct {
*indexLocks
lk sync.RWMutex
@ -206,6 +208,8 @@ func (i *MemIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
i.stores[si.ID].info.AllowTo = si.AllowTo
i.stores[si.ID].info.AllowTypes = allow
i.stores[si.ID].info.DenyTypes = deny
i.stores[si.ID].info.AllowMiners = si.AllowMiners
i.stores[si.ID].info.DenyMiners = si.DenyMiners
return nil
}
@ -476,8 +480,10 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
Primary: isprimary[id],
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
AllowMiners: st.info.AllowMiners,
DenyMiners: st.info.DenyMiners,
})
}
@ -492,6 +498,52 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
continue
}
if len(st.info.AllowMiners) > 0 {
found := false
for _, m := range st.info.AllowMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if !found {
log.Debugf("not allocating on %s, miner %s not allowed", st.info.ID, s.Miner.String())
continue
}
}
if len(st.info.DenyMiners) > 0 {
found := false
for _, m := range st.info.DenyMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if found {
log.Debugf("not allocating on %s, miner %s denied", st.info.ID, s.Miner.String())
continue
}
}
if spaceReq > uint64(st.fsi.Available) {
log.Debugf("not selecting on %s, out of space (available: %d, need: %d)", st.info.ID, st.fsi.Available, spaceReq)
continue
@ -555,8 +607,10 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
Primary: false,
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
AllowMiners: st.info.AllowMiners,
DenyMiners: st.info.DenyMiners,
})
}
}
@ -564,6 +618,21 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
return out, nil
}
// StorageInfo retrieves the storage information for a given storage ID.
//
// The method first acquires a read lock on the MemIndex to ensure thread-safety.
// It then checks if the storage ID exists in the stores map. If not, it returns
// an error indicating that the sector store was not found.
//
// Finally, it returns the storage information of the selected storage.
//
// Parameters:
// - ctx: the context.Context object for cancellation and timeouts
// - id: the ID of the storage to retrieve information for
//
// Returns:
// - storiface.StorageInfo: the storage information of the selected storage ID
// - error: an error indicating any issues encountered during the process
func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {
i.lk.RLock()
defer i.lk.RUnlock()
@ -576,7 +645,33 @@ func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.
return *si.info, nil
}
func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
// StorageBestAlloc selects the best available storage options for allocating
// a sector file. It takes into account the allocation type (sealing or storage),
// sector size, and path type (sealing or storage).
//
// The method first estimates the required space for the allocation based on the
// sector size and path type. It then iterates through all available storage options
// and filters out those that cannot be used for the given path type. It also filters
// out storage options that do not have enough available space or have not received
// heartbeats within a certain threshold.
//
// The remaining storage options are sorted based on their available space and weight,
// with higher availability and weight being prioritized. The method then returns
// the information of the selected storage options.
//
// If no suitable storage options are found, it returns an error indicating that
// no good path is available.
//
// Parameters:
// - ctx: the context.Context object for cancellation and timeouts
// - allocate: the type of allocation (sealing or storage)
// - ssize: the size of the sector file
// - pathType: the path type (sealing or storage)
//
// Returns:
// - []storiface.StorageInfo: the information of the selected storage options
// - error: an error indicating any issues encountered during the process
func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) {
i.lk.RLock()
defer i.lk.RUnlock()
@ -604,6 +699,52 @@ func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sect
continue
}
if len(p.info.AllowMiners) > 0 {
found := false
for _, m := range p.info.AllowMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if !found {
log.Debugf("not allocating on %s, miner %s not allowed", p.info.ID, miner.String())
continue
}
}
if len(p.info.DenyMiners) > 0 {
found := false
for _, m := range p.info.DenyMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if found {
log.Debugf("not allocating on %s, miner %s denied", p.info.ID, miner.String())
continue
}
}
if spaceReq > uint64(p.fsi.Available) {
log.Debugf("not allocating on %s, out of space (available: %d, need: %d)", p.info.ID, p.fsi.Available, spaceReq)
continue

58
storage/paths/index_test.go
View File

@ -152,3 +152,61 @@ func TestFindAllow(t *testing.T) {
}
}
}
func TestStorageBestAlloc(t *testing.T) {
idx := NewMemIndex(nil)
dummyStorageInfo := storiface.StorageInfo{
ID: storiface.ID("dummy"),
CanSeal: true,
CanStore: true,
URLs: []string{"http://localhost:9999/"},
Weight: 10,
AllowMiners: []string{
"t001",
},
}
dummyFsStat := fsutil.FsStat{
Capacity: 10000000000,
Available: 7000000000,
Reserved: 100000000,
Used: 3000000000,
}
err := idx.StorageAttach(context.Background(), dummyStorageInfo, dummyFsStat)
require.NoError(t, err)
t.Run("PathSealing", func(t *testing.T) {
result, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 1)
require.Equal(t, err, nil)
require.NotNil(t, result)
require.Equal(t, len(result), 1)
require.Equal(t, result[0].ID, dummyStorageInfo.ID)
})
t.Run("PathStorage", func(t *testing.T) {
result, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathStorage, 1)
require.Equal(t, err, nil)
require.NotNil(t, result)
require.Equal(t, len(result), 1)
require.Equal(t, result[0].ID, dummyStorageInfo.ID)
})
t.Run("NotAllowedMiner", func(t *testing.T) {
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 2)
require.Error(t, err)
})
t.Run("NoAvailableSpace", func(t *testing.T) {
bigSectorSize := abi.SectorSize(10000000000)
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, bigSectorSize, storiface.PathSealing, 1)
require.Error(t, err)
})
t.Run("AllowedMiner", func(t *testing.T) {
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 1)
require.NoError(t, err)
})
}

121
storage/paths/local.go
View File

@ -14,6 +14,7 @@ import (
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/proof"
@ -204,16 +205,18 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
}
err = st.index.StorageAttach(ctx, storiface.StorageInfo{
ID: meta.ID,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
Groups: meta.Groups,
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
ID: meta.ID,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
Groups: meta.Groups,
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
AllowMiners: meta.AllowMiners,
DenyMiners: meta.DenyMiners,
}, fst)
if err != nil {
return xerrors.Errorf("declaring storage in index: %w", err)
@ -294,16 +297,18 @@ func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMiss
}
err = st.index.StorageAttach(ctx, storiface.StorageInfo{
ID: id,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
Groups: meta.Groups,
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
ID: id,
URLs: st.urls,
Weight: meta.Weight,
MaxStorage: meta.MaxStorage,
CanSeal: meta.CanSeal,
CanStore: meta.CanStore,
Groups: meta.Groups,
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
AllowMiners: meta.AllowMiners,
DenyMiners: meta.DenyMiners,
}, fst)
if err != nil {
return xerrors.Errorf("redeclaring storage in index: %w", err)
@ -495,20 +500,64 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
var out storiface.SectorPaths
var storageIDs storiface.SectorPaths
allocPathOk := func(canSeal, canStore bool, allowTypes, denyTypes []string, fileType storiface.SectorFileType) bool {
allocPathOk := func(canSeal, canStore bool, allowTypes, denyTypes, allowMiners, denyMiners []string, fileType storiface.SectorFileType, miner abi.ActorID) (bool, error) {
if (pathType == storiface.PathSealing) && !canSeal {
return false
return false, nil
}
if (pathType == storiface.PathStorage) && !canStore {
return false
return false, nil
}
if !fileType.Allowed(allowTypes, denyTypes) {
return false
return false, nil
}
return true
if len(allowMiners) > 0 {
found := false
for _, m := range allowMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return false, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return false, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if !found {
return false, nil
}
}
if len(denyMiners) > 0 {
found := false
for _, m := range denyMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return false, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return false, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if found {
return false, nil
}
}
return true, nil
}
// First find existing files
@ -536,8 +585,15 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
if allocate.Has(fileType) && !allocPathOk(info.CanSeal, info.CanStore, info.AllowTypes, info.DenyTypes, fileType) {
continue // allocate request for a path of different type
if allocate.Has(fileType) {
ok, err := allocPathOk(info.CanSeal, info.CanStore, info.AllowTypes, info.DenyTypes, info.AllowMiners, info.DenyMiners, fileType, sid.ID.Miner)
if err != nil {
log.Debug(err)
continue
}
if !ok {
continue // allocate request for a path of different type
}
}
spath := p.sectorPath(sid.ID, fileType)
@ -556,7 +612,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType)
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType, sid.ID.Miner)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("finding best storage for allocating : %w", err)
}
@ -574,7 +630,14 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
if !allocPathOk(si.CanSeal, si.CanStore, si.AllowTypes, si.DenyTypes, fileType) {
alloc, err := allocPathOk(si.CanSeal, si.CanStore, si.AllowTypes, si.DenyTypes, si.AllowMiners, si.DenyMiners, fileType, sid.ID.Miner)
if err != nil {
log.Debug(err)
continue
}
if !alloc {
continue
}

8
storage/paths/mocks/index.go
View File

@ -54,18 +54,18 @@ func (mr *MockSectorIndexMockRecorder) StorageAttach(arg0, arg1, arg2 interface{
}
// StorageBestAlloc mocks base method.
func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType) ([]storiface.StorageInfo, error) {
func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType, arg4 abi.ActorID) ([]storiface.StorageInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3)
ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].([]storiface.StorageInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StorageBestAlloc indicates an expected call of StorageBestAlloc.
func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3, arg4)
}
// StorageDeclareSector mocks base method.

32
storage/sealer/manager.go
View File

@ -356,7 +356,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto
// selector will schedule the Unseal task on a worker that either already has the sealed sector files or has space in
// one of it's sealing scratch spaces to store them after fetching them from another worker.
selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, true)
selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, sector.ID.Miner, true)
log.Debugf("will schedule unseal for sector %d", sector.ID)
err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error {
@ -377,7 +377,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto
}
// get a selector for moving unsealed sector into long-term storage
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTUnsealed, storiface.PathStorage, !m.disallowRemoteFinalize)
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTUnsealed, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
// move unsealed sector to long-term storage
// Possible TODO: Add an option to not keep the unsealed sector in long term storage?
@ -431,9 +431,9 @@ func (m *Manager) AddPiece(ctx context.Context, sector storiface.SectorRef, exis
var selector WorkerSelector
var err error
if len(existingPieces) == 0 { // new
selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing)
selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing, sector.ID.Miner)
} else { // use existing
selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false)
selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, sector.ID.Miner, false)
}
var out abi.PieceInfo
@ -484,7 +484,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector storiface.SectorRef
// TODO: also consider where the unsealed data sits
selector := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed, storiface.PathSealing)
selector := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, sector.ID.Miner)
err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit1, selector, m.schedFetch(sector, storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.SealPreCommit1(ctx, sector, ticket, pieces))
@ -533,7 +533,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector storiface.SectorRef
return storiface.SectorCids{}, xerrors.Errorf("acquiring sector lock: %w", err)
}
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, true)
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, sector.ID.Miner, true)
err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit2, selector, m.schedFetch(sector, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.SealPreCommit2(ctx, sector, phase1Out))
@ -585,7 +585,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sector storiface.SectorRef, t
// NOTE: We set allowFetch to false in so that we always execute on a worker
// with direct access to the data. We want to do that because this step is
// generally very cheap / fast, and transferring data is not worth the effort
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, sector.ID.Miner, false)
err = m.sched.Schedule(ctx, sector, sealtasks.TTCommit1, selector, m.schedFetch(sector, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.SealCommit1(ctx, sector, ticket, seed, pieces, cids))
@ -700,7 +700,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storiface.SectorRef
// do the cache trimming wherever the likely still very large cache lives.
// we really don't want to move it.
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache, sector.ID.Miner, false)
err = m.sched.Schedule(ctx, sector, sealtasks.TTFinalize, selector,
m.schedFetch(sector, storiface.FTCache, cachePathType, storiface.AcquireMove),
@ -713,7 +713,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storiface.SectorRef
}
// get a selector for moving stuff into long-term storage
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, storiface.PathStorage, !m.disallowRemoteFinalize)
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
// only move the unsealed file if it still exists and needs moving
moveUnsealed := storiface.FTUnsealed
@ -769,7 +769,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
// do the cache trimming wherever the likely still large cache lives.
// we really don't want to move it.
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdateCache, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdateCache, sector.ID.Miner, false)
err := m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeReplicaUpdate, selector,
m.schedFetch(sector, storiface.FTCache|storiface.FTUpdateCache, pathType, storiface.AcquireMove),
@ -783,7 +783,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
move := func(types storiface.SectorFileType) error {
// get a selector for moving stuff into long-term storage
fetchSel := newMoveSelector(m.index, sector.ID, types, storiface.PathStorage, !m.disallowRemoteFinalize)
fetchSel := newMoveSelector(m.index, sector.ID, types, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
err = m.sched.Schedule(ctx, sector, sealtasks.TTFetch, fetchSel,
m.schedFetch(sector, types, storiface.PathStorage, storiface.AcquireMove),
@ -834,7 +834,7 @@ func (m *Manager) ReleaseUnsealed(ctx context.Context, sector storiface.SectorRe
return nil
}
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, sector.ID.Miner, false)
return m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeUnsealed, selector, m.schedFetch(sector, storiface.FTUnsealed, pathType, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
_, err := m.waitSimpleCall(ctx)(w.ReleaseUnsealed(ctx, sector, keepUnsealed))
@ -906,7 +906,7 @@ func (m *Manager) GenerateSectorKeyFromData(ctx context.Context, sector storifac
// NOTE: We set allowFetch to false in so that we always execute on a worker
// with direct access to the data. We want to do that because this step is
// generally very cheap / fast, and transferring data is not worth the effort
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, true)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, sector.ID.Miner, true)
err = m.sched.Schedule(ctx, sector, sealtasks.TTRegenSectorKey, selector, m.schedFetch(sector, storiface.FTUpdate|storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.GenerateSectorKeyFromData(ctx, sector, commD))
@ -984,7 +984,7 @@ func (m *Manager) ReplicaUpdate(ctx context.Context, sector storiface.SectorRef,
return storiface.ReplicaUpdateOut{}, xerrors.Errorf("acquiring sector lock: %w", err)
}
selector := newAllocSelector(m.index, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing)
selector := newAllocSelector(m.index, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, sector.ID.Miner)
err = m.sched.Schedule(ctx, sector, sealtasks.TTReplicaUpdate, selector, m.schedFetch(sector, storiface.FTUnsealed|storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.ReplicaUpdate(ctx, sector, pieces))
@ -1035,7 +1035,7 @@ func (m *Manager) ProveReplicaUpdate1(ctx context.Context, sector storiface.Sect
// NOTE: We set allowFetch to false in so that we always execute on a worker
// with direct access to the data. We want to do that because this step is
// generally very cheap / fast, and transferring data is not worth the effort
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, sector.ID.Miner, false)
err = m.sched.Schedule(ctx, sector, sealtasks.TTProveReplicaUpdate1, selector, m.schedFetch(sector, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
@ -1153,7 +1153,7 @@ func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.Secto
ptype = storiface.PathStorage
}
selector := newAllocSelector(m.index, toFetch, ptype)
selector := newAllocSelector(m.index, toFetch, ptype, sector.ID.Miner)
err = m.sched.Schedule(ctx, sector, sealtasks.TTDownloadSector, selector, schedNop, func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.DownloadSectorData(ctx, sector, finalized, src))

2
storage/sealer/sched_test.go
View File

@ -274,7 +274,7 @@ func TestSched(t *testing.T) {
done := make(chan struct{})
rm.done[taskName] = done
sel := newAllocSelector(index, storiface.FTCache, storiface.PathSealing)
sel := newAllocSelector(index, storiface.FTCache, storiface.PathSealing, abi.ActorID(1000))
rm.wg.Add(1)
go func() {

6
storage/sealer/selector_alloc.go
View File

@ -16,13 +16,15 @@ type allocSelector struct {
index paths.SectorIndex
alloc storiface.SectorFileType
ptype storiface.PathType
miner abi.ActorID
}
func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType) *allocSelector {
func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType, miner abi.ActorID) *allocSelector {
return &allocSelector{
index: index,
alloc: alloc,
ptype: ptype,
miner: miner,
}
}
@ -50,7 +52,7 @@ func (s *allocSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi
return false, false, xerrors.Errorf("getting sector size: %w", err)
}
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype)
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype, s.miner)
if err != nil {
return false, false, xerrors.Errorf("finding best alloc storage: %w", err)
}

4
storage/sealer/selector_existing.go
View File

@ -16,14 +16,16 @@ type existingSelector struct {
index paths.SectorIndex
sector abi.SectorID
fileType storiface.SectorFileType
miner abi.ActorID
allowFetch bool
}
func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, allowFetch bool) *existingSelector {
func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, miner abi.ActorID, allowFetch bool) *existingSelector {
return &existingSelector{
index: index,
sector: sector,
fileType: alloc,
miner: miner,
allowFetch: allowFetch,
}
}

5
storage/sealer/selector_move.go
View File

@ -17,10 +17,11 @@ type moveSelector struct {
sector abi.SectorID
alloc storiface.SectorFileType
destPtype storiface.PathType
miner abi.ActorID
allowRemote bool
}
func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, allowRemote bool) *moveSelector {
func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, miner abi.ActorID, allowRemote bool) *moveSelector {
return &moveSelector{
index: index,
sector: sector,
@ -67,7 +68,7 @@ func (s *moveSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.
}
}
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.destPtype)
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.destPtype, s.miner)
if err != nil {
return false, false, xerrors.Errorf("finding best dest storage: %w", err)
}

159
storage/sealer/storiface/filetype.go
View File

@ -8,6 +8,28 @@ import (
"github.com/filecoin-project/go-state-types/abi"
)
// FTUnsealed represents an unsealed sector file type.
// FTSealed represents a sealed sector file type.
// FTCache represents a cache sector file type.
// FTUpdate represents an update sector file type.
// FTUpdateCache represents an update cache sector file type.
// FTPiece represents a Piece Park sector file type.
// FileTypes represents the total number of file types.
//
// The SectorFileType type is an integer type that represents different sector file types.
// It has several methods to manipulate and query the file type.
// The String method returns a string representation of the file type.
// The Strings method returns a slice of string representations of all the file types that are set in the receiver object.
// The AllSet method returns a slice of all the file types that are set in the receiver object.
// The Has method checks whether a specific file type is set in the receiver object.
// The SealSpaceUse method calculates the space used by the receiver object in sealing a sector of a given size.
// The SubAllowed method removes selected file types from the receiver object based on a list of allowed and denied file types.
// The Unset method removes selected file types from the receiver object.
// The AnyAllowed method checks whether any file types in the receiver object are allowed based on a list of allowed and denied file types.
// The Allowed method checks whether all file types in the receiver object are allowed based on a list of allowed and denied file types.
// The StoreSpaceUse method calculates the space used by the receiver object in storing a sector of a given size.
// The All method returns an array that represents which file types are set in the receiver object.
// The IsNone method checks whether the receiver object represents no file types.
const (
// "regular" sectors
FTUnsealed SectorFileType = 1 << iota
@ -24,12 +46,28 @@ const (
FileTypes = iota
)
// PathTypes is a slice of SectorFileType that represents different types of sector file paths.
// It contains the following types of sector file paths:
// - FTUnsealed: represents regular unsealed sectors
// - FTSealed: represents sealed sectors
// - FTCache: represents cache sectors
// - FTUpdate: represents snap sectors
// - FTUpdateCache: represents snap cache sectors
// - FTPiece: represents Piece Park sectors
var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache, FTUpdate, FTUpdateCache, FTPiece}
// FTNone represents a sector file type of none. This constant is used in the StorageLock method to specify that a sector should not have any file types locked.
// Example usage:
// err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache)
const (
FTNone SectorFileType = 0
)
// FTAll represents the combination of all available sector file types.
// It is a variable of type SectorFileType.
// The value of FTAll is calculated by iterating over the PathTypes slice and using the |= operator to perform a bitwise OR operation on each path type.
// The result is assigned to the variable out and returned.
// FTAll is immediately invoked as a function using the anonymous function syntax, so the result is returned as soon as it is calculated.
var FTAll = func() (out SectorFileType) {
for _, pathType := range PathTypes {
out |= pathType
@ -37,8 +75,10 @@ var FTAll = func() (out SectorFileType) {
return out
}()
// FSOverheadDen represents the constant value 10, which is used to calculate the overhead in various storage space utilization calculations.
const FSOverheadDen = 10
// FSOverheadSeal is a map that represents the overheads for different SectorFileType in sealed sectors.
var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
FTUnsealed: FSOverheadDen,
FTSealed: FSOverheadDen,
@ -50,6 +90,30 @@ var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
// sector size * disk / fs overhead. FSOverheadDen is like the unit of sector size
// FsOverheadFinalized is a map that represents the finalized overhead for different types of SectorFileType.
// The keys in the map are the SectorFileType values, and the values are integers representing the overhead.
// It is used to calculate the storage space usage for different types of sectors, as shown in the example below:
//
// func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
// var need uint64
// for _, pathType := range PathTypes {
// if !t.Has(pathType) {
// continue
// }
//
// oh, ok := FsOverheadFinalized[pathType]
// if !ok {
// return 0, xerrors.Errorf("no finalized overhead info for %s", pathType)
// }
//
// need += uint64(oh) * uint64(ssize) / FSOverheadDen
// }
//
// return need, nil
// }
//
// The overhead value is retrieved from FsOverheadFinalized by using the SectorFileType value as the key.
// If the overhead value is not found in the map, an error is returned indicating that there is no finalized overhead information for the given sector type.
var FsOverheadFinalized = map[SectorFileType]int{
FTUnsealed: FSOverheadDen,
FTSealed: FSOverheadDen,
@ -59,8 +123,13 @@ var FsOverheadFinalized = map[SectorFileType]int{
FTPiece: FSOverheadDen,
}
// SectorFileType represents the type of a sector file
// TypeFromString converts a string to a SectorFileType
type SectorFileType int
// TypeFromString converts a string representation of a SectorFileType to its corresponding value.
// It returns the SectorFileType and nil error if the string matches one of the existing types.
// If the string does not match any type, it returns 0 and an error.
func TypeFromString(s string) (SectorFileType, error) {
switch s {
case "unsealed":
@ -80,6 +149,7 @@ func TypeFromString(s string) (SectorFileType, error) {
}
}
// String returns a string representation of the SectorFileType.
func (t SectorFileType) String() string {
switch t {
case FTUnsealed:
@ -99,6 +169,12 @@ func (t SectorFileType) String() string {
}
}
// Strings returns a slice of strings representing the names of the SectorFileType values that are set in the receiver value.
// Example usage:
//
// fileType := SectorFileType(FTSealed | FTCache)
// names := fileType.Strings() // names = ["sealed", "cache"]
// fmt.Println(names)
func (t SectorFileType) Strings() []string {
var out []string
for _, fileType := range PathTypes {
@ -111,6 +187,7 @@ func (t SectorFileType) Strings() []string {
return out
}
// AllSet returns a slice of SectorFileType values that are set in the SectorFileType receiver value
func (t SectorFileType) AllSet() []SectorFileType {
var out []SectorFileType
for _, fileType := range PathTypes {
@ -123,10 +200,33 @@ func (t SectorFileType) AllSet() []SectorFileType {
return out
}
// Has checks if the SectorFileType has a specific singleType.
func (t SectorFileType) Has(singleType SectorFileType) bool {
return t&singleType == singleType
}
// SealSpaceUse calculates the amount of space needed for sealing the sector
// based on the given sector size. It iterates over the different path types
// and calculates the space needed for each path type using the FSOverheadSeal
// map. The overhead value is multiplied by the sector size and divided by the
// FSOverheadDen constant. The total space needed is accumulated and returned.
// If there is no seal overhead information for a particular path type, an error
// is returned.
//
// Example usage:
//
// fileType := FTSealed | FTCache
// sectorSize := abi.SectorSize(32 << 20) // 32 MiB
// spaceNeeded, err := fileType.SealSpaceUse(sectorSize)
//
// Parameters:
//
// ssize: The size of the sector
//
// Returns:
//
// uint64: The amount of space needed for sealing the sector
// error: If there is no seal overhead information for a path type
func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
var need uint64
for _, pathType := range PathTypes {
@ -145,6 +245,15 @@ func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
return need, nil
}
// The method takes in two parameters: allowTypes and denyTypes, both of which are slices of strings.
// If allowTypes is not empty, the method sets a denyMask with all bits set to 1, and then iterates over each allowType,
// converting it to a SectorFileType using the TypeFromString function and unsetting the corresponding bit in the denyMask.
// If a string in allowTypes cannot be converted to a valid SectorFileType, it is ignored.
// After processing allowTypes, the method iterates over each denyType, converting it to a SectorFileType using the TypeFromString function
// and setting the corresponding bit in the denyMask.
// If a string in denyTypes cannot be converted to a valid SectorFileType, it is ignored.
// Finally, the method returns the bitwise AND of the original SectorFileType and the denyMask.
// The returned SectorFileType will only allow the types specified in allowTypes and exclude the types specified in denyTypes.`
func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType {
var denyMask SectorFileType // 1s deny
@ -174,18 +283,30 @@ func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) Sect
return t & denyMask
}
// Unset removes the specified sector file type(s) from the current SectorFileType value.
// It performs a bitwise AND operation between the current value and the bitwise complement of the toUnset value.
// The result is returned as a new SectorFileType value.
// Any bits that are set in toUnset will be cleared in the result.
// Usage: result = value.Unset(typesToUnset)
func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType {
return t &^ toUnset
}
// AnyAllowed checks if the SectorFileType has any allowed types and no denied types.
func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) != t
}
// Allowed checks if the SectorFileType is allowed based on the given allowTypes and denyTypes.
// Returns true if the SectorFileType is allowed, otherwise false.
func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) == 0
}
// StoreSpaceUse calculates the space used for storing sectors of a specific file type.
// It takes the sector size as input and returns the total space needed in bytes and an error, if any.
// The calculation is based on the finalized overhead information for the file type.
// If the overhead information is not available for a particular file type, an error will be returned.
func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
var need uint64
for _, pathType := range PathTypes {
@ -204,6 +325,7 @@ func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
return need, nil
}
// All returns an array indicating whether each FileTypes flag is set in the SectorFileType.
func (t SectorFileType) All() [FileTypes]bool {
var out [FileTypes]bool
@ -214,10 +336,13 @@ func (t SectorFileType) All() [FileTypes]bool {
return out
}
// IsNone checks if the SectorFileType value is equal to zero.
// It returns true if the value is zero, indicating that the type is none.
func (t SectorFileType) IsNone() bool {
return t == 0
}
// SectorPaths represents the paths for different sector files.
type SectorPaths struct {
ID abi.SectorID
@ -229,6 +354,7 @@ type SectorPaths struct {
Piece string
}
// HasAllSet checks if all paths of a SectorPaths struct are set for a given SectorFileType.
func (sp SectorPaths) HasAllSet(ft SectorFileType) bool {
for _, fileType := range ft.AllSet() {
if PathByType(sp, fileType) == "" {
@ -239,6 +365,9 @@ func (sp SectorPaths) HasAllSet(ft SectorFileType) bool {
return true
}
// Subset returns a new instance of SectorPaths that contains only the paths specified by the filter SectorFileType.
// It iterates over each fileType in the filter, retrieves the corresponding path from the original SectorPaths instance, and sets it in the new instance.
// Finally, it sets the ID field of the new instance to be the same as the original instance.
func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths {
var out SectorPaths
@ -251,6 +380,24 @@ func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths {
return out
}
// ParseSectorID parses a sector ID from a given base name.
// It expects the format "s-t0%d-%d", where the first %d represents the miner ID
// and the second %d represents the sector number.
//
// Parameters:
// - baseName: The base name from which to parse the sector ID.
//
// Returns:
// - abi.SectorID: The parsed sector ID.
// - error: An error if parsing fails.
//
// Example usage:
//
// id, err := ParseSectorID(baseName)
// if err != nil {
// // handle error
// }
// // use id
func ParseSectorID(baseName string) (abi.SectorID, error) {
var n abi.SectorNumber
var mid abi.ActorID
@ -269,10 +416,19 @@ func ParseSectorID(baseName string) (abi.SectorID, error) {
}, nil
}
// SectorName returns the name of a sector in the format "s-t0<Miner>-<Number>"
//
// Parameters:
// - sid: The sector ID
//
// Returns:
// - The name of the sector as a string
func SectorName(sid abi.SectorID) string {
return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number)
}
// PathByType returns the path associated with the specified fileType in the given SectorPaths.
// It panics if the requested path type is unknown.
func PathByType(sps SectorPaths, fileType SectorFileType) string {
switch fileType {
case FTUnsealed:
@ -309,11 +465,14 @@ func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string) {
}
}
// PathsWithIDs represents paths and IDs for sector files.
type PathsWithIDs struct {
Paths SectorPaths
IDs SectorPaths
}
// HasAllSet checks if all paths and IDs in PathsWithIDs have a corresponding path set for the specified SectorFileType.
// It returns true if all paths and IDs are set, and false otherwise.
func (p PathsWithIDs) HasAllSet(ft SectorFileType) bool {
return p.Paths.HasAllSet(ft) && p.IDs.HasAllSet(ft)
}

14
storage/sealer/storiface/index.go
View File

@ -86,6 +86,14 @@ type StorageInfo struct {
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
// AllowMiners lists miner IDs which are allowed to store their sector data into
// this path. If empty, all miner IDs are allowed
AllowMiners []string
// DenyMiners lists miner IDs which are denied to store their sector data into
// this path
DenyMiners []string
}
type HealthReport struct {
@ -104,8 +112,10 @@ type SectorStorageInfo struct {
Primary bool
AllowTypes []string
DenyTypes []string
AllowTypes []string
DenyTypes []string
AllowMiners []string
DenyMiners []string
}
type Decl struct {

8
storage/sealer/storiface/storage.go
View File

@ -225,4 +225,12 @@ type LocalStorageMeta struct {
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
// AllowMiners lists miner IDs which are allowed to store their sector data into
// this path. If empty, all miner IDs are allowed
AllowMiners []string
// DenyMiners lists miner IDs which are denied to store their sector data into
// this path
DenyMiners []string
}