cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

v1.27.0-a #10

Closed
jonathanface wants to merge 473 commits from v1.27.0-a into master
pull from: v1.27.0-a
merge into: cerc-io:master
Conversation 0 Commits 473 Files Changed 452 +834 -123
23 changed files with 834 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 2951d038a8 - Show all commits

2
api/api_storage.go
View File

@ -200,7 +200,7 @@ type StorageMiner interface {
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
// Paths with more weight and more % of free space are preferred.
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) //perm:admin
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin

8
api/proxy_gen.go
View File

@ -1185,7 +1185,7 @@ type StorageMinerMethods struct {
StorageAuthVerify func(p0 context.Context, p1 string) ([]auth.Permission, error) `perm:"read"`
StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) `perm:"admin"`
StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) `perm:"admin"`
StorageDeclareSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error `perm:"admin"`
@ -6988,14 +6988,14 @@ func (s *StorageMinerStub) StorageAuthVerify(p0 context.Context, p1 string) ([]a
return *new([]auth.Permission), ErrNotSupported
}
func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {
if s.Internal.StorageBestAlloc == nil {
return *new([]storiface.StorageInfo), ErrNotSupported
}
return s.Internal.StorageBestAlloc(p0, p1, p2, p3)
return s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)
}
func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {
return *new([]storiface.StorageInfo), ErrNotSupported
}

91
build/openrpc/miner.json
View File

@ -8897,11 +8897,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
@ -8920,6 +8932,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
@ -9078,7 +9096,7 @@
},
{
"name": "Filecoin.StorageBestAlloc",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3)\n}\n```",
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)\n}\n```",
"summary": "StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.\nPaths with more weight and more % of free space are preferred.\nNote: This method doesn't filter paths based on AllowTypes/DenyTypes.\n",
"paramStructure": "by-position",
"params": [
@ -9130,6 +9148,23 @@
},
"required": true,
"deprecated": false
},
{
"name": "p4",
"description": "abi.ActorID",
"summary": "",
"schema": {
"title": "number",
"description": "Number is a number",
"examples": [
1000
],
"type": [
"number"
]
},
"required": true,
"deprecated": false
}
],
"result": {
@ -9159,6 +9194,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -9167,6 +9208,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
@ -9185,6 +9232,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
@ -9620,6 +9673,12 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -9628,6 +9687,12 @@
{
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTypes": {
"items": {
"type": "string"
@ -9646,6 +9711,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"
@ -9833,11 +9904,23 @@
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
],
"additionalProperties": false,
"properties": {
"AllowMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"AllowTo": {
"items": {
"type": "string"
@ -9856,6 +9939,12 @@
"CanStore": {
"type": "boolean"
},
"DenyMiners": {
"items": {
"type": "string"
},
"type": "array"
},
"DenyTypes": {
"items": {
"type": "string"

18
cmd/curio/tasks/tasks.go
View File

@ -8,6 +8,8 @@ import (
"github.com/samber/lo"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/cmd/curio/deps"
curio "github.com/filecoin-project/lotus/curiosrc"
"github.com/filecoin-project/lotus/curiosrc/chainsched"
@ -37,6 +39,20 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
si := dependencies.Si
var activeTasks []harmonytask.TaskInterface
// Get all miner address from config
var miners []address.Address
for _, ad := range cfg.Addresses {
ad := ad
for _, m := range ad.MinerAddresses {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("failed to parse the miner address: %w", err)
}
miners = append(miners, maddr)
}
}
sender, sendTask := message.NewSender(full, full, db)
activeTasks = append(activeTasks, sendTask)
@ -76,7 +92,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
{
// Piece handling
if cfg.Subsystems.EnableParkPiece {
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks)
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks, miners)
cleanupPieceTask := piece.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0)
activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask)
}

2
curiosrc/ffi/task_storage.go
View File

@ -68,7 +68,7 @@ func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (Se
func (t *TaskStorage) HasCapacity() bool {
ctx := context.Background()
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType)
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType, abi.ActorID(0))
if err != nil {
log.Errorf("finding best alloc in HasCapacity: %+v", err)
return false

5
curiosrc/piece/task_park_piece.go
View File

@ -9,6 +9,8 @@ import (
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/curiosrc/ffi"
"github.com/filecoin-project/lotus/curiosrc/seal"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
@ -32,13 +34,14 @@ type ParkPieceTask struct {
max int
}
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int) *ParkPieceTask {
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int, miners []address.Address) *ParkPieceTask {
pt := &ParkPieceTask{
db: db,
sc: sc,
max: max,
}
go pt.pollPieceTasks(context.Background())
return pt
}

8
curiosrc/web/hapi/simpleinfo.go
View File

@ -393,6 +393,8 @@ type machineInfo struct {
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string
@ -458,7 +460,7 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
}
// query storage info
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, allow_miners, deny_miners, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
if err != nil {
return nil, err
}
@ -481,13 +483,15 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
FSAvailable int64
Reserved int64
Used int64
AllowMiners string
DenyMiners string
LastHeartbeat time.Time
HeartbeatErr *string
UsedPercent float64
ReservedPercent float64
}
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.AllowMiners, &s.DenyMiners, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
return nil, err
}

12
documentation/en/api-v0-methods-curio.md
View File

@ -179,6 +179,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -218,6 +224,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```

27
documentation/en/api-v0-methods-miner.md
View File

@ -3735,6 +3735,12 @@ Inputs:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
},
{
@ -3782,7 +3788,8 @@ Inputs:
[
1,
34359738368,
"sealing"
"sealing",
1000
]
```
@ -3809,6 +3816,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -3928,6 +3941,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
]
@ -4004,6 +4023,12 @@ Response:
],
"DenyTypes": [
"string value"
],
"AllowMiners": [
"string value"
],
"DenyMiners": [
"string value"
]
}
```

1
lib/harmony/harmonydb/sql/20240401-storage-segregation.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS allow_miners varchar, ADD COLUMN IF NOT EXISTS deny_miners varchar; -- comma separated list of miner addresses

131
storage/paths/db_index.go
View File

@ -15,6 +15,7 @@ import (
"go.opencensus.io/tag"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/journal/alerting"
@ -200,7 +201,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
currUrls = union(currUrls, si.URLs)
_, err = tx.Exec(
"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, last_heartbeat=NOW() WHERE storage_id=$10",
"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, allow_miners=$10, deny_miners=$11, last_heartbeat=NOW() WHERE storage_id=$12",
strings.Join(currUrls, ","),
si.Weight,
si.MaxStorage,
@ -210,6 +211,8 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
strings.Join(si.AllowTo, ","),
strings.Join(si.AllowTypes, ","),
strings.Join(si.DenyTypes, ","),
strings.Join(si.AllowMiners, ","),
strings.Join(si.DenyMiners, ","),
si.ID)
if err != nil {
return false, xerrors.Errorf("storage attach UPDATE fails: %v", err)
@ -221,7 +224,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
// Insert storage id
_, err = tx.Exec(
"INSERT INTO storage_path "+
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW())",
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), $16, $17)",
si.ID,
strings.Join(si.URLs, ","),
si.Weight,
@ -236,7 +239,9 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
st.Available,
st.FSAvailable,
st.Reserved,
st.Used)
st.Used,
strings.Join(si.AllowMiners, ","),
strings.Join(si.DenyMiners, ","))
if err != nil {
return false, xerrors.Errorf("StorageAttach insert fails: %v", err)
}
@ -540,6 +545,8 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
Groups string
AllowTypes string
DenyTypes string
AllowMiners string
DenyMiners string
}
err = dbi.harmonyDB.Select(ctx, &rows,
`SELECT storage_id,
@ -549,7 +556,9 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
can_store,
groups,
allow_types,
deny_types
deny_types,
allow_miners,
deny_miners
FROM storage_path
WHERE can_seal=true
and available >= $1
@ -570,6 +579,53 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
continue
}
if len(row.AllowMiners) > 0 {
found := false
allowMiners := splitString(row.AllowMiners)
for _, m := range allowMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if !found {
log.Debugf("not selecting on %s, not allowed by allow miners filters", row.StorageId)
continue
}
}
if len(row.DenyMiners) > 0 {
found := false
denyMiners := splitString(row.DenyMiners)
for _, m := range denyMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if found {
log.Debugf("not selecting on %s, not allowed by deny miners filters", row.StorageId)
continue
}
}
if allowList != nil {
groups := splitString(row.Groups)
allow := false
@ -627,10 +683,12 @@ func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface
AllowTo string
AllowTypes string
DenyTypes string
AllowMiners string
DenyMiners string
}
err := dbi.harmonyDB.Select(ctx, &qResults,
"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types "+
"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, allow_miners, deny_miners "+
"FROM storage_path WHERE storage_id=$1", string(id))
if err != nil {
return storiface.StorageInfo{}, xerrors.Errorf("StorageInfo query fails: %v", err)
@ -647,11 +705,13 @@ func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface
sinfo.AllowTo = splitString(qResults[0].AllowTo)
sinfo.AllowTypes = splitString(qResults[0].AllowTypes)
sinfo.DenyTypes = splitString(qResults[0].DenyTypes)
sinfo.AllowMiners = splitString(qResults[0].AllowMiners)
sinfo.DenyMiners = splitString(qResults[0].DenyMiners)
return sinfo, nil
}
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) {
var err error
var spaceReq uint64
switch pathType {
@ -677,6 +737,8 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
AllowTo string
AllowTypes string
DenyTypes string
AllowMiners string
DenyMiners string
}
err = dbi.harmonyDB.Select(ctx, &rows,
@ -689,7 +751,9 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
groups,
allow_to,
allow_types,
deny_types
deny_types,
allow_miners,
deny_miners
FROM storage_path
WHERE available >= $1
and NOW()-($2 * INTERVAL '1 second') < last_heartbeat
@ -700,13 +764,64 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
SkippedHeartbeatThresh.Seconds(),
pathType == storiface.PathSealing,
pathType == storiface.PathStorage,
miner.String(),
)
if err != nil {
return nil, xerrors.Errorf("Querying for best storage sectors fails with err %w: ", err)
}
var result []storiface.StorageInfo
for _, row := range rows {
// Matching with 0 as a workaround to avoid having minerID
// present when calling TaskStorage.HasCapacity()
if !(miner == abi.ActorID(0)) {
if len(row.AllowMiners) > 0 {
found := false
allowMiners := splitString(row.AllowMiners)
for _, m := range allowMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if !found {
continue
}
}
if len(row.DenyMiners) > 0 {
found := false
denyMiners := splitString(row.DenyMiners)
for _, m := range denyMiners {
m := m
maddr, err := address.NewFromString(m)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("parsing miner ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if found {
continue
}
}
}
result = append(result, storiface.StorageInfo{
ID: storiface.ID(row.StorageId),
URLs: splitString(row.Urls),
@ -718,6 +833,8 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
AllowTo: splitString(row.AllowTo),
AllowTypes: splitString(row.AllowTypes),
DenyTypes: splitString(row.DenyTypes),
AllowMiners: splitString(row.AllowMiners),
DenyMiners: splitString(row.DenyMiners),
})
}

145
storage/paths/index.go
View File

@ -14,6 +14,7 @@ import (
"go.opencensus.io/tag"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
@ -38,7 +39,7 @@ type SectorIndex interface { // part of storage-miner api
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)
// atomically acquire locks on all sector file types. close ctx to unlock
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error
@ -61,6 +62,7 @@ type storageEntry struct {
heartbeatErr error
}
// MemIndex represents an in-memory index of storage sectors and storage entries.
type MemIndex struct {
*indexLocks
lk sync.RWMutex
@ -206,6 +208,8 @@ func (i *MemIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
i.stores[si.ID].info.AllowTo = si.AllowTo
i.stores[si.ID].info.AllowTypes = allow
i.stores[si.ID].info.DenyTypes = deny
i.stores[si.ID].info.AllowMiners = si.AllowMiners
i.stores[si.ID].info.DenyMiners = si.DenyMiners
return nil
}
@ -478,6 +482,8 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
AllowMiners: st.info.AllowMiners,
DenyMiners: st.info.DenyMiners,
})
}
@ -492,6 +498,52 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
continue
}
if len(st.info.AllowMiners) > 0 {
found := false
for _, m := range st.info.AllowMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if !found {
log.Debugf("not allocating on %s, miner %s not allowed", st.info.ID, s.Miner.String())
continue
}
}
if len(st.info.DenyMiners) > 0 {
found := false
for _, m := range st.info.DenyMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == s.Miner {
found = true
break
}
}
if found {
log.Debugf("not allocating on %s, miner %s denied", st.info.ID, s.Miner.String())
continue
}
}
if spaceReq > uint64(st.fsi.Available) {
log.Debugf("not selecting on %s, out of space (available: %d, need: %d)", st.info.ID, st.fsi.Available, spaceReq)
continue
@ -557,6 +609,8 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
AllowTypes: st.info.AllowTypes,
DenyTypes: st.info.DenyTypes,
AllowMiners: st.info.AllowMiners,
DenyMiners: st.info.DenyMiners,
})
}
}
@ -564,6 +618,21 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
return out, nil
}
// StorageInfo retrieves the storage information for a given storage ID.
//
// The method first acquires a read lock on the MemIndex to ensure thread-safety.
// It then checks if the storage ID exists in the stores map. If not, it returns
// an error indicating that the sector store was not found.
//
// Finally, it returns the storage information of the selected storage.
//
// Parameters:
// - ctx: the context.Context object for cancellation and timeouts
// - id: the ID of the storage to retrieve information for
//
// Returns:
// - storiface.StorageInfo: the storage information of the selected storage ID
// - error: an error indicating any issues encountered during the process
func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {
i.lk.RLock()
defer i.lk.RUnlock()
@ -576,7 +645,33 @@ func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.
return *si.info, nil
}
func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
// StorageBestAlloc selects the best available storage options for allocating
// a sector file. It takes into account the allocation type (sealing or storage),
// sector size, and path type (sealing or storage).
//
// The method first estimates the required space for the allocation based on the
// sector size and path type. It then iterates through all available storage options
// and filters out those that cannot be used for the given path type. It also filters
// out storage options that do not have enough available space or have not received
// heartbeats within a certain threshold.
//
// The remaining storage options are sorted based on their available space and weight,
// with higher availability and weight being prioritized. The method then returns
// the information of the selected storage options.
//
// If no suitable storage options are found, it returns an error indicating that
// no good path is available.
//
// Parameters:
// - ctx: the context.Context object for cancellation and timeouts
// - allocate: the type of allocation (sealing or storage)
// - ssize: the size of the sector file
// - pathType: the path type (sealing or storage)
//
// Returns:
// - []storiface.StorageInfo: the information of the selected storage options
// - error: an error indicating any issues encountered during the process
func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) {
i.lk.RLock()
defer i.lk.RUnlock()
@ -604,6 +699,52 @@ func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sect
continue
}
if len(p.info.AllowMiners) > 0 {
found := false
for _, m := range p.info.AllowMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if !found {
log.Debugf("not allocating on %s, miner %s not allowed", p.info.ID, miner.String())
continue
}
}
if len(p.info.DenyMiners) > 0 {
found := false
for _, m := range p.info.DenyMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if found {
log.Debugf("not allocating on %s, miner %s denied", p.info.ID, miner.String())
continue
}
}
if spaceReq > uint64(p.fsi.Available) {
log.Debugf("not allocating on %s, out of space (available: %d, need: %d)", p.info.ID, p.fsi.Available, spaceReq)
continue

58
storage/paths/index_test.go
View File

@ -152,3 +152,61 @@ func TestFindAllow(t *testing.T) {
}
}
}
func TestStorageBestAlloc(t *testing.T) {
idx := NewMemIndex(nil)
dummyStorageInfo := storiface.StorageInfo{
ID: storiface.ID("dummy"),
CanSeal: true,
CanStore: true,
URLs: []string{"http://localhost:9999/"},
Weight: 10,
AllowMiners: []string{
"t001",
},
}
dummyFsStat := fsutil.FsStat{
Capacity: 10000000000,
Available: 7000000000,
Reserved: 100000000,
Used: 3000000000,
}
err := idx.StorageAttach(context.Background(), dummyStorageInfo, dummyFsStat)
require.NoError(t, err)
t.Run("PathSealing", func(t *testing.T) {
result, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 1)
require.Equal(t, err, nil)
require.NotNil(t, result)
require.Equal(t, len(result), 1)
require.Equal(t, result[0].ID, dummyStorageInfo.ID)
})
t.Run("PathStorage", func(t *testing.T) {
result, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathStorage, 1)
require.Equal(t, err, nil)
require.NotNil(t, result)
require.Equal(t, len(result), 1)
require.Equal(t, result[0].ID, dummyStorageInfo.ID)
})
t.Run("NotAllowedMiner", func(t *testing.T) {
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 2)
require.Error(t, err)
})
t.Run("NoAvailableSpace", func(t *testing.T) {
bigSectorSize := abi.SectorSize(10000000000)
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, bigSectorSize, storiface.PathSealing, 1)
require.Error(t, err)
})
t.Run("AllowedMiner", func(t *testing.T) {
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 1)
require.NoError(t, err)
})
}

79
storage/paths/local.go
View File

@ -14,6 +14,7 @@ import (
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/proof"
@ -214,6 +215,8 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
AllowMiners: meta.AllowMiners,
DenyMiners: meta.DenyMiners,
}, fst)
if err != nil {
return xerrors.Errorf("declaring storage in index: %w", err)
@ -304,6 +307,8 @@ func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMiss
AllowTo: meta.AllowTo,
AllowTypes: meta.AllowTypes,
DenyTypes: meta.DenyTypes,
AllowMiners: meta.AllowMiners,
DenyMiners: meta.DenyMiners,
}, fst)
if err != nil {
return xerrors.Errorf("redeclaring storage in index: %w", err)
@ -495,20 +500,64 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
var out storiface.SectorPaths
var storageIDs storiface.SectorPaths
allocPathOk := func(canSeal, canStore bool, allowTypes, denyTypes []string, fileType storiface.SectorFileType) bool {
allocPathOk := func(canSeal, canStore bool, allowTypes, denyTypes, allowMiners, denyMiners []string, fileType storiface.SectorFileType, miner abi.ActorID) (bool, error) {
if (pathType == storiface.PathSealing) && !canSeal {
return false
return false, nil
}
if (pathType == storiface.PathStorage) && !canStore {
return false
return false, nil
}
if !fileType.Allowed(allowTypes, denyTypes) {
return false
return false, nil
}
return true
if len(allowMiners) > 0 {
found := false
for _, m := range allowMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return false, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return false, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if !found {
return false, nil
}
}
if len(denyMiners) > 0 {
found := false
for _, m := range denyMiners {
minerIDStr := m
maddr, err := address.NewFromString(minerIDStr)
if err != nil {
return false, xerrors.Errorf("parsing miner address: %w", err)
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return false, xerrors.Errorf("converting miner address to ID: %w", err)
}
if abi.ActorID(mid) == miner {
found = true
break
}
}
if found {
return false, nil
}
}
return true, nil
}
// First find existing files
@ -536,9 +585,16 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
if allocate.Has(fileType) && !allocPathOk(info.CanSeal, info.CanStore, info.AllowTypes, info.DenyTypes, fileType) {
if allocate.Has(fileType) {
ok, err := allocPathOk(info.CanSeal, info.CanStore, info.AllowTypes, info.DenyTypes, info.AllowMiners, info.DenyMiners, fileType, sid.ID.Miner)
if err != nil {
log.Debug(err)
continue
}
if !ok {
continue // allocate request for a path of different type
}
}
spath := p.sectorPath(sid.ID, fileType)
storiface.SetPathByType(&out, fileType, spath)
@ -556,7 +612,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType)
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType, sid.ID.Miner)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("finding best storage for allocating : %w", err)
}
@ -574,7 +630,14 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
continue
}
if !allocPathOk(si.CanSeal, si.CanStore, si.AllowTypes, si.DenyTypes, fileType) {
alloc, err := allocPathOk(si.CanSeal, si.CanStore, si.AllowTypes, si.DenyTypes, si.AllowMiners, si.DenyMiners, fileType, sid.ID.Miner)
if err != nil {
log.Debug(err)
continue
}
if !alloc {
continue
}

8
storage/paths/mocks/index.go
View File

@ -54,18 +54,18 @@ func (mr *MockSectorIndexMockRecorder) StorageAttach(arg0, arg1, arg2 interface{
}
// StorageBestAlloc mocks base method.
func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType) ([]storiface.StorageInfo, error) {
func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType, arg4 abi.ActorID) ([]storiface.StorageInfo, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3)
ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].([]storiface.StorageInfo)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StorageBestAlloc indicates an expected call of StorageBestAlloc.
func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3, arg4)
}
// StorageDeclareSector mocks base method.

32
storage/sealer/manager.go
View File

@ -356,7 +356,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto
// selector will schedule the Unseal task on a worker that either already has the sealed sector files or has space in
// one of it's sealing scratch spaces to store them after fetching them from another worker.
selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, true)
selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, sector.ID.Miner, true)
log.Debugf("will schedule unseal for sector %d", sector.ID)
err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error {
@ -377,7 +377,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto
}
// get a selector for moving unsealed sector into long-term storage
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTUnsealed, storiface.PathStorage, !m.disallowRemoteFinalize)
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTUnsealed, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
// move unsealed sector to long-term storage
// Possible TODO: Add an option to not keep the unsealed sector in long term storage?
@ -431,9 +431,9 @@ func (m *Manager) AddPiece(ctx context.Context, sector storiface.SectorRef, exis
var selector WorkerSelector
var err error
if len(existingPieces) == 0 { // new
selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing)
selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing, sector.ID.Miner)
} else { // use existing
selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false)
selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, sector.ID.Miner, false)
}
var out abi.PieceInfo
@ -484,7 +484,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector storiface.SectorRef
// TODO: also consider where the unsealed data sits
selector := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed, storiface.PathSealing)
selector := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, sector.ID.Miner)
err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit1, selector, m.schedFetch(sector, storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.SealPreCommit1(ctx, sector, ticket, pieces))
@ -533,7 +533,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector storiface.SectorRef
return storiface.SectorCids{}, xerrors.Errorf("acquiring sector lock: %w", err)
}
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, true)
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, sector.ID.Miner, true)
err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit2, selector, m.schedFetch(sector, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.SealPreCommit2(ctx, sector, phase1Out))
@ -585,7 +585,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sector storiface.SectorRef, t
// NOTE: We set allowFetch to false in so that we always execute on a worker
// with direct access to the data. We want to do that because this step is
// generally very cheap / fast, and transferring data is not worth the effort
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, sector.ID.Miner, false)
err = m.sched.Schedule(ctx, sector, sealtasks.TTCommit1, selector, m.schedFetch(sector, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.SealCommit1(ctx, sector, ticket, seed, pieces, cids))
@ -700,7 +700,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storiface.SectorRef
// do the cache trimming wherever the likely still very large cache lives.
// we really don't want to move it.
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache, sector.ID.Miner, false)
err = m.sched.Schedule(ctx, sector, sealtasks.TTFinalize, selector,
m.schedFetch(sector, storiface.FTCache, cachePathType, storiface.AcquireMove),
@ -713,7 +713,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storiface.SectorRef
}
// get a selector for moving stuff into long-term storage
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, storiface.PathStorage, !m.disallowRemoteFinalize)
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
// only move the unsealed file if it still exists and needs moving
moveUnsealed := storiface.FTUnsealed
@ -769,7 +769,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
// do the cache trimming wherever the likely still large cache lives.
// we really don't want to move it.
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdateCache, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdateCache, sector.ID.Miner, false)
err := m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeReplicaUpdate, selector,
m.schedFetch(sector, storiface.FTCache|storiface.FTUpdateCache, pathType, storiface.AcquireMove),
@ -783,7 +783,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
move := func(types storiface.SectorFileType) error {
// get a selector for moving stuff into long-term storage
fetchSel := newMoveSelector(m.index, sector.ID, types, storiface.PathStorage, !m.disallowRemoteFinalize)
fetchSel := newMoveSelector(m.index, sector.ID, types, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
err = m.sched.Schedule(ctx, sector, sealtasks.TTFetch, fetchSel,
m.schedFetch(sector, types, storiface.PathStorage, storiface.AcquireMove),
@ -834,7 +834,7 @@ func (m *Manager) ReleaseUnsealed(ctx context.Context, sector storiface.SectorRe
return nil
}
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, sector.ID.Miner, false)
return m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeUnsealed, selector, m.schedFetch(sector, storiface.FTUnsealed, pathType, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
_, err := m.waitSimpleCall(ctx)(w.ReleaseUnsealed(ctx, sector, keepUnsealed))
@ -906,7 +906,7 @@ func (m *Manager) GenerateSectorKeyFromData(ctx context.Context, sector storifac
// NOTE: We set allowFetch to false in so that we always execute on a worker
// with direct access to the data. We want to do that because this step is
// generally very cheap / fast, and transferring data is not worth the effort
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, true)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, sector.ID.Miner, true)
err = m.sched.Schedule(ctx, sector, sealtasks.TTRegenSectorKey, selector, m.schedFetch(sector, storiface.FTUpdate|storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.GenerateSectorKeyFromData(ctx, sector, commD))
@ -984,7 +984,7 @@ func (m *Manager) ReplicaUpdate(ctx context.Context, sector storiface.SectorRef,
return storiface.ReplicaUpdateOut{}, xerrors.Errorf("acquiring sector lock: %w", err)
}
selector := newAllocSelector(m.index, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing)
selector := newAllocSelector(m.index, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, sector.ID.Miner)
err = m.sched.Schedule(ctx, sector, sealtasks.TTReplicaUpdate, selector, m.schedFetch(sector, storiface.FTUnsealed|storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.ReplicaUpdate(ctx, sector, pieces))
@ -1035,7 +1035,7 @@ func (m *Manager) ProveReplicaUpdate1(ctx context.Context, sector storiface.Sect
// NOTE: We set allowFetch to false in so that we always execute on a worker
// with direct access to the data. We want to do that because this step is
// generally very cheap / fast, and transferring data is not worth the effort
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, false)
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, sector.ID.Miner, false)
err = m.sched.Schedule(ctx, sector, sealtasks.TTProveReplicaUpdate1, selector, m.schedFetch(sector, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
@ -1153,7 +1153,7 @@ func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.Secto
ptype = storiface.PathStorage
}
selector := newAllocSelector(m.index, toFetch, ptype)
selector := newAllocSelector(m.index, toFetch, ptype, sector.ID.Miner)
err = m.sched.Schedule(ctx, sector, sealtasks.TTDownloadSector, selector, schedNop, func(ctx context.Context, w Worker) error {
err := m.startWork(ctx, w, wk)(w.DownloadSectorData(ctx, sector, finalized, src))

2
storage/sealer/sched_test.go
View File

@ -274,7 +274,7 @@ func TestSched(t *testing.T) {
done := make(chan struct{})
rm.done[taskName] = done
sel := newAllocSelector(index, storiface.FTCache, storiface.PathSealing)
sel := newAllocSelector(index, storiface.FTCache, storiface.PathSealing, abi.ActorID(1000))
rm.wg.Add(1)
go func() {

6
storage/sealer/selector_alloc.go
View File

@ -16,13 +16,15 @@ type allocSelector struct {
index paths.SectorIndex
alloc storiface.SectorFileType
ptype storiface.PathType
miner abi.ActorID
}
func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType) *allocSelector {
func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType, miner abi.ActorID) *allocSelector {
return &allocSelector{
index: index,
alloc: alloc,
ptype: ptype,
miner: miner,
}
}
@ -50,7 +52,7 @@ func (s *allocSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi
return false, false, xerrors.Errorf("getting sector size: %w", err)
}
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype)
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype, s.miner)
if err != nil {
return false, false, xerrors.Errorf("finding best alloc storage: %w", err)
}

4
storage/sealer/selector_existing.go
View File

@ -16,14 +16,16 @@ type existingSelector struct {
index paths.SectorIndex
sector abi.SectorID
fileType storiface.SectorFileType
miner abi.ActorID
allowFetch bool
}
func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, allowFetch bool) *existingSelector {
func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, miner abi.ActorID, allowFetch bool) *existingSelector {
return &existingSelector{
index: index,
sector: sector,
fileType: alloc,
miner: miner,
allowFetch: allowFetch,
}
}

5
storage/sealer/selector_move.go
View File

@ -17,10 +17,11 @@ type moveSelector struct {
sector abi.SectorID
alloc storiface.SectorFileType
destPtype storiface.PathType
miner abi.ActorID
allowRemote bool
}
func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, allowRemote bool) *moveSelector {
func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, miner abi.ActorID, allowRemote bool) *moveSelector {
return &moveSelector{
index: index,
sector: sector,
@ -67,7 +68,7 @@ func (s *moveSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.
}
}
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.destPtype)
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.destPtype, s.miner)
if err != nil {
return false, false, xerrors.Errorf("finding best dest storage: %w", err)
}

159
storage/sealer/storiface/filetype.go
View File

@ -8,6 +8,28 @@ import (
"github.com/filecoin-project/go-state-types/abi"
)
// FTUnsealed represents an unsealed sector file type.
// FTSealed represents a sealed sector file type.
// FTCache represents a cache sector file type.
// FTUpdate represents an update sector file type.
// FTUpdateCache represents an update cache sector file type.
// FTPiece represents a Piece Park sector file type.
// FileTypes represents the total number of file types.
//
// The SectorFileType type is an integer type that represents different sector file types.
// It has several methods to manipulate and query the file type.
// The String method returns a string representation of the file type.
// The Strings method returns a slice of string representations of all the file types that are set in the receiver object.
// The AllSet method returns a slice of all the file types that are set in the receiver object.
// The Has method checks whether a specific file type is set in the receiver object.
// The SealSpaceUse method calculates the space used by the receiver object in sealing a sector of a given size.
// The SubAllowed method removes selected file types from the receiver object based on a list of allowed and denied file types.
// The Unset method removes selected file types from the receiver object.
// The AnyAllowed method checks whether any file types in the receiver object are allowed based on a list of allowed and denied file types.
// The Allowed method checks whether all file types in the receiver object are allowed based on a list of allowed and denied file types.
// The StoreSpaceUse method calculates the space used by the receiver object in storing a sector of a given size.
// The All method returns an array that represents which file types are set in the receiver object.
// The IsNone method checks whether the receiver object represents no file types.
const (
// "regular" sectors
FTUnsealed SectorFileType = 1 << iota
@ -24,12 +46,28 @@ const (
FileTypes = iota
)
// PathTypes is a slice of SectorFileType that represents different types of sector file paths.
// It contains the following types of sector file paths:
// - FTUnsealed: represents regular unsealed sectors
// - FTSealed: represents sealed sectors
// - FTCache: represents cache sectors
// - FTUpdate: represents snap sectors
// - FTUpdateCache: represents snap cache sectors
// - FTPiece: represents Piece Park sectors
var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache, FTUpdate, FTUpdateCache, FTPiece}
// FTNone represents a sector file type of none. This constant is used in the StorageLock method to specify that a sector should not have any file types locked.
// Example usage:
// err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache)
const (
FTNone SectorFileType = 0
)
// FTAll represents the combination of all available sector file types.
// It is a variable of type SectorFileType.
// The value of FTAll is calculated by iterating over the PathTypes slice and using the |= operator to perform a bitwise OR operation on each path type.
// The result is assigned to the variable out and returned.
// FTAll is immediately invoked as a function using the anonymous function syntax, so the result is returned as soon as it is calculated.
var FTAll = func() (out SectorFileType) {
for _, pathType := range PathTypes {
out |= pathType
@ -37,8 +75,10 @@ var FTAll = func() (out SectorFileType) {
return out
}()
// FSOverheadDen represents the constant value 10, which is used to calculate the overhead in various storage space utilization calculations.
const FSOverheadDen = 10
// FSOverheadSeal is a map that represents the overheads for different SectorFileType in sealed sectors.
var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
FTUnsealed: FSOverheadDen,
FTSealed: FSOverheadDen,
@ -50,6 +90,30 @@ var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
// sector size * disk / fs overhead. FSOverheadDen is like the unit of sector size
// FsOverheadFinalized is a map that represents the finalized overhead for different types of SectorFileType.
// The keys in the map are the SectorFileType values, and the values are integers representing the overhead.
// It is used to calculate the storage space usage for different types of sectors, as shown in the example below:
//
// func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
// var need uint64
// for _, pathType := range PathTypes {
// if !t.Has(pathType) {
// continue
// }
//
// oh, ok := FsOverheadFinalized[pathType]
// if !ok {
// return 0, xerrors.Errorf("no finalized overhead info for %s", pathType)
// }
//
// need += uint64(oh) * uint64(ssize) / FSOverheadDen
// }
//
// return need, nil
// }
//
// The overhead value is retrieved from FsOverheadFinalized by using the SectorFileType value as the key.
// If the overhead value is not found in the map, an error is returned indicating that there is no finalized overhead information for the given sector type.
var FsOverheadFinalized = map[SectorFileType]int{
FTUnsealed: FSOverheadDen,
FTSealed: FSOverheadDen,
@ -59,8 +123,13 @@ var FsOverheadFinalized = map[SectorFileType]int{
FTPiece: FSOverheadDen,
}
// SectorFileType represents the type of a sector file
// TypeFromString converts a string to a SectorFileType
type SectorFileType int
// TypeFromString converts a string representation of a SectorFileType to its corresponding value.
// It returns the SectorFileType and nil error if the string matches one of the existing types.
// If the string does not match any type, it returns 0 and an error.
func TypeFromString(s string) (SectorFileType, error) {
switch s {
case "unsealed":
@ -80,6 +149,7 @@ func TypeFromString(s string) (SectorFileType, error) {
}
}
// String returns a string representation of the SectorFileType.
func (t SectorFileType) String() string {
switch t {
case FTUnsealed:
@ -99,6 +169,12 @@ func (t SectorFileType) String() string {
}
}
// Strings returns a slice of strings representing the names of the SectorFileType values that are set in the receiver value.
// Example usage:
//
// fileType := SectorFileType(FTSealed | FTCache)
// names := fileType.Strings() // names = ["sealed", "cache"]
// fmt.Println(names)
func (t SectorFileType) Strings() []string {
var out []string
for _, fileType := range PathTypes {
@ -111,6 +187,7 @@ func (t SectorFileType) Strings() []string {
return out
}
// AllSet returns a slice of SectorFileType values that are set in the SectorFileType receiver value
func (t SectorFileType) AllSet() []SectorFileType {
var out []SectorFileType
for _, fileType := range PathTypes {
@ -123,10 +200,33 @@ func (t SectorFileType) AllSet() []SectorFileType {
return out
}
// Has checks if the SectorFileType has a specific singleType.
func (t SectorFileType) Has(singleType SectorFileType) bool {
return t&singleType == singleType
}
// SealSpaceUse calculates the amount of space needed for sealing the sector
// based on the given sector size. It iterates over the different path types
// and calculates the space needed for each path type using the FSOverheadSeal
// map. The overhead value is multiplied by the sector size and divided by the
// FSOverheadDen constant. The total space needed is accumulated and returned.
// If there is no seal overhead information for a particular path type, an error
// is returned.
//
// Example usage:
//
// fileType := FTSealed | FTCache
// sectorSize := abi.SectorSize(32 << 20) // 32 MiB
// spaceNeeded, err := fileType.SealSpaceUse(sectorSize)
//
// Parameters:
//
// ssize: The size of the sector
//
// Returns:
//
// uint64: The amount of space needed for sealing the sector
// error: If there is no seal overhead information for a path type
func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
var need uint64
for _, pathType := range PathTypes {
@ -145,6 +245,15 @@ func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
return need, nil
}
// The method takes in two parameters: allowTypes and denyTypes, both of which are slices of strings.
// If allowTypes is not empty, the method sets a denyMask with all bits set to 1, and then iterates over each allowType,
// converting it to a SectorFileType using the TypeFromString function and unsetting the corresponding bit in the denyMask.
// If a string in allowTypes cannot be converted to a valid SectorFileType, it is ignored.
// After processing allowTypes, the method iterates over each denyType, converting it to a SectorFileType using the TypeFromString function
// and setting the corresponding bit in the denyMask.
// If a string in denyTypes cannot be converted to a valid SectorFileType, it is ignored.
// Finally, the method returns the bitwise AND of the original SectorFileType and the denyMask.
// The returned SectorFileType will only allow the types specified in allowTypes and exclude the types specified in denyTypes.`
func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType {
var denyMask SectorFileType // 1s deny
@ -174,18 +283,30 @@ func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) Sect
return t & denyMask
}
// Unset removes the specified sector file type(s) from the current SectorFileType value.
// It performs a bitwise AND operation between the current value and the bitwise complement of the toUnset value.
// The result is returned as a new SectorFileType value.
// Any bits that are set in toUnset will be cleared in the result.
// Usage: result = value.Unset(typesToUnset)
func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType {
return t &^ toUnset
}
// AnyAllowed checks if the SectorFileType has any allowed types and no denied types.
func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) != t
}
// Allowed checks if the SectorFileType is allowed based on the given allowTypes and denyTypes.
// Returns true if the SectorFileType is allowed, otherwise false.
func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool {
return t.SubAllowed(allowTypes, denyTypes) == 0
}
// StoreSpaceUse calculates the space used for storing sectors of a specific file type.
// It takes the sector size as input and returns the total space needed in bytes and an error, if any.
// The calculation is based on the finalized overhead information for the file type.
// If the overhead information is not available for a particular file type, an error will be returned.
func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
var need uint64
for _, pathType := range PathTypes {
@ -204,6 +325,7 @@ func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
return need, nil
}
// All returns an array indicating whether each FileTypes flag is set in the SectorFileType.
func (t SectorFileType) All() [FileTypes]bool {
var out [FileTypes]bool
@ -214,10 +336,13 @@ func (t SectorFileType) All() [FileTypes]bool {
return out
}
// IsNone checks if the SectorFileType value is equal to zero.
// It returns true if the value is zero, indicating that the type is none.
func (t SectorFileType) IsNone() bool {
return t == 0
}
// SectorPaths represents the paths for different sector files.
type SectorPaths struct {
ID abi.SectorID
@ -229,6 +354,7 @@ type SectorPaths struct {
Piece string
}
// HasAllSet checks if all paths of a SectorPaths struct are set for a given SectorFileType.
func (sp SectorPaths) HasAllSet(ft SectorFileType) bool {
for _, fileType := range ft.AllSet() {
if PathByType(sp, fileType) == "" {
@ -239,6 +365,9 @@ func (sp SectorPaths) HasAllSet(ft SectorFileType) bool {
return true
}
// Subset returns a new instance of SectorPaths that contains only the paths specified by the filter SectorFileType.
// It iterates over each fileType in the filter, retrieves the corresponding path from the original SectorPaths instance, and sets it in the new instance.
// Finally, it sets the ID field of the new instance to be the same as the original instance.
func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths {
var out SectorPaths
@ -251,6 +380,24 @@ func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths {
return out
}
// ParseSectorID parses a sector ID from a given base name.
// It expects the format "s-t0%d-%d", where the first %d represents the miner ID
// and the second %d represents the sector number.
//
// Parameters:
// - baseName: The base name from which to parse the sector ID.
//
// Returns:
// - abi.SectorID: The parsed sector ID.
// - error: An error if parsing fails.
//
// Example usage:
//
// id, err := ParseSectorID(baseName)
// if err != nil {
// // handle error
// }
// // use id
func ParseSectorID(baseName string) (abi.SectorID, error) {
var n abi.SectorNumber
var mid abi.ActorID
@ -269,10 +416,19 @@ func ParseSectorID(baseName string) (abi.SectorID, error) {
}, nil
}
// SectorName returns the name of a sector in the format "s-t0<Miner>-<Number>"
//
// Parameters:
// - sid: The sector ID
//
// Returns:
// - The name of the sector as a string
func SectorName(sid abi.SectorID) string {
return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number)
}
// PathByType returns the path associated with the specified fileType in the given SectorPaths.
// It panics if the requested path type is unknown.
func PathByType(sps SectorPaths, fileType SectorFileType) string {
switch fileType {
case FTUnsealed:
@ -309,11 +465,14 @@ func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string) {
}
}
// PathsWithIDs represents paths and IDs for sector files.
type PathsWithIDs struct {
Paths SectorPaths
IDs SectorPaths
}
// HasAllSet checks if all paths and IDs in PathsWithIDs have a corresponding path set for the specified SectorFileType.
// It returns true if all paths and IDs are set, and false otherwise.
func (p PathsWithIDs) HasAllSet(ft SectorFileType) bool {
return p.Paths.HasAllSet(ft) && p.IDs.HasAllSet(ft)
}

10
storage/sealer/storiface/index.go
View File

@ -86,6 +86,14 @@ type StorageInfo struct {
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
// AllowMiners lists miner IDs which are allowed to store their sector data into
// this path. If empty, all miner IDs are allowed
AllowMiners []string
// DenyMiners lists miner IDs which are denied to store their sector data into
// this path
DenyMiners []string
}
type HealthReport struct {
@ -106,6 +114,8 @@ type SectorStorageInfo struct {
AllowTypes []string
DenyTypes []string
AllowMiners []string
DenyMiners []string
}
type Decl struct {

8
storage/sealer/storiface/storage.go
View File

@ -225,4 +225,12 @@ type LocalStorageMeta struct {
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
// AllowMiners lists miner IDs which are allowed to store their sector data into
// this path. If empty, all miner IDs are allowed
AllowMiners []string
// DenyMiners lists miner IDs which are denied to store their sector data into
// this path
DenyMiners []string
}