add miner fliter
This commit is contained in:
LexLuthr
Łukasz Magiera
parent
a99d8c8791
commit
2951d038a8
@ -200,7 +200,7 @@ type StorageMiner interface {
|
||||
// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
|
||||
// Paths with more weight and more % of free space are preferred.
|
||||
// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
|
||||
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
|
||||
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) //perm:admin
|
||||
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error //perm:admin
|
||||
StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error) //perm:admin
|
||||
StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error) //perm:admin
|
||||
|
||||
@ -1185,7 +1185,7 @@ type StorageMinerMethods struct {
|
||||
|
||||
StorageAuthVerify func(p0 context.Context, p1 string) ([]auth.Permission, error) `perm:"read"`
|
||||
|
||||
StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) `perm:"admin"`
|
||||
StorageBestAlloc func(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) `perm:"admin"`
|
||||
|
||||
StorageDeclareSector func(p0 context.Context, p1 storiface.ID, p2 abi.SectorID, p3 storiface.SectorFileType, p4 bool) error `perm:"admin"`
|
||||
|
||||
@ -6988,14 +6988,14 @@ func (s *StorageMinerStub) StorageAuthVerify(p0 context.Context, p1 string) ([]a
|
||||
return *new([]auth.Permission), ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
|
||||
func (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {
|
||||
if s.Internal.StorageBestAlloc == nil {
|
||||
return *new([]storiface.StorageInfo), ErrNotSupported
|
||||
}
|
||||
return s.Internal.StorageBestAlloc(p0, p1, p2, p3)
|
||||
return s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)
|
||||
}
|
||||
|
||||
func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {
|
||||
func (s *StorageMinerStub) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {
|
||||
return *new([]storiface.StorageInfo), ErrNotSupported
|
||||
}
|
||||
|
||||
|
||||
@ -8897,11 +8897,23 @@
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"AllowMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"AllowTo": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -8920,6 +8932,12 @@
|
||||
"CanStore": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"DenyMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"DenyTypes": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -9078,7 +9096,7 @@
|
||||
},
|
||||
{
|
||||
"name": "Filecoin.StorageBestAlloc",
|
||||
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3)\n}\n```",
|
||||
"description": "```go\nfunc (s *StorageMinerStruct) StorageBestAlloc(p0 context.Context, p1 storiface.SectorFileType, p2 abi.SectorSize, p3 storiface.PathType, p4 abi.ActorID) ([]storiface.StorageInfo, error) {\n\tif s.Internal.StorageBestAlloc == nil {\n\t\treturn *new([]storiface.StorageInfo), ErrNotSupported\n\t}\n\treturn s.Internal.StorageBestAlloc(p0, p1, p2, p3, p4)\n}\n```",
|
||||
"summary": "StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.\nPaths with more weight and more % of free space are preferred.\nNote: This method doesn't filter paths based on AllowTypes/DenyTypes.\n",
|
||||
"paramStructure": "by-position",
|
||||
"params": [
|
||||
@ -9130,6 +9148,23 @@
|
||||
},
|
||||
"required": true,
|
||||
"deprecated": false
|
||||
},
|
||||
{
|
||||
"name": "p4",
|
||||
"description": "abi.ActorID",
|
||||
"summary": "",
|
||||
"schema": {
|
||||
"title": "number",
|
||||
"description": "Number is a number",
|
||||
"examples": [
|
||||
1000
|
||||
],
|
||||
"type": [
|
||||
"number"
|
||||
]
|
||||
},
|
||||
"required": true,
|
||||
"deprecated": false
|
||||
}
|
||||
],
|
||||
"result": {
|
||||
@ -9159,6 +9194,12 @@
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
]
|
||||
@ -9167,6 +9208,12 @@
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"AllowMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"AllowTo": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -9185,6 +9232,12 @@
|
||||
"CanStore": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"DenyMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"DenyTypes": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -9620,6 +9673,12 @@
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
]
|
||||
@ -9628,6 +9687,12 @@
|
||||
{
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"AllowMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"AllowTypes": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -9646,6 +9711,12 @@
|
||||
"CanStore": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"DenyMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"DenyTypes": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -9833,11 +9904,23 @@
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"AllowMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"AllowTo": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@ -9856,6 +9939,12 @@
|
||||
"CanStore": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"DenyMiners": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"DenyTypes": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
|
||||
@ -8,6 +8,8 @@ import (
|
||||
"github.com/samber/lo"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
|
||||
"github.com/filecoin-project/lotus/cmd/curio/deps"
|
||||
curio "github.com/filecoin-project/lotus/curiosrc"
|
||||
"github.com/filecoin-project/lotus/curiosrc/chainsched"
|
||||
@ -37,6 +39,20 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
|
||||
si := dependencies.Si
|
||||
var activeTasks []harmonytask.TaskInterface
|
||||
|
||||
// Get all miner address from config
|
||||
var miners []address.Address
|
||||
for _, ad := range cfg.Addresses {
|
||||
ad := ad
|
||||
for _, m := range ad.MinerAddresses {
|
||||
m := m
|
||||
maddr, err := address.NewFromString(m)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to parse the miner address: %w", err)
|
||||
}
|
||||
miners = append(miners, maddr)
|
||||
}
|
||||
}
|
||||
|
||||
sender, sendTask := message.NewSender(full, full, db)
|
||||
activeTasks = append(activeTasks, sendTask)
|
||||
|
||||
@ -76,7 +92,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
|
||||
{
|
||||
// Piece handling
|
||||
if cfg.Subsystems.EnableParkPiece {
|
||||
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks)
|
||||
parkPieceTask := piece.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks, miners)
|
||||
cleanupPieceTask := piece.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0)
|
||||
activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask)
|
||||
}
|
||||
|
||||
@ -68,7 +68,7 @@ func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (Se
|
||||
func (t *TaskStorage) HasCapacity() bool {
|
||||
ctx := context.Background()
|
||||
|
||||
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType)
|
||||
paths, err := t.sc.sectors.sindex.StorageBestAlloc(ctx, t.alloc, t.ssize, t.pathType, abi.ActorID(0))
|
||||
if err != nil {
|
||||
log.Errorf("finding best alloc in HasCapacity: %+v", err)
|
||||
return false
|
||||
|
||||
@ -9,6 +9,8 @@ import (
|
||||
logging "github.com/ipfs/go-log/v2"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
|
||||
"github.com/filecoin-project/lotus/curiosrc/ffi"
|
||||
"github.com/filecoin-project/lotus/curiosrc/seal"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
@ -32,13 +34,14 @@ type ParkPieceTask struct {
|
||||
max int
|
||||
}
|
||||
|
||||
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int) *ParkPieceTask {
|
||||
func NewParkPieceTask(db *harmonydb.DB, sc *ffi.SealCalls, max int, miners []address.Address) *ParkPieceTask {
|
||||
pt := &ParkPieceTask{
|
||||
db: db,
|
||||
sc: sc,
|
||||
|
||||
max: max,
|
||||
}
|
||||
|
||||
go pt.pollPieceTasks(context.Background())
|
||||
return pt
|
||||
}
|
||||
|
||||
@ -393,6 +393,8 @@ type machineInfo struct {
|
||||
FSAvailable int64
|
||||
Reserved int64
|
||||
Used int64
|
||||
AllowMiners string
|
||||
DenyMiners string
|
||||
LastHeartbeat time.Time
|
||||
HeartbeatErr *string
|
||||
|
||||
@ -458,7 +460,7 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
|
||||
}
|
||||
|
||||
// query storage info
|
||||
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
|
||||
rows2, err := a.db.Query(ctx, "SELECT storage_id, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, allow_miners, deny_miners, last_heartbeat, heartbeat_err FROM storage_path WHERE urls LIKE '%' || $1 || '%'", summaries[0].Info.Host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -481,13 +483,15 @@ func (a *app) clusterNodeInfo(ctx context.Context, id int64) (*machineInfo, erro
|
||||
FSAvailable int64
|
||||
Reserved int64
|
||||
Used int64
|
||||
AllowMiners string
|
||||
DenyMiners string
|
||||
LastHeartbeat time.Time
|
||||
HeartbeatErr *string
|
||||
|
||||
UsedPercent float64
|
||||
ReservedPercent float64
|
||||
}
|
||||
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
|
||||
if err := rows2.Scan(&s.ID, &s.Weight, &s.MaxStorage, &s.CanSeal, &s.CanStore, &s.Groups, &s.AllowTo, &s.AllowTypes, &s.DenyTypes, &s.Capacity, &s.Available, &s.FSAvailable, &s.Reserved, &s.Used, &s.AllowMiners, &s.DenyMiners, &s.LastHeartbeat, &s.HeartbeatErr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@ -179,6 +179,12 @@ Response:
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
]
|
||||
@ -218,6 +224,12 @@ Response:
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
@ -3735,6 +3735,12 @@ Inputs:
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -3782,7 +3788,8 @@ Inputs:
|
||||
[
|
||||
1,
|
||||
34359738368,
|
||||
"sealing"
|
||||
"sealing",
|
||||
1000
|
||||
]
|
||||
```
|
||||
|
||||
@ -3809,6 +3816,12 @@ Response:
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
]
|
||||
@ -3928,6 +3941,12 @@ Response:
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
]
|
||||
@ -4004,6 +4023,12 @@ Response:
|
||||
],
|
||||
"DenyTypes": [
|
||||
"string value"
|
||||
],
|
||||
"AllowMiners": [
|
||||
"string value"
|
||||
],
|
||||
"DenyMiners": [
|
||||
"string value"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
@ -0,0 +1 @@
|
||||
ALTER TABLE storage_path ADD COLUMN IF NOT EXISTS allow_miners varchar, ADD COLUMN IF NOT EXISTS deny_miners varchar; -- comma separated list of miner addresses
|
||||
@ -15,6 +15,7 @@ import (
|
||||
"go.opencensus.io/tag"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/journal/alerting"
|
||||
@ -200,7 +201,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
|
||||
currUrls = union(currUrls, si.URLs)
|
||||
|
||||
_, err = tx.Exec(
|
||||
"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, last_heartbeat=NOW() WHERE storage_id=$10",
|
||||
"UPDATE storage_path set urls=$1, weight=$2, max_storage=$3, can_seal=$4, can_store=$5, groups=$6, allow_to=$7, allow_types=$8, deny_types=$9, allow_miners=$10, deny_miners=$11, last_heartbeat=NOW() WHERE storage_id=$12",
|
||||
strings.Join(currUrls, ","),
|
||||
si.Weight,
|
||||
si.MaxStorage,
|
||||
@ -210,6 +211,8 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
|
||||
strings.Join(si.AllowTo, ","),
|
||||
strings.Join(si.AllowTypes, ","),
|
||||
strings.Join(si.DenyTypes, ","),
|
||||
strings.Join(si.AllowMiners, ","),
|
||||
strings.Join(si.DenyMiners, ","),
|
||||
si.ID)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("storage attach UPDATE fails: %v", err)
|
||||
@ -221,7 +224,7 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
|
||||
// Insert storage id
|
||||
_, err = tx.Exec(
|
||||
"INSERT INTO storage_path "+
|
||||
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW())",
|
||||
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), $16, $17)",
|
||||
si.ID,
|
||||
strings.Join(si.URLs, ","),
|
||||
si.Weight,
|
||||
@ -236,7 +239,9 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
|
||||
st.Available,
|
||||
st.FSAvailable,
|
||||
st.Reserved,
|
||||
st.Used)
|
||||
st.Used,
|
||||
strings.Join(si.AllowMiners, ","),
|
||||
strings.Join(si.DenyMiners, ","))
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("StorageAttach insert fails: %v", err)
|
||||
}
|
||||
@ -540,6 +545,8 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
|
||||
Groups string
|
||||
AllowTypes string
|
||||
DenyTypes string
|
||||
AllowMiners string
|
||||
DenyMiners string
|
||||
}
|
||||
err = dbi.harmonyDB.Select(ctx, &rows,
|
||||
`SELECT storage_id,
|
||||
@ -549,7 +556,9 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
|
||||
can_store,
|
||||
groups,
|
||||
allow_types,
|
||||
deny_types
|
||||
deny_types,
|
||||
allow_miners,
|
||||
deny_miners
|
||||
FROM storage_path
|
||||
WHERE can_seal=true
|
||||
and available >= $1
|
||||
@ -570,6 +579,53 @@ func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft st
|
||||
continue
|
||||
}
|
||||
|
||||
if len(row.AllowMiners) > 0 {
|
||||
found := false
|
||||
allowMiners := splitString(row.AllowMiners)
|
||||
for _, m := range allowMiners {
|
||||
m := m
|
||||
maddr, err := address.NewFromString(m)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == s.Miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
log.Debugf("not selecting on %s, not allowed by allow miners filters", row.StorageId)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(row.DenyMiners) > 0 {
|
||||
found := false
|
||||
denyMiners := splitString(row.DenyMiners)
|
||||
for _, m := range denyMiners {
|
||||
m := m
|
||||
maddr, err := address.NewFromString(m)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == s.Miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
log.Debugf("not selecting on %s, not allowed by deny miners filters", row.StorageId)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if allowList != nil {
|
||||
groups := splitString(row.Groups)
|
||||
allow := false
|
||||
@ -627,10 +683,12 @@ func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface
|
||||
AllowTo string
|
||||
AllowTypes string
|
||||
DenyTypes string
|
||||
AllowMiners string
|
||||
DenyMiners string
|
||||
}
|
||||
|
||||
err := dbi.harmonyDB.Select(ctx, &qResults,
|
||||
"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types "+
|
||||
"SELECT urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, allow_miners, deny_miners "+
|
||||
"FROM storage_path WHERE storage_id=$1", string(id))
|
||||
if err != nil {
|
||||
return storiface.StorageInfo{}, xerrors.Errorf("StorageInfo query fails: %v", err)
|
||||
@ -647,11 +705,13 @@ func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface
|
||||
sinfo.AllowTo = splitString(qResults[0].AllowTo)
|
||||
sinfo.AllowTypes = splitString(qResults[0].AllowTypes)
|
||||
sinfo.DenyTypes = splitString(qResults[0].DenyTypes)
|
||||
sinfo.AllowMiners = splitString(qResults[0].AllowMiners)
|
||||
sinfo.DenyMiners = splitString(qResults[0].DenyMiners)
|
||||
|
||||
return sinfo, nil
|
||||
}
|
||||
|
||||
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
|
||||
func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) {
|
||||
var err error
|
||||
var spaceReq uint64
|
||||
switch pathType {
|
||||
@ -677,6 +737,8 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
|
||||
AllowTo string
|
||||
AllowTypes string
|
||||
DenyTypes string
|
||||
AllowMiners string
|
||||
DenyMiners string
|
||||
}
|
||||
|
||||
err = dbi.harmonyDB.Select(ctx, &rows,
|
||||
@ -689,7 +751,9 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
|
||||
groups,
|
||||
allow_to,
|
||||
allow_types,
|
||||
deny_types
|
||||
deny_types,
|
||||
allow_miners,
|
||||
deny_miners
|
||||
FROM storage_path
|
||||
WHERE available >= $1
|
||||
and NOW()-($2 * INTERVAL '1 second') < last_heartbeat
|
||||
@ -700,13 +764,64 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
|
||||
SkippedHeartbeatThresh.Seconds(),
|
||||
pathType == storiface.PathSealing,
|
||||
pathType == storiface.PathStorage,
|
||||
miner.String(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("Querying for best storage sectors fails with err %w: ", err)
|
||||
}
|
||||
|
||||
var result []storiface.StorageInfo
|
||||
|
||||
for _, row := range rows {
|
||||
// Matching with 0 as a workaround to avoid having minerID
|
||||
// present when calling TaskStorage.HasCapacity()
|
||||
if !(miner == abi.ActorID(0)) {
|
||||
if len(row.AllowMiners) > 0 {
|
||||
found := false
|
||||
allowMiners := splitString(row.AllowMiners)
|
||||
for _, m := range allowMiners {
|
||||
m := m
|
||||
maddr, err := address.NewFromString(m)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(row.DenyMiners) > 0 {
|
||||
found := false
|
||||
denyMiners := splitString(row.DenyMiners)
|
||||
for _, m := range denyMiners {
|
||||
m := m
|
||||
maddr, err := address.NewFromString(m)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, storiface.StorageInfo{
|
||||
ID: storiface.ID(row.StorageId),
|
||||
URLs: splitString(row.Urls),
|
||||
@ -718,6 +833,8 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
|
||||
AllowTo: splitString(row.AllowTo),
|
||||
AllowTypes: splitString(row.AllowTypes),
|
||||
DenyTypes: splitString(row.DenyTypes),
|
||||
AllowMiners: splitString(row.AllowMiners),
|
||||
DenyMiners: splitString(row.DenyMiners),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import (
|
||||
"go.opencensus.io/tag"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
|
||||
@ -38,7 +39,7 @@ type SectorIndex interface { // part of storage-miner api
|
||||
StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error
|
||||
StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)
|
||||
|
||||
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)
|
||||
StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)
|
||||
|
||||
// atomically acquire locks on all sector file types. close ctx to unlock
|
||||
StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error
|
||||
@ -61,6 +62,7 @@ type storageEntry struct {
|
||||
heartbeatErr error
|
||||
}
|
||||
|
||||
// MemIndex represents an in-memory index of storage sectors and storage entries.
|
||||
type MemIndex struct {
|
||||
*indexLocks
|
||||
lk sync.RWMutex
|
||||
@ -206,6 +208,8 @@ func (i *MemIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
|
||||
i.stores[si.ID].info.AllowTo = si.AllowTo
|
||||
i.stores[si.ID].info.AllowTypes = allow
|
||||
i.stores[si.ID].info.DenyTypes = deny
|
||||
i.stores[si.ID].info.AllowMiners = si.AllowMiners
|
||||
i.stores[si.ID].info.DenyMiners = si.DenyMiners
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -478,6 +482,8 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
|
||||
|
||||
AllowTypes: st.info.AllowTypes,
|
||||
DenyTypes: st.info.DenyTypes,
|
||||
AllowMiners: st.info.AllowMiners,
|
||||
DenyMiners: st.info.DenyMiners,
|
||||
})
|
||||
}
|
||||
|
||||
@ -492,6 +498,52 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
|
||||
continue
|
||||
}
|
||||
|
||||
if len(st.info.AllowMiners) > 0 {
|
||||
found := false
|
||||
for _, m := range st.info.AllowMiners {
|
||||
minerIDStr := m
|
||||
maddr, err := address.NewFromString(minerIDStr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == s.Miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
log.Debugf("not allocating on %s, miner %s not allowed", st.info.ID, s.Miner.String())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if len(st.info.DenyMiners) > 0 {
|
||||
found := false
|
||||
for _, m := range st.info.DenyMiners {
|
||||
minerIDStr := m
|
||||
maddr, err := address.NewFromString(minerIDStr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == s.Miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
log.Debugf("not allocating on %s, miner %s denied", st.info.ID, s.Miner.String())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if spaceReq > uint64(st.fsi.Available) {
|
||||
log.Debugf("not selecting on %s, out of space (available: %d, need: %d)", st.info.ID, st.fsi.Available, spaceReq)
|
||||
continue
|
||||
@ -557,6 +609,8 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
|
||||
|
||||
AllowTypes: st.info.AllowTypes,
|
||||
DenyTypes: st.info.DenyTypes,
|
||||
AllowMiners: st.info.AllowMiners,
|
||||
DenyMiners: st.info.DenyMiners,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -564,6 +618,21 @@ func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft sto
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// StorageInfo retrieves the storage information for a given storage ID.
|
||||
//
|
||||
// The method first acquires a read lock on the MemIndex to ensure thread-safety.
|
||||
// It then checks if the storage ID exists in the stores map. If not, it returns
|
||||
// an error indicating that the sector store was not found.
|
||||
//
|
||||
// Finally, it returns the storage information of the selected storage.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: the context.Context object for cancellation and timeouts
|
||||
// - id: the ID of the storage to retrieve information for
|
||||
//
|
||||
// Returns:
|
||||
// - storiface.StorageInfo: the storage information of the selected storage ID
|
||||
// - error: an error indicating any issues encountered during the process
|
||||
func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error) {
|
||||
i.lk.RLock()
|
||||
defer i.lk.RUnlock()
|
||||
@ -576,7 +645,33 @@ func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.
|
||||
return *si.info, nil
|
||||
}
|
||||
|
||||
func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) {
|
||||
// StorageBestAlloc selects the best available storage options for allocating
|
||||
// a sector file. It takes into account the allocation type (sealing or storage),
|
||||
// sector size, and path type (sealing or storage).
|
||||
//
|
||||
// The method first estimates the required space for the allocation based on the
|
||||
// sector size and path type. It then iterates through all available storage options
|
||||
// and filters out those that cannot be used for the given path type. It also filters
|
||||
// out storage options that do not have enough available space or have not received
|
||||
// heartbeats within a certain threshold.
|
||||
//
|
||||
// The remaining storage options are sorted based on their available space and weight,
|
||||
// with higher availability and weight being prioritized. The method then returns
|
||||
// the information of the selected storage options.
|
||||
//
|
||||
// If no suitable storage options are found, it returns an error indicating that
|
||||
// no good path is available.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: the context.Context object for cancellation and timeouts
|
||||
// - allocate: the type of allocation (sealing or storage)
|
||||
// - ssize: the size of the sector file
|
||||
// - pathType: the path type (sealing or storage)
|
||||
//
|
||||
// Returns:
|
||||
// - []storiface.StorageInfo: the information of the selected storage options
|
||||
// - error: an error indicating any issues encountered during the process
|
||||
func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error) {
|
||||
i.lk.RLock()
|
||||
defer i.lk.RUnlock()
|
||||
|
||||
@ -604,6 +699,52 @@ func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sect
|
||||
continue
|
||||
}
|
||||
|
||||
if len(p.info.AllowMiners) > 0 {
|
||||
found := false
|
||||
for _, m := range p.info.AllowMiners {
|
||||
minerIDStr := m
|
||||
maddr, err := address.NewFromString(minerIDStr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
log.Debugf("not allocating on %s, miner %s not allowed", p.info.ID, miner.String())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if len(p.info.DenyMiners) > 0 {
|
||||
found := false
|
||||
for _, m := range p.info.DenyMiners {
|
||||
minerIDStr := m
|
||||
maddr, err := address.NewFromString(minerIDStr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("converting miner address to ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
log.Debugf("not allocating on %s, miner %s denied", p.info.ID, miner.String())
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if spaceReq > uint64(p.fsi.Available) {
|
||||
log.Debugf("not allocating on %s, out of space (available: %d, need: %d)", p.info.ID, p.fsi.Available, spaceReq)
|
||||
continue
|
||||
|
||||
@ -152,3 +152,61 @@ func TestFindAllow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageBestAlloc(t *testing.T) {
|
||||
idx := NewMemIndex(nil)
|
||||
|
||||
dummyStorageInfo := storiface.StorageInfo{
|
||||
ID: storiface.ID("dummy"),
|
||||
CanSeal: true,
|
||||
CanStore: true,
|
||||
URLs: []string{"http://localhost:9999/"},
|
||||
Weight: 10,
|
||||
AllowMiners: []string{
|
||||
"t001",
|
||||
},
|
||||
}
|
||||
|
||||
dummyFsStat := fsutil.FsStat{
|
||||
Capacity: 10000000000,
|
||||
Available: 7000000000,
|
||||
Reserved: 100000000,
|
||||
Used: 3000000000,
|
||||
}
|
||||
|
||||
err := idx.StorageAttach(context.Background(), dummyStorageInfo, dummyFsStat)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("PathSealing", func(t *testing.T) {
|
||||
result, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 1)
|
||||
require.Equal(t, err, nil)
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, len(result), 1)
|
||||
require.Equal(t, result[0].ID, dummyStorageInfo.ID)
|
||||
})
|
||||
|
||||
t.Run("PathStorage", func(t *testing.T) {
|
||||
result, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathStorage, 1)
|
||||
require.Equal(t, err, nil)
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, len(result), 1)
|
||||
require.Equal(t, result[0].ID, dummyStorageInfo.ID)
|
||||
})
|
||||
|
||||
t.Run("NotAllowedMiner", func(t *testing.T) {
|
||||
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 2)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("NoAvailableSpace", func(t *testing.T) {
|
||||
bigSectorSize := abi.SectorSize(10000000000)
|
||||
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, bigSectorSize, storiface.PathSealing, 1)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("AllowedMiner", func(t *testing.T) {
|
||||
_, err := idx.StorageBestAlloc(context.Background(), storiface.FTUnsealed, 123, storiface.PathSealing, 1)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@ import (
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/proof"
|
||||
|
||||
@ -214,6 +215,8 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
|
||||
AllowTo: meta.AllowTo,
|
||||
AllowTypes: meta.AllowTypes,
|
||||
DenyTypes: meta.DenyTypes,
|
||||
AllowMiners: meta.AllowMiners,
|
||||
DenyMiners: meta.DenyMiners,
|
||||
}, fst)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("declaring storage in index: %w", err)
|
||||
@ -304,6 +307,8 @@ func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMiss
|
||||
AllowTo: meta.AllowTo,
|
||||
AllowTypes: meta.AllowTypes,
|
||||
DenyTypes: meta.DenyTypes,
|
||||
AllowMiners: meta.AllowMiners,
|
||||
DenyMiners: meta.DenyMiners,
|
||||
}, fst)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("redeclaring storage in index: %w", err)
|
||||
@ -495,20 +500,64 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
|
||||
var out storiface.SectorPaths
|
||||
var storageIDs storiface.SectorPaths
|
||||
|
||||
allocPathOk := func(canSeal, canStore bool, allowTypes, denyTypes []string, fileType storiface.SectorFileType) bool {
|
||||
allocPathOk := func(canSeal, canStore bool, allowTypes, denyTypes, allowMiners, denyMiners []string, fileType storiface.SectorFileType, miner abi.ActorID) (bool, error) {
|
||||
if (pathType == storiface.PathSealing) && !canSeal {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if (pathType == storiface.PathStorage) && !canStore {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if !fileType.Allowed(allowTypes, denyTypes) {
|
||||
return false
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true
|
||||
if len(allowMiners) > 0 {
|
||||
found := false
|
||||
for _, m := range allowMiners {
|
||||
minerIDStr := m
|
||||
maddr, err := address.NewFromString(minerIDStr)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("converting miner address to ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
if len(denyMiners) > 0 {
|
||||
found := false
|
||||
for _, m := range denyMiners {
|
||||
minerIDStr := m
|
||||
maddr, err := address.NewFromString(minerIDStr)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("parsing miner address: %w", err)
|
||||
}
|
||||
mid, err := address.IDFromAddress(maddr)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("converting miner address to ID: %w", err)
|
||||
}
|
||||
if abi.ActorID(mid) == miner {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// First find existing files
|
||||
@ -536,9 +585,16 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
|
||||
continue
|
||||
}
|
||||
|
||||
if allocate.Has(fileType) && !allocPathOk(info.CanSeal, info.CanStore, info.AllowTypes, info.DenyTypes, fileType) {
|
||||
if allocate.Has(fileType) {
|
||||
ok, err := allocPathOk(info.CanSeal, info.CanStore, info.AllowTypes, info.DenyTypes, info.AllowMiners, info.DenyMiners, fileType, sid.ID.Miner)
|
||||
if err != nil {
|
||||
log.Debug(err)
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
continue // allocate request for a path of different type
|
||||
}
|
||||
}
|
||||
|
||||
spath := p.sectorPath(sid.ID, fileType)
|
||||
storiface.SetPathByType(&out, fileType, spath)
|
||||
@ -556,7 +612,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
|
||||
continue
|
||||
}
|
||||
|
||||
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType)
|
||||
sis, err := st.index.StorageBestAlloc(ctx, fileType, ssize, pathType, sid.ID.Miner)
|
||||
if err != nil {
|
||||
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("finding best storage for allocating : %w", err)
|
||||
}
|
||||
@ -574,7 +630,14 @@ func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, exi
|
||||
continue
|
||||
}
|
||||
|
||||
if !allocPathOk(si.CanSeal, si.CanStore, si.AllowTypes, si.DenyTypes, fileType) {
|
||||
alloc, err := allocPathOk(si.CanSeal, si.CanStore, si.AllowTypes, si.DenyTypes, si.AllowMiners, si.DenyMiners, fileType, sid.ID.Miner)
|
||||
|
||||
if err != nil {
|
||||
log.Debug(err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !alloc {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@ -54,18 +54,18 @@ func (mr *MockSectorIndexMockRecorder) StorageAttach(arg0, arg1, arg2 interface{
|
||||
}
|
||||
|
||||
// StorageBestAlloc mocks base method.
|
||||
func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType) ([]storiface.StorageInfo, error) {
|
||||
func (m *MockSectorIndex) StorageBestAlloc(arg0 context.Context, arg1 storiface.SectorFileType, arg2 abi.SectorSize, arg3 storiface.PathType, arg4 abi.ActorID) ([]storiface.StorageInfo, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3)
|
||||
ret := m.ctrl.Call(m, "StorageBestAlloc", arg0, arg1, arg2, arg3, arg4)
|
||||
ret0, _ := ret[0].([]storiface.StorageInfo)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StorageBestAlloc indicates an expected call of StorageBestAlloc.
|
||||
func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
|
||||
func (mr *MockSectorIndexMockRecorder) StorageBestAlloc(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageBestAlloc", reflect.TypeOf((*MockSectorIndex)(nil).StorageBestAlloc), arg0, arg1, arg2, arg3, arg4)
|
||||
}
|
||||
|
||||
// StorageDeclareSector mocks base method.
|
||||
|
||||
@ -356,7 +356,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto
|
||||
|
||||
// selector will schedule the Unseal task on a worker that either already has the sealed sector files or has space in
|
||||
// one of it's sealing scratch spaces to store them after fetching them from another worker.
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, true)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, sector.ID.Miner, true)
|
||||
|
||||
log.Debugf("will schedule unseal for sector %d", sector.ID)
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error {
|
||||
@ -377,7 +377,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storiface.Secto
|
||||
}
|
||||
|
||||
// get a selector for moving unsealed sector into long-term storage
|
||||
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTUnsealed, storiface.PathStorage, !m.disallowRemoteFinalize)
|
||||
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTUnsealed, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
|
||||
|
||||
// move unsealed sector to long-term storage
|
||||
// Possible TODO: Add an option to not keep the unsealed sector in long term storage?
|
||||
@ -431,9 +431,9 @@ func (m *Manager) AddPiece(ctx context.Context, sector storiface.SectorRef, exis
|
||||
var selector WorkerSelector
|
||||
var err error
|
||||
if len(existingPieces) == 0 { // new
|
||||
selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing)
|
||||
selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing, sector.ID.Miner)
|
||||
} else { // use existing
|
||||
selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false)
|
||||
selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, sector.ID.Miner, false)
|
||||
}
|
||||
|
||||
var out abi.PieceInfo
|
||||
@ -484,7 +484,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector storiface.SectorRef
|
||||
|
||||
// TODO: also consider where the unsealed data sits
|
||||
|
||||
selector := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed, storiface.PathSealing)
|
||||
selector := newAllocSelector(m.index, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, sector.ID.Miner)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit1, selector, m.schedFetch(sector, storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
|
||||
err := m.startWork(ctx, w, wk)(w.SealPreCommit1(ctx, sector, ticket, pieces))
|
||||
@ -533,7 +533,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector storiface.SectorRef
|
||||
return storiface.SectorCids{}, xerrors.Errorf("acquiring sector lock: %w", err)
|
||||
}
|
||||
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, true)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, sector.ID.Miner, true)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit2, selector, m.schedFetch(sector, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
|
||||
err := m.startWork(ctx, w, wk)(w.SealPreCommit2(ctx, sector, phase1Out))
|
||||
@ -585,7 +585,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sector storiface.SectorRef, t
|
||||
// NOTE: We set allowFetch to false in so that we always execute on a worker
|
||||
// with direct access to the data. We want to do that because this step is
|
||||
// generally very cheap / fast, and transferring data is not worth the effort
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, false)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, sector.ID.Miner, false)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTCommit1, selector, m.schedFetch(sector, storiface.FTCache|storiface.FTSealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
|
||||
err := m.startWork(ctx, w, wk)(w.SealCommit1(ctx, sector, ticket, seed, pieces, cids))
|
||||
@ -700,7 +700,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storiface.SectorRef
|
||||
|
||||
// do the cache trimming wherever the likely still very large cache lives.
|
||||
// we really don't want to move it.
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache, false)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTCache, sector.ID.Miner, false)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTFinalize, selector,
|
||||
m.schedFetch(sector, storiface.FTCache, cachePathType, storiface.AcquireMove),
|
||||
@ -713,7 +713,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector storiface.SectorRef
|
||||
}
|
||||
|
||||
// get a selector for moving stuff into long-term storage
|
||||
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, storiface.PathStorage, !m.disallowRemoteFinalize)
|
||||
fetchSel := newMoveSelector(m.index, sector.ID, storiface.FTCache|storiface.FTSealed, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
|
||||
|
||||
// only move the unsealed file if it still exists and needs moving
|
||||
moveUnsealed := storiface.FTUnsealed
|
||||
@ -769,7 +769,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
|
||||
|
||||
// do the cache trimming wherever the likely still large cache lives.
|
||||
// we really don't want to move it.
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdateCache, false)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdateCache, sector.ID.Miner, false)
|
||||
|
||||
err := m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeReplicaUpdate, selector,
|
||||
m.schedFetch(sector, storiface.FTCache|storiface.FTUpdateCache, pathType, storiface.AcquireMove),
|
||||
@ -783,7 +783,7 @@ func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storiface.Se
|
||||
|
||||
move := func(types storiface.SectorFileType) error {
|
||||
// get a selector for moving stuff into long-term storage
|
||||
fetchSel := newMoveSelector(m.index, sector.ID, types, storiface.PathStorage, !m.disallowRemoteFinalize)
|
||||
fetchSel := newMoveSelector(m.index, sector.ID, types, storiface.PathStorage, sector.ID.Miner, !m.disallowRemoteFinalize)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTFetch, fetchSel,
|
||||
m.schedFetch(sector, types, storiface.PathStorage, storiface.AcquireMove),
|
||||
@ -834,7 +834,7 @@ func (m *Manager) ReleaseUnsealed(ctx context.Context, sector storiface.SectorRe
|
||||
return nil
|
||||
}
|
||||
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, sector.ID.Miner, false)
|
||||
|
||||
return m.sched.Schedule(ctx, sector, sealtasks.TTFinalizeUnsealed, selector, m.schedFetch(sector, storiface.FTUnsealed, pathType, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
|
||||
_, err := m.waitSimpleCall(ctx)(w.ReleaseUnsealed(ctx, sector, keepUnsealed))
|
||||
@ -906,7 +906,7 @@ func (m *Manager) GenerateSectorKeyFromData(ctx context.Context, sector storifac
|
||||
// NOTE: We set allowFetch to false in so that we always execute on a worker
|
||||
// with direct access to the data. We want to do that because this step is
|
||||
// generally very cheap / fast, and transferring data is not worth the effort
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, true)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, sector.ID.Miner, true)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTRegenSectorKey, selector, m.schedFetch(sector, storiface.FTUpdate|storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error {
|
||||
err := m.startWork(ctx, w, wk)(w.GenerateSectorKeyFromData(ctx, sector, commD))
|
||||
@ -984,7 +984,7 @@ func (m *Manager) ReplicaUpdate(ctx context.Context, sector storiface.SectorRef,
|
||||
return storiface.ReplicaUpdateOut{}, xerrors.Errorf("acquiring sector lock: %w", err)
|
||||
}
|
||||
|
||||
selector := newAllocSelector(m.index, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing)
|
||||
selector := newAllocSelector(m.index, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, sector.ID.Miner)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTReplicaUpdate, selector, m.schedFetch(sector, storiface.FTUnsealed|storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
|
||||
err := m.startWork(ctx, w, wk)(w.ReplicaUpdate(ctx, sector, pieces))
|
||||
@ -1035,7 +1035,7 @@ func (m *Manager) ProveReplicaUpdate1(ctx context.Context, sector storiface.Sect
|
||||
// NOTE: We set allowFetch to false in so that we always execute on a worker
|
||||
// with direct access to the data. We want to do that because this step is
|
||||
// generally very cheap / fast, and transferring data is not worth the effort
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, false)
|
||||
selector := newExistingSelector(m.index, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, sector.ID.Miner, false)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTProveReplicaUpdate1, selector, m.schedFetch(sector, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy), func(ctx context.Context, w Worker) error {
|
||||
|
||||
@ -1153,7 +1153,7 @@ func (m *Manager) DownloadSectorData(ctx context.Context, sector storiface.Secto
|
||||
ptype = storiface.PathStorage
|
||||
}
|
||||
|
||||
selector := newAllocSelector(m.index, toFetch, ptype)
|
||||
selector := newAllocSelector(m.index, toFetch, ptype, sector.ID.Miner)
|
||||
|
||||
err = m.sched.Schedule(ctx, sector, sealtasks.TTDownloadSector, selector, schedNop, func(ctx context.Context, w Worker) error {
|
||||
err := m.startWork(ctx, w, wk)(w.DownloadSectorData(ctx, sector, finalized, src))
|
||||
|
||||
@ -274,7 +274,7 @@ func TestSched(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
rm.done[taskName] = done
|
||||
|
||||
sel := newAllocSelector(index, storiface.FTCache, storiface.PathSealing)
|
||||
sel := newAllocSelector(index, storiface.FTCache, storiface.PathSealing, abi.ActorID(1000))
|
||||
|
||||
rm.wg.Add(1)
|
||||
go func() {
|
||||
|
||||
@ -16,13 +16,15 @@ type allocSelector struct {
|
||||
index paths.SectorIndex
|
||||
alloc storiface.SectorFileType
|
||||
ptype storiface.PathType
|
||||
miner abi.ActorID
|
||||
}
|
||||
|
||||
func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType) *allocSelector {
|
||||
func newAllocSelector(index paths.SectorIndex, alloc storiface.SectorFileType, ptype storiface.PathType, miner abi.ActorID) *allocSelector {
|
||||
return &allocSelector{
|
||||
index: index,
|
||||
alloc: alloc,
|
||||
ptype: ptype,
|
||||
miner: miner,
|
||||
}
|
||||
}
|
||||
|
||||
@ -50,7 +52,7 @@ func (s *allocSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi
|
||||
return false, false, xerrors.Errorf("getting sector size: %w", err)
|
||||
}
|
||||
|
||||
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype)
|
||||
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.ptype, s.miner)
|
||||
if err != nil {
|
||||
return false, false, xerrors.Errorf("finding best alloc storage: %w", err)
|
||||
}
|
||||
|
||||
@ -16,14 +16,16 @@ type existingSelector struct {
|
||||
index paths.SectorIndex
|
||||
sector abi.SectorID
|
||||
fileType storiface.SectorFileType
|
||||
miner abi.ActorID
|
||||
allowFetch bool
|
||||
}
|
||||
|
||||
func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, allowFetch bool) *existingSelector {
|
||||
func newExistingSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, miner abi.ActorID, allowFetch bool) *existingSelector {
|
||||
return &existingSelector{
|
||||
index: index,
|
||||
sector: sector,
|
||||
fileType: alloc,
|
||||
miner: miner,
|
||||
allowFetch: allowFetch,
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,10 +17,11 @@ type moveSelector struct {
|
||||
sector abi.SectorID
|
||||
alloc storiface.SectorFileType
|
||||
destPtype storiface.PathType
|
||||
miner abi.ActorID
|
||||
allowRemote bool
|
||||
}
|
||||
|
||||
func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, allowRemote bool) *moveSelector {
|
||||
func newMoveSelector(index paths.SectorIndex, sector abi.SectorID, alloc storiface.SectorFileType, destPtype storiface.PathType, miner abi.ActorID, allowRemote bool) *moveSelector {
|
||||
return &moveSelector{
|
||||
index: index,
|
||||
sector: sector,
|
||||
@ -67,7 +68,7 @@ func (s *moveSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.
|
||||
}
|
||||
}
|
||||
|
||||
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.destPtype)
|
||||
best, err := s.index.StorageBestAlloc(ctx, s.alloc, ssize, s.destPtype, s.miner)
|
||||
if err != nil {
|
||||
return false, false, xerrors.Errorf("finding best dest storage: %w", err)
|
||||
}
|
||||
|
||||
@ -8,6 +8,28 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
)
|
||||
|
||||
// FTUnsealed represents an unsealed sector file type.
|
||||
// FTSealed represents a sealed sector file type.
|
||||
// FTCache represents a cache sector file type.
|
||||
// FTUpdate represents an update sector file type.
|
||||
// FTUpdateCache represents an update cache sector file type.
|
||||
// FTPiece represents a Piece Park sector file type.
|
||||
// FileTypes represents the total number of file types.
|
||||
//
|
||||
// The SectorFileType type is an integer type that represents different sector file types.
|
||||
// It has several methods to manipulate and query the file type.
|
||||
// The String method returns a string representation of the file type.
|
||||
// The Strings method returns a slice of string representations of all the file types that are set in the receiver object.
|
||||
// The AllSet method returns a slice of all the file types that are set in the receiver object.
|
||||
// The Has method checks whether a specific file type is set in the receiver object.
|
||||
// The SealSpaceUse method calculates the space used by the receiver object in sealing a sector of a given size.
|
||||
// The SubAllowed method removes selected file types from the receiver object based on a list of allowed and denied file types.
|
||||
// The Unset method removes selected file types from the receiver object.
|
||||
// The AnyAllowed method checks whether any file types in the receiver object are allowed based on a list of allowed and denied file types.
|
||||
// The Allowed method checks whether all file types in the receiver object are allowed based on a list of allowed and denied file types.
|
||||
// The StoreSpaceUse method calculates the space used by the receiver object in storing a sector of a given size.
|
||||
// The All method returns an array that represents which file types are set in the receiver object.
|
||||
// The IsNone method checks whether the receiver object represents no file types.
|
||||
const (
|
||||
// "regular" sectors
|
||||
FTUnsealed SectorFileType = 1 << iota
|
||||
@ -24,12 +46,28 @@ const (
|
||||
FileTypes = iota
|
||||
)
|
||||
|
||||
// PathTypes is a slice of SectorFileType that represents different types of sector file paths.
|
||||
// It contains the following types of sector file paths:
|
||||
// - FTUnsealed: represents regular unsealed sectors
|
||||
// - FTSealed: represents sealed sectors
|
||||
// - FTCache: represents cache sectors
|
||||
// - FTUpdate: represents snap sectors
|
||||
// - FTUpdateCache: represents snap cache sectors
|
||||
// - FTPiece: represents Piece Park sectors
|
||||
var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache, FTUpdate, FTUpdateCache, FTPiece}
|
||||
|
||||
// FTNone represents a sector file type of none. This constant is used in the StorageLock method to specify that a sector should not have any file types locked.
|
||||
// Example usage:
|
||||
// err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache)
|
||||
const (
|
||||
FTNone SectorFileType = 0
|
||||
)
|
||||
|
||||
// FTAll represents the combination of all available sector file types.
|
||||
// It is a variable of type SectorFileType.
|
||||
// The value of FTAll is calculated by iterating over the PathTypes slice and using the |= operator to perform a bitwise OR operation on each path type.
|
||||
// The result is assigned to the variable out and returned.
|
||||
// FTAll is immediately invoked as a function using the anonymous function syntax, so the result is returned as soon as it is calculated.
|
||||
var FTAll = func() (out SectorFileType) {
|
||||
for _, pathType := range PathTypes {
|
||||
out |= pathType
|
||||
@ -37,8 +75,10 @@ var FTAll = func() (out SectorFileType) {
|
||||
return out
|
||||
}()
|
||||
|
||||
// FSOverheadDen represents the constant value 10, which is used to calculate the overhead in various storage space utilization calculations.
|
||||
const FSOverheadDen = 10
|
||||
|
||||
// FSOverheadSeal is a map that represents the overheads for different SectorFileType in sealed sectors.
|
||||
var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
|
||||
FTUnsealed: FSOverheadDen,
|
||||
FTSealed: FSOverheadDen,
|
||||
@ -50,6 +90,30 @@ var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
|
||||
|
||||
// sector size * disk / fs overhead. FSOverheadDen is like the unit of sector size
|
||||
|
||||
// FsOverheadFinalized is a map that represents the finalized overhead for different types of SectorFileType.
|
||||
// The keys in the map are the SectorFileType values, and the values are integers representing the overhead.
|
||||
// It is used to calculate the storage space usage for different types of sectors, as shown in the example below:
|
||||
//
|
||||
// func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
|
||||
// var need uint64
|
||||
// for _, pathType := range PathTypes {
|
||||
// if !t.Has(pathType) {
|
||||
// continue
|
||||
// }
|
||||
//
|
||||
// oh, ok := FsOverheadFinalized[pathType]
|
||||
// if !ok {
|
||||
// return 0, xerrors.Errorf("no finalized overhead info for %s", pathType)
|
||||
// }
|
||||
//
|
||||
// need += uint64(oh) * uint64(ssize) / FSOverheadDen
|
||||
// }
|
||||
//
|
||||
// return need, nil
|
||||
// }
|
||||
//
|
||||
// The overhead value is retrieved from FsOverheadFinalized by using the SectorFileType value as the key.
|
||||
// If the overhead value is not found in the map, an error is returned indicating that there is no finalized overhead information for the given sector type.
|
||||
var FsOverheadFinalized = map[SectorFileType]int{
|
||||
FTUnsealed: FSOverheadDen,
|
||||
FTSealed: FSOverheadDen,
|
||||
@ -59,8 +123,13 @@ var FsOverheadFinalized = map[SectorFileType]int{
|
||||
FTPiece: FSOverheadDen,
|
||||
}
|
||||
|
||||
// SectorFileType represents the type of a sector file
|
||||
// TypeFromString converts a string to a SectorFileType
|
||||
type SectorFileType int
|
||||
|
||||
// TypeFromString converts a string representation of a SectorFileType to its corresponding value.
|
||||
// It returns the SectorFileType and nil error if the string matches one of the existing types.
|
||||
// If the string does not match any type, it returns 0 and an error.
|
||||
func TypeFromString(s string) (SectorFileType, error) {
|
||||
switch s {
|
||||
case "unsealed":
|
||||
@ -80,6 +149,7 @@ func TypeFromString(s string) (SectorFileType, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// String returns a string representation of the SectorFileType.
|
||||
func (t SectorFileType) String() string {
|
||||
switch t {
|
||||
case FTUnsealed:
|
||||
@ -99,6 +169,12 @@ func (t SectorFileType) String() string {
|
||||
}
|
||||
}
|
||||
|
||||
// Strings returns a slice of strings representing the names of the SectorFileType values that are set in the receiver value.
|
||||
// Example usage:
|
||||
//
|
||||
// fileType := SectorFileType(FTSealed | FTCache)
|
||||
// names := fileType.Strings() // names = ["sealed", "cache"]
|
||||
// fmt.Println(names)
|
||||
func (t SectorFileType) Strings() []string {
|
||||
var out []string
|
||||
for _, fileType := range PathTypes {
|
||||
@ -111,6 +187,7 @@ func (t SectorFileType) Strings() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// AllSet returns a slice of SectorFileType values that are set in the SectorFileType receiver value
|
||||
func (t SectorFileType) AllSet() []SectorFileType {
|
||||
var out []SectorFileType
|
||||
for _, fileType := range PathTypes {
|
||||
@ -123,10 +200,33 @@ func (t SectorFileType) AllSet() []SectorFileType {
|
||||
return out
|
||||
}
|
||||
|
||||
// Has checks if the SectorFileType has a specific singleType.
|
||||
func (t SectorFileType) Has(singleType SectorFileType) bool {
|
||||
return t&singleType == singleType
|
||||
}
|
||||
|
||||
// SealSpaceUse calculates the amount of space needed for sealing the sector
|
||||
// based on the given sector size. It iterates over the different path types
|
||||
// and calculates the space needed for each path type using the FSOverheadSeal
|
||||
// map. The overhead value is multiplied by the sector size and divided by the
|
||||
// FSOverheadDen constant. The total space needed is accumulated and returned.
|
||||
// If there is no seal overhead information for a particular path type, an error
|
||||
// is returned.
|
||||
//
|
||||
// Example usage:
|
||||
//
|
||||
// fileType := FTSealed | FTCache
|
||||
// sectorSize := abi.SectorSize(32 << 20) // 32 MiB
|
||||
// spaceNeeded, err := fileType.SealSpaceUse(sectorSize)
|
||||
//
|
||||
// Parameters:
|
||||
//
|
||||
// ssize: The size of the sector
|
||||
//
|
||||
// Returns:
|
||||
//
|
||||
// uint64: The amount of space needed for sealing the sector
|
||||
// error: If there is no seal overhead information for a path type
|
||||
func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
|
||||
var need uint64
|
||||
for _, pathType := range PathTypes {
|
||||
@ -145,6 +245,15 @@ func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
|
||||
return need, nil
|
||||
}
|
||||
|
||||
// The method takes in two parameters: allowTypes and denyTypes, both of which are slices of strings.
|
||||
// If allowTypes is not empty, the method sets a denyMask with all bits set to 1, and then iterates over each allowType,
|
||||
// converting it to a SectorFileType using the TypeFromString function and unsetting the corresponding bit in the denyMask.
|
||||
// If a string in allowTypes cannot be converted to a valid SectorFileType, it is ignored.
|
||||
// After processing allowTypes, the method iterates over each denyType, converting it to a SectorFileType using the TypeFromString function
|
||||
// and setting the corresponding bit in the denyMask.
|
||||
// If a string in denyTypes cannot be converted to a valid SectorFileType, it is ignored.
|
||||
// Finally, the method returns the bitwise AND of the original SectorFileType and the denyMask.
|
||||
// The returned SectorFileType will only allow the types specified in allowTypes and exclude the types specified in denyTypes.`
|
||||
func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType {
|
||||
var denyMask SectorFileType // 1s deny
|
||||
|
||||
@ -174,18 +283,30 @@ func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) Sect
|
||||
return t & denyMask
|
||||
}
|
||||
|
||||
// Unset removes the specified sector file type(s) from the current SectorFileType value.
|
||||
// It performs a bitwise AND operation between the current value and the bitwise complement of the toUnset value.
|
||||
// The result is returned as a new SectorFileType value.
|
||||
// Any bits that are set in toUnset will be cleared in the result.
|
||||
// Usage: result = value.Unset(typesToUnset)
|
||||
func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType {
|
||||
return t &^ toUnset
|
||||
}
|
||||
|
||||
// AnyAllowed checks if the SectorFileType has any allowed types and no denied types.
|
||||
func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool {
|
||||
return t.SubAllowed(allowTypes, denyTypes) != t
|
||||
}
|
||||
|
||||
// Allowed checks if the SectorFileType is allowed based on the given allowTypes and denyTypes.
|
||||
// Returns true if the SectorFileType is allowed, otherwise false.
|
||||
func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool {
|
||||
return t.SubAllowed(allowTypes, denyTypes) == 0
|
||||
}
|
||||
|
||||
// StoreSpaceUse calculates the space used for storing sectors of a specific file type.
|
||||
// It takes the sector size as input and returns the total space needed in bytes and an error, if any.
|
||||
// The calculation is based on the finalized overhead information for the file type.
|
||||
// If the overhead information is not available for a particular file type, an error will be returned.
|
||||
func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
|
||||
var need uint64
|
||||
for _, pathType := range PathTypes {
|
||||
@ -204,6 +325,7 @@ func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) {
|
||||
return need, nil
|
||||
}
|
||||
|
||||
// All returns an array indicating whether each FileTypes flag is set in the SectorFileType.
|
||||
func (t SectorFileType) All() [FileTypes]bool {
|
||||
var out [FileTypes]bool
|
||||
|
||||
@ -214,10 +336,13 @@ func (t SectorFileType) All() [FileTypes]bool {
|
||||
return out
|
||||
}
|
||||
|
||||
// IsNone checks if the SectorFileType value is equal to zero.
|
||||
// It returns true if the value is zero, indicating that the type is none.
|
||||
func (t SectorFileType) IsNone() bool {
|
||||
return t == 0
|
||||
}
|
||||
|
||||
// SectorPaths represents the paths for different sector files.
|
||||
type SectorPaths struct {
|
||||
ID abi.SectorID
|
||||
|
||||
@ -229,6 +354,7 @@ type SectorPaths struct {
|
||||
Piece string
|
||||
}
|
||||
|
||||
// HasAllSet checks if all paths of a SectorPaths struct are set for a given SectorFileType.
|
||||
func (sp SectorPaths) HasAllSet(ft SectorFileType) bool {
|
||||
for _, fileType := range ft.AllSet() {
|
||||
if PathByType(sp, fileType) == "" {
|
||||
@ -239,6 +365,9 @@ func (sp SectorPaths) HasAllSet(ft SectorFileType) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Subset returns a new instance of SectorPaths that contains only the paths specified by the filter SectorFileType.
|
||||
// It iterates over each fileType in the filter, retrieves the corresponding path from the original SectorPaths instance, and sets it in the new instance.
|
||||
// Finally, it sets the ID field of the new instance to be the same as the original instance.
|
||||
func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths {
|
||||
var out SectorPaths
|
||||
|
||||
@ -251,6 +380,24 @@ func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths {
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseSectorID parses a sector ID from a given base name.
|
||||
// It expects the format "s-t0%d-%d", where the first %d represents the miner ID
|
||||
// and the second %d represents the sector number.
|
||||
//
|
||||
// Parameters:
|
||||
// - baseName: The base name from which to parse the sector ID.
|
||||
//
|
||||
// Returns:
|
||||
// - abi.SectorID: The parsed sector ID.
|
||||
// - error: An error if parsing fails.
|
||||
//
|
||||
// Example usage:
|
||||
//
|
||||
// id, err := ParseSectorID(baseName)
|
||||
// if err != nil {
|
||||
// // handle error
|
||||
// }
|
||||
// // use id
|
||||
func ParseSectorID(baseName string) (abi.SectorID, error) {
|
||||
var n abi.SectorNumber
|
||||
var mid abi.ActorID
|
||||
@ -269,10 +416,19 @@ func ParseSectorID(baseName string) (abi.SectorID, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SectorName returns the name of a sector in the format "s-t0<Miner>-<Number>"
|
||||
//
|
||||
// Parameters:
|
||||
// - sid: The sector ID
|
||||
//
|
||||
// Returns:
|
||||
// - The name of the sector as a string
|
||||
func SectorName(sid abi.SectorID) string {
|
||||
return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number)
|
||||
}
|
||||
|
||||
// PathByType returns the path associated with the specified fileType in the given SectorPaths.
|
||||
// It panics if the requested path type is unknown.
|
||||
func PathByType(sps SectorPaths, fileType SectorFileType) string {
|
||||
switch fileType {
|
||||
case FTUnsealed:
|
||||
@ -309,11 +465,14 @@ func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string) {
|
||||
}
|
||||
}
|
||||
|
||||
// PathsWithIDs represents paths and IDs for sector files.
|
||||
type PathsWithIDs struct {
|
||||
Paths SectorPaths
|
||||
IDs SectorPaths
|
||||
}
|
||||
|
||||
// HasAllSet checks if all paths and IDs in PathsWithIDs have a corresponding path set for the specified SectorFileType.
|
||||
// It returns true if all paths and IDs are set, and false otherwise.
|
||||
func (p PathsWithIDs) HasAllSet(ft SectorFileType) bool {
|
||||
return p.Paths.HasAllSet(ft) && p.IDs.HasAllSet(ft)
|
||||
}
|
||||
|
||||
@ -86,6 +86,14 @@ type StorageInfo struct {
|
||||
// - "update-cache"
|
||||
// Any other value will generate a warning and be ignored.
|
||||
DenyTypes []string
|
||||
|
||||
// AllowMiners lists miner IDs which are allowed to store their sector data into
|
||||
// this path. If empty, all miner IDs are allowed
|
||||
AllowMiners []string
|
||||
|
||||
// DenyMiners lists miner IDs which are denied to store their sector data into
|
||||
// this path
|
||||
DenyMiners []string
|
||||
}
|
||||
|
||||
type HealthReport struct {
|
||||
@ -106,6 +114,8 @@ type SectorStorageInfo struct {
|
||||
|
||||
AllowTypes []string
|
||||
DenyTypes []string
|
||||
AllowMiners []string
|
||||
DenyMiners []string
|
||||
}
|
||||
|
||||
type Decl struct {
|
||||
|
||||
@ -225,4 +225,12 @@ type LocalStorageMeta struct {
|
||||
// - "update-cache"
|
||||
// Any other value will generate a warning and be ignored.
|
||||
DenyTypes []string
|
||||
|
||||
// AllowMiners lists miner IDs which are allowed to store their sector data into
|
||||
// this path. If empty, all miner IDs are allowed
|
||||
AllowMiners []string
|
||||
|
||||
// DenyMiners lists miner IDs which are denied to store their sector data into
|
||||
// this path
|
||||
DenyMiners []string
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user